mirror of
https://github.com/pgsty/minio.git
synced 2026-11-10 17:45:32 +03:00
Compare commits
631 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 099b7ddf37 | |||
| c054569b9f | |||
| af3064f2b5 | |||
| 9d2be5daaa | |||
| 1f3377862b | |||
| 16738c2301 | |||
| 53a8f7583f | |||
| 657fc5e827 | |||
| 6a7beb8a41 | |||
| bd39bab339 | |||
| 04d7d91973 | |||
| 7a13780d81 | |||
| b8ebaffd2e | |||
| 5cb91e7d74 | |||
| 07b89b98fa | |||
| 378f281bef | |||
| 5fb983d3d2 | |||
| 9f9398a1a3 | |||
| 1516273162 | |||
| 210fd568cc | |||
| f3e84d54a9 | |||
| 479a49b28b | |||
| 8cc2360c3a | |||
| 41b71a8bc1 | |||
| 00270e9f61 | |||
| c78ef6a6f6 | |||
| e9da6f91f4 | |||
| 6434d768de | |||
| b524fc4fb7 | |||
| bdd17b0f47 | |||
| ade15ea5d0 | |||
| 98fd7004e8 | |||
| 96b3a19d0a | |||
| db0c0ffedd | |||
| 7af07298ce | |||
| b10fd0e9c0 | |||
| 852e2a6707 | |||
| 04c657d03f | |||
| 3268fbea7e | |||
| 94d91703b5 | |||
| 1eee720f74 | |||
| 1780eb8ffc | |||
| b32a389e9d | |||
| 2a66ab65d7 | |||
| 0c6aaca6e2 | |||
| 1bd0b1b04d | |||
| d56b1063b9 | |||
| a4453a1a55 | |||
| 617d8ed67b | |||
| 928c827508 | |||
| f4b6c04c9f | |||
| 54186040ca | |||
| 69a4f938a2 | |||
| 3e954ad06e | |||
| c45b26a75c | |||
| 77ad2fc12b | |||
| adc824087b | |||
| 21b6711562 | |||
| 9fe32b7743 | |||
| ae54718d84 | |||
| 303e340292 | |||
| a717711b11 | |||
| 57c2b0fb76 | |||
| 9fefb756ad | |||
| a7e7682e1b | |||
| 09d6a22993 | |||
| b7062cae75 | |||
| e5bcbc7d82 | |||
| b3a5307980 | |||
| 45b1e4bc00 | |||
| 14d49f14f7 | |||
| 584bb3cc53 | |||
| f85d4a5fa1 | |||
| 67a6532dec | |||
| 28a9e67ac7 | |||
| 7fac646923 | |||
| 32ab506924 | |||
| 7f38bf6282 | |||
| 2b228f678c | |||
| 73025824a3 | |||
| 0fa4a8c503 | |||
| dd0a2da55e | |||
| c8dd5678bb | |||
| 9f7b49369d | |||
| ed05d982a1 | |||
| f321f7f1c7 | |||
| 82dd12faf8 | |||
| 3ddf83182f | |||
| 374b9ce309 | |||
| 0c5a309507 | |||
| f83d02831c | |||
| 496eab4b76 | |||
| add36e7052 | |||
| ea483e453e | |||
| ce60cc0c84 | |||
| 997846f413 | |||
| 7c9b6386c6 | |||
| ba0b59d0b2 | |||
| d9837fb319 | |||
| 8579466557 | |||
| 4f72d0ea94 | |||
| 3c4980eb80 | |||
| 50a234d6c3 | |||
| a02ce222f0 | |||
| 8ae19de7a8 | |||
| 7a4f0b5e69 | |||
| 5ac27f7533 | |||
| 3c8699e216 | |||
| fa42a09826 | |||
| fcbf2c7d49 | |||
| 18ba0ef32a | |||
| 65c59dd860 | |||
| e7103d77cb | |||
| 167b95dfd8 | |||
| 3a1e132f16 | |||
| 30588eb830 | |||
| 393a84076f | |||
| 69fcf50cd2 | |||
| a21f420de3 | |||
| 6da8036eb3 | |||
| dcad2ae3e7 | |||
| 9897589bc5 | |||
| 525ab5fa40 | |||
| d4e7716085 | |||
| a803027b09 | |||
| 001078e9ca | |||
| 8c1473a618 | |||
| 347d9763fd | |||
| 0fc5a0b527 | |||
| a47bacbc0f | |||
| 0a546c7a76 | |||
| 8dac7f7c14 | |||
| 4cc3bc1de6 | |||
| 0cfca29cb2 | |||
| a7310d59fe | |||
| a995921e57 | |||
| 19910230aa | |||
| 7ce7fc8d1e | |||
| fb1a619ee1 | |||
| 3f13c26eed | |||
| dd905b29f4 | |||
| 2f728c7643 | |||
| c3ad836aed | |||
| dc6dc3d806 | |||
| dcc2f20ba8 | |||
| a13efd5760 | |||
| 39278e068e | |||
| 263ee0510d | |||
| 641eb51a40 | |||
| 45d6e5d065 | |||
| dabc894132 | |||
| fb08ca7b6a | |||
| 41822710e4 | |||
| 034f7bbf73 | |||
| 79eca5e920 | |||
| aec7ddac46 | |||
| 475640e7e1 | |||
| 62124c779c | |||
| 030c6d740e | |||
| 5f1f4178ae | |||
| 0828861bd0 | |||
| 6921f8c186 | |||
| 25d928c9df | |||
| ee3e7ea716 | |||
| 2a04d5f485 | |||
| 9449e6a4a8 | |||
| 9c4bb39571 | |||
| 86ce8d6012 | |||
| b15aa97c94 | |||
| f54c04846e | |||
| 6fdea3b8d9 | |||
| 380fcbfe47 | |||
| a02fa9644c | |||
| 652a1c09d3 | |||
| c86df14f4d | |||
| 03875d61e3 | |||
| 1865985aa1 | |||
| 076a13f9d2 | |||
| 9a4e089f56 | |||
| 9255f443a6 | |||
| ea7c794a94 | |||
| 16c21428d9 | |||
| 7e83f9950a | |||
| 1e3ab56b06 | |||
| 5ad1f344b7 | |||
| 8a4097309f | |||
| 0501a198e3 | |||
| d2eb0ade91 | |||
| 5c074efd1d | |||
| 03fe5e9c21 | |||
| 0b1c21cb4b | |||
| b55495aef0 | |||
| d8b77e5f3b | |||
| cd1f6cec3c | |||
| 4f17a23e33 | |||
| be44ff2669 | |||
| d16f5ed9c8 | |||
| 6d890fc860 | |||
| aafe4f8886 | |||
| a2073d2514 | |||
| 3c82748e14 | |||
| 1e94c5a30d | |||
| 000e653038 | |||
| 945dc16060 | |||
| a30270d9ba | |||
| 1841fce4c4 | |||
| f2b8d4b9cb | |||
| 8ed5c90dcd | |||
| 789a2327ba | |||
| e3e0c6215d | |||
| b91e1c6f2a | |||
| 969a6a42d3 | |||
| 1b5477f3df | |||
| a07d655dc6 | |||
| d43adc0f2b | |||
| 3df51549bb | |||
| 324bb308dd | |||
| 625c11e7e1 | |||
| 4059c25e22 | |||
| 011b0df5f5 | |||
| 9faeb7a114 | |||
| 856f8f4c55 | |||
| 0cd388c666 | |||
| 1748f62c7e | |||
| 60c7179ddb | |||
| 7007c164a5 | |||
| 51b7de6a0d | |||
| 4733e1423c | |||
| 4189869e43 | |||
| 0f9199e0c3 | |||
| bb60871f57 | |||
| 2c820895d7 | |||
| 2080b3d031 | |||
| 9954392605 | |||
| 3dd53eb81f | |||
| 86fb2b833f | |||
| fbcb511978 | |||
| 87c1c6b3dc | |||
| 1e0e5ac53c | |||
| 7e9454456e | |||
| 4ed86dfe62 | |||
| 4e55fc7db4 | |||
| 2616d43fbc | |||
| 2a4db6a4f9 | |||
| e90cb2067d | |||
| 00a375c4e2 | |||
| ab4c97e607 | |||
| 721dbfd1b7 | |||
| 2926474688 | |||
| a7370aa9ac | |||
| a592d69f45 | |||
| bfc7906064 | |||
| eeff112491 | |||
| a70156e45b | |||
| 99245e3ef8 | |||
| 7a3ca0f821 | |||
| d94b3b3c92 | |||
| 86c5a92eef | |||
| cdc3b8cb10 | |||
| 0d3ba5d4cb | |||
| 71a1631f13 | |||
| 2d57b975d1 | |||
| 14042b5aab | |||
| 38a5ab549c | |||
| fa565ec9fc | |||
| 3196480da9 | |||
| cb85461c42 | |||
| ba6e8c8d69 | |||
| a54d0c4354 | |||
| 00d4a206f7 | |||
| f480086b6b | |||
| 968c30238c | |||
| fec1e0804a | |||
| bcb5fc9566 | |||
| 394d9db212 | |||
| 8c784f6133 | |||
| 258b167b1c | |||
| 69d5e92791 | |||
| efd5698597 | |||
| 2fde3cf535 | |||
| 2a4d51406b | |||
| 0596685ae7 | |||
| 358ab38fb0 | |||
| 254b19ac07 | |||
| eb4f5e5b31 | |||
| 8d06424b12 | |||
| fced863036 | |||
| 5af0865aab | |||
| 83821f0f1f | |||
| bf053386a4 | |||
| e7963381ba | |||
| 37c0edc7ca | |||
| a2fe70424e | |||
| 027d43b4eb | |||
| f99ed829b5 | |||
| 956a1a8e33 | |||
| 82f0a9828e | |||
| 584b5a3a8f | |||
| 39f49d548f | |||
| a168576adb | |||
| e85c4c8dfb | |||
| 0e3c43778e | |||
| 9101fe78db | |||
| 82948d6306 | |||
| dfb4b2a1d2 | |||
| 143f6970d8 | |||
| ea5dac5a99 | |||
| 3c26a8b0b5 | |||
| 2fabd436c0 | |||
| 07c68d6054 | |||
| 15090dc4fd | |||
| e791640dac | |||
| 9b4ae82a29 | |||
| 4620be394b | |||
| 5e7d603083 | |||
| fb7c406ddc | |||
| 416826f61f | |||
| a2e2f3ee82 | |||
| 70c7ec4a9f | |||
| 2b722b7e92 | |||
| 4cbb074ccd | |||
| 40220bd836 | |||
| df0dfa0a34 | |||
| 80684fed59 | |||
| 055030ea53 | |||
| aea3882c95 | |||
| 0c61128d23 | |||
| 680eac66e4 | |||
| 9f3037e941 | |||
| 5f00f6762f | |||
| af2b1794d3 | |||
| d371f77dcc | |||
| 022722a7a7 | |||
| 03027727d1 | |||
| 4fcdf37ce6 | |||
| 9ebe81c1b3 | |||
| e5f5c9e7f6 | |||
| 7b4cacc392 | |||
| a0dd7dae9b | |||
| 709d50a916 | |||
| dff81f293b | |||
| f653a6ea03 | |||
| 47d239f84f | |||
| e069fe9d92 | |||
| d848fb52b5 | |||
| 3ce8319251 | |||
| 9df0f4abaf | |||
| 4d0693cb8c | |||
| bf59e3f222 | |||
| 41aa846097 | |||
| 4093fa0d78 | |||
| 13bf126ebd | |||
| 1cf529ce8a | |||
| 9b76a21675 | |||
| 89637554d6 | |||
| 2dd1e00da4 | |||
| 5d955b5b74 | |||
| f7808a172c | |||
| acc9b514f5 | |||
| 31ba01c5d5 | |||
| 48ec10312f | |||
| 114dc10529 | |||
| 461e9a7210 | |||
| fcbb93e895 | |||
| 62cf066ff5 | |||
| 1ee64a8d89 | |||
| 01aaef2b50 | |||
| bcc62afe3d | |||
| 5c57658163 | |||
| f175e98c34 | |||
| 12f631b502 | |||
| ccb676e60c | |||
| 51d41345f7 | |||
| e59a3d938e | |||
| b32f2d9dd0 | |||
| d63c92e393 | |||
| 68127c5a63 | |||
| c4b5e1cb45 | |||
| 9a303f5096 | |||
| 87d8b5967f | |||
| f760046c44 | |||
| 93e7ef4bcc | |||
| a4229b366f | |||
| 420340bc14 | |||
| e7e87402ed | |||
| a164e1dda1 | |||
| 2f61325d4a | |||
| a6145e1d2e | |||
| 5232546690 | |||
| 0c46cb641b | |||
| 1233254309 | |||
| 8a2fe9b7a0 | |||
| f26ee6bf0a | |||
| d69c4ccfe4 | |||
| 086e619505 | |||
| 48e1846525 | |||
| bcc8871b1d | |||
| 25cb3511c9 | |||
| cfefc049c1 | |||
| af56d17630 | |||
| d1105bbb3d | |||
| 66fe61ff65 | |||
| a1141a43f2 | |||
| 702f113f51 | |||
| 63aace4099 | |||
| 9d7094b770 | |||
| 450dcb8484 | |||
| 4074d00b96 | |||
| 75ba0ce402 | |||
| da142327f2 | |||
| a3df317ae0 | |||
| 2c50d11f72 | |||
| 5ac33e1583 | |||
| d57c4e8407 | |||
| 374de0fa32 | |||
| 80e23dc9f2 | |||
| 2cc0e3c6ed | |||
| f9da3b919d | |||
| 1309853f57 | |||
| 39b8e6c30a | |||
| 9a6e1477f4 | |||
| 49375ed2d3 | |||
| f817b5261c | |||
| 885bd2c20a | |||
| 89b75e7913 | |||
| 079ebb1926 | |||
| 04c29aac11 | |||
| 95e7a190b3 | |||
| 8e2392e48f | |||
| 3abe0d95a5 | |||
| 9c6c9805de | |||
| 5cb900bfad | |||
| 0af5d22286 | |||
| f1687f402b | |||
| ce606df2c4 | |||
| fd44dc4e9b | |||
| 479745e764 | |||
| cc1c54475f | |||
| e7654d470c | |||
| 4b25f7e819 | |||
| 711b092f86 | |||
| 2bc103b80c | |||
| ad873c7357 | |||
| 0af0907eff | |||
| e27ba2bc14 | |||
| 236e163c0b | |||
| 7220210e8d | |||
| 34cbca97ea | |||
| 109d824e5f | |||
| 87746913fc | |||
| 7935c84f9a | |||
| c185635b43 | |||
| c201148738 | |||
| 53adb21c52 | |||
| 58b0ee36ca | |||
| 8a1f594add | |||
| 5b9959617e | |||
| da19d91b64 | |||
| 40bee4b7ba | |||
| 6a9b5d6763 | |||
| 0720ed477e | |||
| 46e82eb54d | |||
| f8ca4a8656 | |||
| e7d0e62f16 | |||
| aee290fc34 | |||
| 62cce2b152 | |||
| 65d4806a7b | |||
| 765757473a | |||
| 425bd7fff1 | |||
| e12e739a53 | |||
| 8b736dee34 | |||
| 32e75c27bf | |||
| 885ca604a1 | |||
| d10382d0dc | |||
| 0db4bf3b00 | |||
| 87c621965d | |||
| b2dca43fda | |||
| 33a91d972f | |||
| 2cbd48a3c3 | |||
| b5409ca112 | |||
| 35bd75948a | |||
| 0c8d74205b | |||
| fcc4d77895 | |||
| c52acc1a5d | |||
| 5703426b3c | |||
| f0bd164b92 | |||
| 9936a69d89 | |||
| ce2326c946 | |||
| 4c164907f5 | |||
| 9b11dc9469 | |||
| 1eccc6908b | |||
| 202371afbd | |||
| 41ef4411d9 | |||
| 035aa6c201 | |||
| edf36bcbfa | |||
| 3598c4305d | |||
| 94fbb6df6d | |||
| f4c1286c9d | |||
| 6e112d1856 | |||
| e62dc6e023 | |||
| 84e1580a47 | |||
| ebac0ca73b | |||
| 3f9c79e919 | |||
| ec2979ca48 | |||
| 21646eebd2 | |||
| 5711996231 | |||
| 0079723d35 | |||
| bc3b35f975 | |||
| 00d864ed0e | |||
| 8f2a30d9af | |||
| d9766d7378 | |||
| 5594d284fc | |||
| 632eb4729a | |||
| 3b5de82f5a | |||
| 76195f1c68 | |||
| 6586fbfd0d | |||
| 53b09f4e25 | |||
| 59812446ad | |||
| 1c2f59604d | |||
| b3a6d5dbf8 | |||
| 11de51ec26 | |||
| 10f3a8590b | |||
| 55c22abec1 | |||
| 32a1b81e4c | |||
| 62d8c649fe | |||
| 312397739e | |||
| 7ade0c045b | |||
| f9f9fa6c90 | |||
| 0a9c777795 | |||
| d2d47a41fb | |||
| dee2c3a02b | |||
| c4140609b7 | |||
| 75a6734e49 | |||
| f8b598f1d3 | |||
| fe9dd90b22 | |||
| 04b097fd9f | |||
| ab3ae99ca3 | |||
| 5db7be4ee4 | |||
| ff44527a3c | |||
| c9ad746732 | |||
| f3438b2602 | |||
| 938603458d | |||
| d5e763b072 | |||
| 74c97d005d | |||
| 1be684ba29 | |||
| 1367364c40 | |||
| 5f44fe369c | |||
| c6cc136833 | |||
| 97e31af331 | |||
| 0507c3d56a | |||
| 87b9895bc0 | |||
| ca1f11b708 | |||
| 514a041f4a | |||
| f8106cd693 | |||
| b1ba685ac5 | |||
| bc7658f035 | |||
| f377b5c95b | |||
| 45381222b7 | |||
| 4d6e1ea8ea | |||
| 20aaefa89c | |||
| 82509ddc0a | |||
| f2ba439022 | |||
| 2ba1f38503 | |||
| 8d58343263 | |||
| 2b2e0d2a5d | |||
| e407dc58eb | |||
| 32b2aa49f1 | |||
| 8448512a1f | |||
| 150e7b5f9e | |||
| 21870fa2e7 | |||
| d4c8da162b | |||
| fc7bf7b295 | |||
| d28885d0e5 | |||
| c4fd97d0bf | |||
| b73581b05d | |||
| 474cd5801e | |||
| ee9252a608 | |||
| fb406fdc94 | |||
| dd3bdb8086 | |||
| 7c103389f5 | |||
| 3861f33cba | |||
| 07d92db523 | |||
| 5732930102 | |||
| 2aea7fe9c4 | |||
| ffb70eb373 | |||
| e73436c99d | |||
| 0b0ae2423a | |||
| 7e079ff05c | |||
| 229fe2b3c3 | |||
| 38ed9d1e1f | |||
| 47cd7807d3 | |||
| 04d3d316d2 | |||
| 4bb8c813ac | |||
| b6ef7e430c | |||
| 91d9091758 | |||
| 0eebc928f7 | |||
| 724f8703d8 | |||
| e4e3007da6 | |||
| 13e6458d90 | |||
| 590aeaa7d1 | |||
| 5d152416de | |||
| edc8be6ed1 | |||
| 49c8aeac40 | |||
| e9c5340be9 | |||
| 2e2377d1c6 | |||
| 58735ee382 | |||
| a96116b128 | |||
| c4b9d38d8a | |||
| 8d76a255c4 | |||
| c5bc57b7a3 | |||
| 3814818537 | |||
| 7a49a7a3da | |||
| ff3395d3c6 | |||
| ce4525632f | |||
| 1c9a2431fe | |||
| f2520f3346 | |||
| 05df6e70d7 | |||
| c0e7159771 | |||
| 56c67dacf1 | |||
| eee05a17c3 | |||
| 45eb2e423d | |||
| 6b0998157c | |||
| 7fea6d5a5f | |||
| 68eeb002f6 | |||
| c565987b9c | |||
| f1ba683582 | |||
| 43f4bb7ed4 | |||
| 7a060cab1e | |||
| d014a12cff | |||
| 100e2e57a7 | |||
| 6e20e74774 |
@@ -1,54 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: community, triage
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
## NOTE
|
||||
|
||||
Silo issues are handled by community maintainers on a best-effort basis. There
|
||||
is no SLA, SLO, or emergency production-support channel. Follow the local
|
||||
[Code of Conduct](../code_of_conduct.md) when participating. Report suspected
|
||||
vulnerabilities through the private process in [SECURITY.md](../SECURITY.md),
|
||||
not in a public issue.
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
<!--- If you're describing a bug, tell us what should happen -->
|
||||
<!--- If you're suggesting a change/improvement, tell us how it should work -->
|
||||
|
||||
## Current Behavior
|
||||
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
|
||||
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
|
||||
|
||||
## Possible Solution
|
||||
<!--- Not obligatory, but suggest a fix/reason for the bug, -->
|
||||
<!--- or ideas how to implement the addition or change -->
|
||||
|
||||
## Steps to Reproduce (for bugs)
|
||||
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
|
||||
<!--- reproduce this bug. Include code to reproduce, if relevant -->
|
||||
<!--- and include relevant Silo logs with secrets and credentials removed -->
|
||||
|
||||
1.
|
||||
2.
|
||||
3.
|
||||
4.
|
||||
|
||||
## Context
|
||||
<!--- How has this issue affected you? What are you trying to accomplish? -->
|
||||
<!--- Providing context helps us come up with a solution that is most useful in the real world -->
|
||||
|
||||
## Regression
|
||||
<!-- Is this issue a regression? (Yes / No) -->
|
||||
<!-- If Yes, optionally include the Silo version, commit id, or PR that caused this regression. -->
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used (`silo --version`):
|
||||
* Server setup and configuration:
|
||||
* Operating System and version (`uname -a`):
|
||||
@@ -7,6 +7,14 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Report bugs in the PGSTY SILO server (`pgsty/silo`) here. Community maintainers
|
||||
handle reports on a best-effort basis. There is no SLA, SLO, or emergency
|
||||
production-support channel. Follow the
|
||||
[Code of Conduct](https://github.com/pgsty/silo/blob/main/code_of_conduct.md).
|
||||
Report suspected vulnerabilities privately through
|
||||
[SECURITY.md](https://github.com/pgsty/silo/blob/main/SECURITY.md).
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
<!--- Provide a general summary of the issue in the Title above -->
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
@@ -7,6 +7,9 @@ assignees: ''
|
||||
|
||||
---
|
||||
|
||||
Suggest improvements to the PGSTY SILO server (`pgsty/silo`) here.
|
||||
For patches, see the [contribution guide](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
## Contribution Licensing (no CLA, inbound=outbound, DCO required)
|
||||
|
||||
This project does not use a CLA; contributions are accepted inbound=outbound.
|
||||
This pull request contributes to PGSTY SILO (`pgsty/silo`). Code contributions
|
||||
are accepted under AGPL-3.0-or-later, the same license as the server.
|
||||
This project does not use a CLA or require a separate Apache-2.0 license grant.
|
||||
By submitting this pull request I represent that I have the right to contribute
|
||||
the changes, which are licensed under this repository's
|
||||
the code changes under this repository's
|
||||
[GNU Affero General Public License v3.0 or later](https://www.gnu.org/licenses/agpl-3.0.html)
|
||||
and remain my copyright. Every commit must carry a DCO `Signed-off-by` trailer
|
||||
and retain copyright in my original work. Existing copyright and license
|
||||
notices remain intact; separately licensed material keeps its applicable terms.
|
||||
Every commit must carry a DCO `Signed-off-by` trailer
|
||||
(`git commit -s`) certifying the
|
||||
[Developer Certificate of Origin](https://developercertificate.org/) — see
|
||||
[CONTRIBUTING.md](https://github.com/pgsty/silo/blob/main/CONTRIBUTING.md).
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
# Dockerfile.goreleaser and Dockerfile.distroless pin their bases by digest
|
||||
# so a release tag rebuilds from the same bytes. Without this the pins would
|
||||
# silently rot and stop receiving base-image CVE fixes.
|
||||
- package-ecosystem: docker
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
commit-message:
|
||||
prefix: build
|
||||
@@ -27,6 +27,10 @@ archives:
|
||||
ids:
|
||||
- silo
|
||||
name_template: "silo_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}"
|
||||
formats: ["tar.gz"]
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
formats: ["zip"]
|
||||
# Explicit so the license materials cannot silently drop out of the
|
||||
# binary archives: GoReleaser's default file globs would miss NOTICE.
|
||||
# CREDITS stays out deliberately -- at 1.8MB it would dominate the
|
||||
@@ -77,8 +81,10 @@ release:
|
||||
name: silo
|
||||
draft: true
|
||||
prerelease: false
|
||||
mode: append
|
||||
replace_existing_draft: true
|
||||
replace_existing_artifacts: false
|
||||
mode: replace
|
||||
# Draft replacement matches the release name; keep it identical to the tag.
|
||||
name_template: "{{ .Tag }}"
|
||||
|
||||
changelog:
|
||||
|
||||
@@ -7,6 +7,11 @@ on:
|
||||
description: "Published RELEASE.* tag to package as pgsty/silo"
|
||||
required: true
|
||||
type: string
|
||||
recovery:
|
||||
description: "Run the current main workflow against an already-published tag"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -75,12 +80,18 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Verify workflow identity matches release source
|
||||
env:
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
RECOVERY: ${{ inputs.recovery }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
CHECKED_OUT_REVISION="$(git rev-parse HEAD)"
|
||||
if [ "${CHECKED_OUT_REVISION}" != "${GITHUB_SHA}" ]; then
|
||||
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch this workflow from ${RELEASE_TAG}." >&2
|
||||
exit 1
|
||||
if [ "${RECOVERY}" != "true" ] || [ "${GITHUB_REF}" != "refs/heads/${DEFAULT_BRANCH}" ]; then
|
||||
echo "Checked out ${CHECKED_OUT_REVISION}, but workflow identity is ${GITHUB_SHA}. Dispatch from ${RELEASE_TAG}, or use recovery from ${DEFAULT_BRANCH}." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Recovery workflow ${GITHUB_SHA} is packaging published source ${CHECKED_OUT_REVISION}."
|
||||
fi
|
||||
|
||||
- name: Prepare verified Docker contexts
|
||||
@@ -129,10 +140,50 @@ jobs:
|
||||
mkdir -p "${context}/dockerscripts"
|
||||
tar -xzf "${archive}" -C "${context}" silo
|
||||
cp Dockerfile.goreleaser Dockerfile.distroless LICENSE NOTICE CREDITS "${context}/"
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||
"${context}/dockerscripts/"
|
||||
done
|
||||
|
||||
# The classic image bundles mcli. Resolve its two archive digests
|
||||
# from the immutable published release instead of trusting defaults
|
||||
# copied into an older Server tag. This also gives a recovery run a
|
||||
# narrow override when a tag selected the right mcli release but
|
||||
# accidentally retained stale archive pins.
|
||||
MC_REPO="$(awk -F= '/^ARG MC_REPO=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||
MC_VERSION="$(awk -F= '/^ARG MC_VERSION=/{print $2; exit}' Dockerfile.goreleaser)"
|
||||
test -n "${MC_REPO}"
|
||||
test -n "${MC_VERSION}"
|
||||
MC_VERSION_HYPHEN="${MC_VERSION#RELEASE.}"
|
||||
MC_PKG_VERSION="$(echo "${MC_VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
if [ "${MC_PKG_VERSION}" = "${MC_VERSION_HYPHEN}" ]; then
|
||||
echo "Invalid bundled mcli tag: ${MC_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isDraft --jq .isDraft)" != false ] || \
|
||||
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isPrerelease --jq .isPrerelease)" != false ] || \
|
||||
[ "$(gh release view "${MC_VERSION}" --repo "${MC_REPO}" --json isImmutable --jq .isImmutable)" != true ]; then
|
||||
echo "Bundled mcli ${MC_REPO}@${MC_VERSION} must be a published immutable release" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mc_checksums="mcli_${MC_PKG_VERSION}_checksums.txt"
|
||||
gh release download "${MC_VERSION}" --repo "${MC_REPO}" \
|
||||
--dir "${assets_dir}" --pattern "${mc_checksums}"
|
||||
gh attestation verify "${assets_dir}/${mc_checksums}" \
|
||||
--repo "${MC_REPO}" \
|
||||
--signer-workflow "${MC_REPO}/.github/workflows/release.yml" \
|
||||
--source-ref "refs/tags/${MC_VERSION}" >/dev/null
|
||||
|
||||
MC_AMD64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_amd64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||
MC_ARM64_SHA256="$(awk -v name="mcli_${MC_PKG_VERSION}_linux_arm64.tar.gz" '$2 == name {print $1}' "${assets_dir}/${mc_checksums}")"
|
||||
[[ "${MC_AMD64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
[[ "${MC_ARM64_SHA256}" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
{
|
||||
echo "MC_AMD64_SHA256=${MC_AMD64_SHA256}"
|
||||
echo "MC_ARM64_SHA256=${MC_ARM64_SHA256}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
echo "RELEASE_REVISION=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set up QEMU
|
||||
@@ -162,6 +213,9 @@ jobs:
|
||||
file: docker-release/amd64/Dockerfile.goreleaser
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||
tags: |
|
||||
pgsty/silo:${{ env.RELEASE_TAG }}-amd64
|
||||
pgsty/silo:latest-amd64
|
||||
@@ -178,6 +232,9 @@ jobs:
|
||||
file: docker-release/arm64/Dockerfile.goreleaser
|
||||
platforms: linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
MC_AMD64_SHA256=${{ env.MC_AMD64_SHA256 }}
|
||||
MC_ARM64_SHA256=${{ env.MC_ARM64_SHA256 }}
|
||||
tags: |
|
||||
pgsty/silo:${{ env.RELEASE_TAG }}-arm64
|
||||
pgsty/silo:latest-arm64
|
||||
|
||||
@@ -23,7 +23,7 @@ permissions:
|
||||
artifact-metadata: write
|
||||
|
||||
concurrency:
|
||||
group: finalize-release
|
||||
group: release-${{ inputs.tag }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
@@ -148,7 +148,9 @@ jobs:
|
||||
for rpm_file in finalize/packages/*.rpm; do
|
||||
signature_output="$(sudo rpmkeys --checksig --verbose "${rpm_file}")"
|
||||
printf '%s\n' "${signature_output}"
|
||||
if ! printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]' | grep -q "key id ${key_id}: ok"; then
|
||||
normalized_signature="$(printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]')"
|
||||
full_fingerprint="$(printf '%s' "${expected_fingerprint}" | tr '[:upper:]' '[:lower:]')"
|
||||
if ! printf '%s\n' "${normalized_signature}" | grep -Eq "key id ${key_id}: ok|key fingerprint: ${full_fingerprint}: ok"; then
|
||||
echo "Signature verification failed for ${rpm_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -90,6 +90,56 @@ jobs:
|
||||
- name: Run S3 Select tests under race detector
|
||||
run: go test -race ./internal/s3select/... -count=1
|
||||
|
||||
- name: Run conditional PUT and multipart tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP|PoolsMultipartCondition)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run multi-pool delete consistency tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m
|
||||
|
||||
# Build tags: kqueue selects the kqueue backend of rjeczalik/notify on
|
||||
# BSD and macOS and changes nothing on Linux. dev came with upstream's
|
||||
# 2024-06 Makefile change for development builds; no file in this
|
||||
# repository or its dependencies carries a dev build constraint, so it
|
||||
# selects nothing and is kept only so that the documented command
|
||||
# lines (make test, make install-race) stay valid.
|
||||
- name: Run lifecycle expiry and native restore tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(PoolsLifecycle|PoolsMigrationLifecycle|ExpireRestored|NativeRestore)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run multipart listing, cancellation and cleanup tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race ./cmd -run '^Test(ObjectCompleteMultipartUpload|Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run listing metadata recovery, pagination and cancellation tests under race detector
|
||||
run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity|ListObjectVersionsCrossParityNull|MigrationCrossParity)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run lock cancellation and late-grant cleanup tests under race detector
|
||||
run: go test -race ./internal/dsync ./internal/lsync -run '^Test(DRWMutex|LRWMutex)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run CPU metrics tests under race detector
|
||||
run: go test -race ./cmd -run '^TestLoadCPUMetrics' -count=1 -timeout=5m
|
||||
|
||||
- name: Run tag replication tests under race detector
|
||||
run: go test -race ./cmd -run '^TestAPITagging' -count=1 -timeout=5m
|
||||
|
||||
- name: Run Object Lock authorization and overwrite tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run maintenance storage timeout and compatibility tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd ./internal/grid ./internal/ioutil -run '^Test(StorageTimeout|StorageRollback|StoragePrerelease|MaintenancePartialFailedOverwriteAvailability|SingleRequest|WithDeadline)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run logger target lifecycle tests under race detector
|
||||
run: go test -race ./cmd -run '^TestDynamicWebhookOutlivesConfigContext' -count=1 -timeout=5m
|
||||
|
||||
- name: Compile deferred consistency diagnostics
|
||||
run: go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
name: Release
|
||||
|
||||
# Retry contract: an absent or single unfinalized Draft may be rebuilt from
|
||||
# scratch; a published release or a Draft carrying finalize's GPG-derived
|
||||
# provenance marker is terminal for this lane. The per-tag lock serializes
|
||||
# workflows, but a maintainer must not publish the Draft while this job runs.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
@@ -16,6 +21,10 @@ permissions:
|
||||
attestations: write
|
||||
artifact-metadata: write
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -61,6 +70,12 @@ jobs:
|
||||
buildscripts/verify-rebrand.sh
|
||||
dockerscripts/docker-entrypoint_test.sh
|
||||
|
||||
# Keep this gate aligned with the Object Lock race regression in go.yml.
|
||||
- name: Gate release on Object Lock authorization and overwrite tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
- name: Compute release variables
|
||||
env:
|
||||
# Passed through the environment, never interpolated into the script
|
||||
@@ -78,6 +93,15 @@ jobs:
|
||||
echo "Invalid release tag format: ${TAG}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! TAG_COMMIT="$(git rev-parse "${TAG}^{commit}" 2>/dev/null)"; then
|
||||
echo "Release tag ${TAG} does not resolve to a commit" >&2
|
||||
exit 1
|
||||
fi
|
||||
HEAD_COMMIT="$(git rev-parse HEAD)"
|
||||
if [ "${TAG_COMMIT}" != "${HEAD_COMMIT}" ]; then
|
||||
echo "Release tag ${TAG} resolves to ${TAG_COMMIT}, checkout is ${HEAD_COMMIT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
VERSION_HYPHEN="${TAG#RELEASE.}"
|
||||
PKG_VERSION="$(echo "${VERSION_HYPHEN}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')"
|
||||
VERSION_COLON="$(echo "${VERSION_HYPHEN}" | sed -E 's/T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/T\1:\2:\3Z/')"
|
||||
@@ -93,6 +117,13 @@ jobs:
|
||||
echo "Package version: ${PKG_VERSION}"
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: Check existing release state
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
buildscripts/check-release-state.sh "${RELEASE_TAG}"
|
||||
|
||||
# Both installer actions are pinned to immutable commits. The explicit
|
||||
# tool versions keep the release format reproducible across workflow
|
||||
# reruns while the installers verify the downloaded executables.
|
||||
@@ -107,12 +138,13 @@ jobs:
|
||||
cosign-release: v3.1.2
|
||||
|
||||
- name: Build Draft release with GoReleaser
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
args: release --clean --skip=validate --config .github/goreleaser.yml
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GORELEASER_CURRENT_TAG: ${{ env.RELEASE_TAG }}
|
||||
LDFLAGS: ${{ env.LDFLAGS }}
|
||||
PKG_VERSION: ${{ env.PKG_VERSION }}
|
||||
|
||||
@@ -124,7 +156,7 @@ jobs:
|
||||
- name: Install nFPM
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Build nFPM packages
|
||||
@@ -189,6 +221,14 @@ jobs:
|
||||
test -s "${BUNDLE_PATH}"
|
||||
cp "${BUNDLE_PATH}" "dist/silo_${PKG_VERSION}_provenance.sigstore.json"
|
||||
|
||||
- name: Confirm unfinalized Draft release state
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REQUIRE_DRAFT: "true"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
buildscripts/check-release-state.sh "${RELEASE_TAG}"
|
||||
|
||||
- name: Upload nFPM packages to Draft release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
name: Repository Cards
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 00:00 UTC = 08:00 Asia/Shanghai. GitHub may queue scheduled runs.
|
||||
- cron: "0 0 * * *"
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- .github/workflows/repository-cards.yml
|
||||
- .github/silo.svg
|
||||
- buildscripts/repository-cards/**
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- .github/workflows/repository-cards.yml
|
||||
- .github/silo.svg
|
||||
- buildscripts/repository-cards/**
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: repository-cards-${{ github.event.pull_request.number || 'publish' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Validate repository cards
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Use the runner's Python in an isolated environment
|
||||
run: |
|
||||
python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
|
||||
"$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
|
||||
- run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v
|
||||
|
||||
publish:
|
||||
name: Update README images
|
||||
needs: check
|
||||
if: github.repository == 'pgsty/silo' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: write
|
||||
issues: read
|
||||
pull-requests: read
|
||||
env:
|
||||
OUTPUT_BRANCH: codex/repository-cards
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- name: Use the runner's Python in an isolated environment
|
||||
run: |
|
||||
python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
|
||||
"$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
|
||||
echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Load the generated-assets branch
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
artifacts="$RUNNER_TEMP/repository-cards"
|
||||
if git ls-remote --exit-code --heads origin "$OUTPUT_BRANCH"; then
|
||||
git fetch --depth=1 origin "$OUTPUT_BRANCH"
|
||||
git worktree add --detach "$artifacts" FETCH_HEAD
|
||||
else
|
||||
status=$?
|
||||
# Exit 2 means no matching ref; transport/auth failures must stop.
|
||||
if [ "$status" -ne 2 ]; then exit "$status"; fi
|
||||
git worktree add --detach "$artifacts" HEAD
|
||||
git -C "$artifacts" checkout --orphan "$OUTPUT_BRANCH"
|
||||
git -C "$artifacts" rm -rf .
|
||||
fi
|
||||
|
||||
- name: Refresh contributor and star cards
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: python buildscripts/repository-cards/update.py --output "$RUNNER_TEMP/repository-cards"
|
||||
|
||||
- name: Publish changed assets
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd "$RUNNER_TEMP/repository-cards"
|
||||
git add README.md history.json curated.json contributors.json \
|
||||
contributors-light.svg contributors-dark.svg \
|
||||
star-history-light.svg star-history-dark.svg
|
||||
if git diff --cached --quiet; then
|
||||
echo "Repository cards are already current."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name 'github-actions[bot]'
|
||||
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||
git commit -s -m "chore: update repository cards $(date -u +%F)"
|
||||
# A normal push preserves history and refuses concurrent overwrites.
|
||||
git push origin "HEAD:refs/heads/$OUTPUT_BRANCH"
|
||||
@@ -4,13 +4,15 @@ on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- "go.mod"
|
||||
- "go.sum"
|
||||
- ".github/goreleaser.yml"
|
||||
- ".github/nfpm.yml"
|
||||
- "Dockerfile.goreleaser"
|
||||
- "Dockerfile.distroless"
|
||||
- "cmd/healthcheck-main.go"
|
||||
- "cmd/main.go"
|
||||
- "dockerscripts/download-static-curl.sh"
|
||||
- "dockerscripts/build-static-curl.sh"
|
||||
- "dockerscripts/docker-entrypoint.sh"
|
||||
- "dockerscripts/docker-entrypoint_test.sh"
|
||||
- "silo.service"
|
||||
@@ -23,6 +25,8 @@ on:
|
||||
- "buildscripts/minio-upgrade.sh"
|
||||
- "buildscripts/sign-release-rpms.sh"
|
||||
- "buildscripts/verify-build-provenance.sh"
|
||||
- "buildscripts/check-release-state.sh"
|
||||
- "buildscripts/check-release-state_test.sh"
|
||||
- "buildscripts/verify-rebrand.sh"
|
||||
- "buildscripts/verify-helm-migration.sh"
|
||||
- "buildscripts/helm-migration-guard/**"
|
||||
@@ -39,6 +43,28 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
curl:
|
||||
name: Static curl (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.runner }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
runner: ubuntu-latest
|
||||
- arch: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
- name: Build and exercise curl in an empty runtime
|
||||
run: |
|
||||
docker build --build-arg TARGETARCH=${{ matrix.arch }} \
|
||||
--target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
|
||||
docker run --rm silo-curl-test --version | tee curl-version.txt
|
||||
grep -F 'curl 8.23.0 ' curl-version.txt
|
||||
grep -F 'HTTP2' curl-version.txt
|
||||
docker run --rm silo-curl-test --fail --silent --show-error \
|
||||
--connect-timeout 15 --max-time 60 https://curl.se/robots.txt
|
||||
|
||||
validate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -69,9 +95,9 @@ jobs:
|
||||
echo "LDFLAGS: ${LDFLAGS}"
|
||||
|
||||
- name: GoReleaser config check
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
args: check --config .github/goreleaser.yml
|
||||
|
||||
- name: Validate Helm chart and legacy upgrade identity
|
||||
@@ -83,9 +109,9 @@ jobs:
|
||||
syft-version: v1.50.0
|
||||
|
||||
- name: Build snapshot artifacts
|
||||
uses: goreleaser/goreleaser-action@v7
|
||||
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
|
||||
with:
|
||||
version: "~> v2"
|
||||
version: v2.18.3
|
||||
# A pull-request snapshot has no trusted release identity. Exercise
|
||||
# the SBOM/checksum pipeline here, and reserve keyless signing for
|
||||
# the tag-triggered release workflow with GitHub OIDC.
|
||||
@@ -94,13 +120,34 @@ jobs:
|
||||
LDFLAGS: ${{ env.LDFLAGS }}
|
||||
PKG_VERSION: ${{ env.PKG_VERSION }}
|
||||
|
||||
- name: Verify archive SBOM and checksum coverage
|
||||
- name: Verify archive formats, documentation, SBOMs, and checksums
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mapfile -t archives < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz' | sort)
|
||||
mapfile -t sboms < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz.sbom.json' | sort)
|
||||
mapfile -t archives < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz' -o -name 'silo_*.zip' \) | sort)
|
||||
mapfile -t sboms < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz.sbom.json' -o -name 'silo_*.zip.sbom.json' \) | sort)
|
||||
test "${#archives[@]}" -eq 6
|
||||
test "${#sboms[@]}" -eq 6
|
||||
for os in linux darwin windows; do
|
||||
format=tar.gz
|
||||
binary=silo
|
||||
if [ "${os}" = windows ]; then
|
||||
format=zip
|
||||
binary=silo.exe
|
||||
fi
|
||||
for arch in amd64 arm64; do
|
||||
archive="dist/silo_${PKG_VERSION}_${os}_${arch}.${format}"
|
||||
test -s "${archive}"
|
||||
jq -e '.spdxVersion | startswith("SPDX-")' "${archive}.sbom.json" >/dev/null
|
||||
if [ "${format}" = zip ]; then
|
||||
listing="$(unzip -Z1 "${archive}")"
|
||||
else
|
||||
listing="$(tar -tzf "${archive}")"
|
||||
fi
|
||||
for member in "${binary}" LICENSE NOTICE README.md; do
|
||||
grep -Fx "${member}" <<< "${listing}"
|
||||
done
|
||||
done
|
||||
done
|
||||
manifest="dist/silo_${PKG_VERSION}_checksums.txt"
|
||||
test -s "${manifest}"
|
||||
(
|
||||
@@ -117,7 +164,7 @@ jobs:
|
||||
- name: Install package validation tools
|
||||
run: |
|
||||
set -euo pipefail
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0
|
||||
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
sudo apt-get update
|
||||
sudo apt-get install --yes rpm binutils
|
||||
@@ -435,6 +482,17 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
# Both variants must trust the same roots: the UBI bundle with the
|
||||
# RHSB-2026-012 blocklist applied, not Debian's bundle.
|
||||
tar -xOf "${ctx}/rootfs.tar" etc/ssl/certs/ca-certificates.crt > "${ctx}/ca.pem"
|
||||
openssl crl2pkcs7 -nocrl -certfile "${ctx}/ca.pem" \
|
||||
| openssl pkcs7 -print_certs -noout > "${ctx}/ca-subjects.txt"
|
||||
test "$(grep -c '^subject=' "${ctx}/ca-subjects.txt")" -gt 100
|
||||
if grep -E '^subject=.*(AAA Certificate Services|emSign ECC Root CA - G3)' "${ctx}/ca-subjects.txt"; then
|
||||
echo "distroless CA bundle still trusts a blocklisted root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The baked-in healthcheck must drive Docker's health state on its
|
||||
# own, the probe binary must be directly exec-able without any
|
||||
# shell, and SIGTERM must still reach PID 1 (the server binary is
|
||||
@@ -479,6 +537,8 @@ jobs:
|
||||
bash -n buildscripts/minio-upgrade.sh
|
||||
bash -n buildscripts/sign-release-rpms.sh
|
||||
bash -n buildscripts/verify-build-provenance.sh
|
||||
bash -n buildscripts/check-release-state.sh
|
||||
bash -n buildscripts/check-release-state_test.sh
|
||||
bash -n buildscripts/verify-rebrand.sh
|
||||
bash -n buildscripts/verify-helm-migration.sh
|
||||
sh -n buildscripts/package/postinstall.sh
|
||||
@@ -486,16 +546,19 @@ jobs:
|
||||
bash -n buildscripts/package/lifecycle_test.sh
|
||||
buildscripts/package/lifecycle_test.sh
|
||||
bash -n dockerscripts/docker-entrypoint_test.sh
|
||||
bash -n dockerscripts/download-static-curl.sh
|
||||
bash -n dockerscripts/build-static-curl.sh
|
||||
dockerscripts/docker-entrypoint_test.sh
|
||||
go run ./buildscripts/rebrand-guard
|
||||
buildscripts/verify-rebrand.sh
|
||||
test -x buildscripts/package-release.sh
|
||||
test -x buildscripts/sign-release-rpms.sh
|
||||
test -x buildscripts/verify-build-provenance.sh
|
||||
test -x buildscripts/check-release-state.sh
|
||||
test -x buildscripts/check-release-state_test.sh
|
||||
test -x buildscripts/verify-rebrand.sh
|
||||
test -x buildscripts/verify-helm-migration.sh
|
||||
test -x buildscripts/package/postinstall.sh
|
||||
test -x buildscripts/package/preremove.sh
|
||||
test -x buildscripts/package/lifecycle_test.sh
|
||||
test -x dockerscripts/docker-entrypoint_test.sh
|
||||
buildscripts/check-release-state_test.sh
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
|
||||
- name: Install govulncheck
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
|
||||
go install golang.org/x/vuln/cmd/govulncheck@v1.9.0
|
||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Run govulncheck
|
||||
|
||||
+2
-2
@@ -62,10 +62,10 @@ dist/
|
||||
|
||||
.claude/
|
||||
.codex/
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
_bmad/
|
||||
_bmad-output/
|
||||
# Investigation and editorial working files belong outside this repository.
|
||||
docs/investigations/
|
||||
docs/security/
|
||||
docs/rebranding.md
|
||||
.release-sign/
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ linters:
|
||||
- durationcheck
|
||||
- forcetypeassert
|
||||
- gocritic
|
||||
- gomodguard
|
||||
- gomodguard_v2
|
||||
- govet
|
||||
- ineffassign
|
||||
- misspell
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# Copyright 2026 PGSTY contributors.
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
FROM golang:1.27.3-bookworm
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
GOTOOLCHAIN=local \
|
||||
CGO_ENABLED=0 \
|
||||
MINIO_API_REQUESTS_MAX=10000
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl git jq openssl python3 procps \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Cache dependencies in the final image: subsequent analysis runs offline.
|
||||
# Honor go.mod's PGSTY replacements and leave the module files unchanged.
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download && go mod verify
|
||||
|
||||
COPY . /src
|
||||
|
||||
# Keep normal Go debug information; the release Makefile strips it.
|
||||
RUN go build -tags kqueue -o /usr/local/bin/silo . \
|
||||
&& silo --version
|
||||
|
||||
# Compile test packages while online, including the server and internal code.
|
||||
# Retain both the module and build caches so new reproducers work offline.
|
||||
RUN go test -tags kqueue,dev -run '^$' -timeout=10m ./cmd ./internal/...
|
||||
|
||||
ENV GOPROXY=off \
|
||||
GOSUMDB=off
|
||||
|
||||
CMD ["bash"]
|
||||
@@ -0,0 +1,43 @@
|
||||
# Override the release .dockerignore: keep source, CI files and all test fixtures.
|
||||
# The scanner adds Git history separately after building the image.
|
||||
.git
|
||||
.bin
|
||||
.claude
|
||||
.codex
|
||||
.idea
|
||||
.release-sign
|
||||
.vscode
|
||||
_bmad
|
||||
_bmad-output
|
||||
dist
|
||||
vendor
|
||||
**/node_modules
|
||||
**/*.test
|
||||
**/.DS_Store
|
||||
default.etcd
|
||||
docs/investigations
|
||||
docs/security
|
||||
docs/rebranding.md
|
||||
|
||||
# Local server and debugging binaries.
|
||||
minio
|
||||
silo
|
||||
hash-set
|
||||
healing-bin
|
||||
inspect
|
||||
pprofgoparser
|
||||
reorder-disks
|
||||
s3-check-md5
|
||||
s3-verify
|
||||
xattr
|
||||
xl-meta
|
||||
docs/debugging/s3-verify/s3-verify
|
||||
docs/debugging/xl-meta/xl-meta
|
||||
docs/debugging/s3-check-md5/s3-check-md5
|
||||
docs/debugging/hash-set/hash-set
|
||||
docs/debugging/healing-bin/healing-bin
|
||||
docs/debugging/inspect/inspect
|
||||
docs/debugging/pprofgoparser/pprofgoparser
|
||||
docs/debugging/reorder-disks/reorder-disks
|
||||
docs/debugging/populate-hard-links/populate-hardlinks
|
||||
docs/debugging/xattr/xattr
|
||||
@@ -0,0 +1,131 @@
|
||||
# PGSTY Silo security scope
|
||||
|
||||
Silo is PGSTY's independently maintained S3-compatible object-storage server,
|
||||
derived from MinIO. It serves untrusted network requests and stores user data,
|
||||
including PostgreSQL backups. Scan the current `pgsty/silo` `main` branch.
|
||||
Read `/src/SECURITY.md`, `/src/VULNERABILITY_REPORT.md`, and `/src/CHANGELOG.md`.
|
||||
|
||||
## Repository boundaries and lineage
|
||||
|
||||
The Go module intentionally remains `github.com/minio/minio`. Historical
|
||||
`MINIO_*` environment variables, S3 headers, `/minio/*` routes, and `.minio.sys`
|
||||
storage names are compatibility interfaces, not evidence that upstream MinIO
|
||||
is the target. Use this checkout's code and dependency versions for reproducers.
|
||||
|
||||
This enrollment covers the server code, including inherited code still present
|
||||
in the supported branch and its PGSTY changes. The maintained Console, client,
|
||||
and shared packages appear as dependencies in `go.mod`. Do not replace them
|
||||
with upstream releases. They are not additional independent scan targets under
|
||||
this enrollment. A server-side integration defect is in scope; clearly identify
|
||||
a dependency-only root cause rather than attributing it to this repository.
|
||||
|
||||
## Attackers and important surfaces
|
||||
|
||||
- Unauthenticated S3/HTTP clients, including hostile object names, headers,
|
||||
query parameters, XML/JSON bodies, signatures, and multipart data.
|
||||
- Authenticated users, service accounts, and temporary STS credentials whose
|
||||
authority must remain bounded by the effective IAM and bucket policies.
|
||||
- Bucket owners and delegated administrators with explicitly limited actions;
|
||||
distinguish them from the unrestricted deployment/root administrator.
|
||||
- Inputs from configured identity providers, replication/tiering targets, and
|
||||
webhooks where the configuration and permissions make an attack reachable.
|
||||
|
||||
Prioritize authentication/signature verification, IAM/STS and policy
|
||||
composition, administrative API authorization, object/version isolation,
|
||||
Object Lock retention and legal holds, multipart operations, replication,
|
||||
lifecycle/tiering, and multi-pool deletion/migration. Include archive, XML,
|
||||
JSON and S3 Select parsing when remotely reachable. Investigate credential
|
||||
exposure, path traversal, unauthorized object access, SSRF across a meaningful
|
||||
boundary, and attacks that corrupt or irreversibly delete other users' data.
|
||||
|
||||
The deployment administrator controls root credentials, storage directories,
|
||||
trusted peers, and identity/KMS/replication endpoint configuration. Host root
|
||||
access, stolen root credentials, or a malicious replacement of the server
|
||||
binary do not by themselves demonstrate a vulnerability. An unprivileged
|
||||
account gaining those capabilities does. An administrator's documented ability
|
||||
to configure an outbound endpoint is not sufficient evidence of SSRF; show
|
||||
an unauthorized actor, forbidden destination, or confused-deputy effect.
|
||||
|
||||
Object Lock has explicit retention semantics. Do not discard a retention bypass
|
||||
just because the caller has some administrative permission: describe the mode,
|
||||
version, legal hold, and exact IAM actions, and compare with the intended rules.
|
||||
Likewise, do not assume bucket access implies access to other buckets or users.
|
||||
|
||||
Known consistency limitations are described in `CONTRIBUTING.md` and the
|
||||
changelog. Tests tagged `consistency_strict` exercise stronger guarantees
|
||||
outside the current maintenance contract and can fail. A failing diagnostic
|
||||
alone is not a new vulnerability; demonstrate a reachable security impact.
|
||||
These limitations do not excuse an authorization or retention bypass.
|
||||
|
||||
## Offline environment
|
||||
|
||||
Source is in `/src`; the unstripped server is `/usr/local/bin/silo`. Go, the
|
||||
module cache, and the test build cache are retained. The scanner adds Git
|
||||
history to `/src/.git` after building this image; a plain Docker build omits it.
|
||||
The Dockerfile compiles the server and tests with the selected PGSTY dependency
|
||||
replacements. `GOTOOLCHAIN=local` and `GOPROXY=off` prevent surprise downloads.
|
||||
CGO is disabled as in the normal server/unit-test CI; enable it explicitly for
|
||||
race tests when needed (the Go base image includes a C compiler).
|
||||
|
||||
Useful offline checks:
|
||||
|
||||
```sh
|
||||
cd /src
|
||||
silo --version
|
||||
go test -tags kqueue,dev ./internal/auth ./internal/config/identity/... ./internal/s3select/... -count=1 -timeout=10m
|
||||
go test -tags kqueue,dev ./cmd -run '^(TestGetRequestAuthType|TestS3SupportedAuthType|TestIsReqAuthenticated|TestCheckAdminRequestAuthType|TestValidateAdminSignature|TestDoesPolicySignatureMatch|TestDoesPresignedSignatureMatch|TestDoesPresignedV2SignatureMatch|TestSTSLDAPLogin.*|TestGetSTSLDAPLoginSourceIP.*)$' -count=1 -timeout=10m
|
||||
```
|
||||
|
||||
For HTTP reproducers, start a disposable loopback server with synthetic
|
||||
credentials and a fresh data directory:
|
||||
|
||||
```sh
|
||||
MINIO_ROOT_USER=scanneradmin MINIO_ROOT_PASSWORD=scanner-local-test-only \
|
||||
silo server /tmp/silo-scanner-data \
|
||||
--address 127.0.0.1:9000 --console-address 127.0.0.1:9001
|
||||
```
|
||||
|
||||
The broader suites are `go test -tags kqueue,dev ./internal/... -timeout=15m`
|
||||
and `go test -tags kqueue,dev ./cmd -timeout=30m`. Some integration suites need
|
||||
LDAP, OIDC, etcd, KMS, multiple servers, or IPv6; those services are not started
|
||||
by this Dockerfile. Do not treat a missing external service as a security
|
||||
finding. The focused commands above do not require those providers.
|
||||
|
||||
## Reports and severity
|
||||
|
||||
Include the commit, exact attacker privileges/policy, relevant deployment
|
||||
settings, API requests or a standalone test, observed impact, and affected
|
||||
versions where established. For data-loss or cross-pool reports, include the
|
||||
object/version state and a deterministic sequence. Deduplicate by root cause
|
||||
and distinguish a new flaw from an already documented limitation or fixed bug.
|
||||
|
||||
Unauthenticated code execution or broad credential/authorization compromise
|
||||
can be critical. Cross-user object access, unauthorized destructive operations,
|
||||
and retention bypass can be high or critical according to their demonstrated
|
||||
reach. Rate availability issues by practical cost, scope and recovery; a
|
||||
single malformed request and an authorized full-disk upload are different
|
||||
threats. Do not claim exploitability solely from a static pattern.
|
||||
|
||||
Prefer minimal patches and regression tests that preserve S3/storage format
|
||||
compatibility. Use only synthetic data and local test instances; do not test
|
||||
against public Silo/MinIO deployments or real backup repositories.
|
||||
|
||||
## Maintaining the scanner image
|
||||
|
||||
Build from the repository root, then run the checks above without networking:
|
||||
|
||||
```sh
|
||||
docker build -f .oss-scanner/Dockerfile -t silo-scanner:local .
|
||||
docker run --rm -it --network=none silo-scanner:local bash
|
||||
```
|
||||
|
||||
Keep the Go image version aligned with `go.mod`. The adjacent
|
||||
`Dockerfile.dockerignore` preserves compressed test fixtures that the release
|
||||
image's root `.dockerignore` excludes. Local build outputs, editor state, and
|
||||
private working directories are excluded from the scanner context.
|
||||
|
||||
Enrollment lives in `anthropics/oss-scanner`, under `projects/silo/project.yaml`.
|
||||
Once these files are published on `main`, that configuration can use
|
||||
`dockerfile: .oss-scanner/Dockerfile` and
|
||||
`threat_model: .oss-scanner/threat_model.md`; remove its adjacent copies in the
|
||||
same update. Do not switch the enrollment paths before the files are public.
|
||||
@@ -0,0 +1,84 @@
|
||||
# SILO Repository Guide
|
||||
|
||||
## Project map
|
||||
|
||||
- Server: `pgsty/silo` (this checkout, default branch `main`)
|
||||
- Console: `pgsty/silo-console` (usual sibling checkout `../silo-console`)
|
||||
- Client: `pgsty/mc`, shipped as `mcli` (usual sibling checkout `../mc`)
|
||||
- Shared packages: `pgsty/silo-pkg` (usual sibling checkout `../silo-pkg`)
|
||||
- Documentation: `pgsty/silo.pgsty.com` (usual sibling checkout `../silo.pgsty.com`),
|
||||
published at <https://silo.pgsty.com/>
|
||||
|
||||
The server executable, package, systemd service, and container image use `silo`;
|
||||
the public Docker image is `docker.io/pgsty/silo`. The server module remains
|
||||
`github.com/minio/minio`. Consult the current `go.mod` and release configuration
|
||||
for selected component versions and compatibility replacements.
|
||||
|
||||
## Supported stack and compatibility policy
|
||||
|
||||
The maintained, release-gating product graph is the coordinated PGSTY stack:
|
||||
`silo` + `silo-console` + `mc` + `silo-pkg`.
|
||||
|
||||
Keep compatibility with upstream MinIO/MC on a best-effort basis. Preserve
|
||||
inexpensive wire, configuration, CLI, migration, and import compatibility when
|
||||
it helps users, and document known differences. Do not infer from source
|
||||
lineage, MinIO-compatible protocols, retained `MINIO_*`/`MC_*` names, or an
|
||||
inherited upstream test that unmodified upstream MinIO is a supported release
|
||||
target.
|
||||
|
||||
Upstream-only compatibility checks may remain as advisory evidence, but they
|
||||
must not force a downgrade of a maintained SILO component, a fork-only API
|
||||
workaround, or a release block. Make upstream compatibility a hard gate only
|
||||
when the user explicitly requests that scope.
|
||||
|
||||
## Dependency selection
|
||||
|
||||
- When PGSTY maintains a component, import and require it directly under its own
|
||||
module path. In particular, prefer `github.com/pgsty/silo-pkg/v3` over
|
||||
`github.com/minio/pkg/v3` in maintained SILO source.
|
||||
- Do not use an upstream package merely to make unmodified upstream MinIO or MC
|
||||
compile. Unavoidable transitive upstream modules should be documented and
|
||||
kept separate from the maintained product dependency.
|
||||
- `github.com/minio/minio-go/v7` is the explicit exception: use the verified
|
||||
upstream module/commit while it contains the required fixes; do not recreate
|
||||
a SILO fork without a concrete functional divergence.
|
||||
- Check the current `go.mod` before changing versions. Coordinate breaking
|
||||
import-path changes across package, client, Console, and server releases.
|
||||
|
||||
## Documentation and delivery
|
||||
|
||||
The companion site owns product, operations, migration, security-advisory,
|
||||
design, and release documentation. Maintain English and Chinese content in
|
||||
that repository and link to its canonical URLs from this one. The site's
|
||||
`content/docs/` is the documentation entry point; detailed pages live under
|
||||
`content/operations/`, `administration/`, `reference/`, `compatibility/`,
|
||||
`about/`, and `blog/`, following the existing structure.
|
||||
|
||||
Keep repository entry points and contributor instructions here. Retained
|
||||
upstream documents, examples, and test fixtures under `docs/` may be updated
|
||||
when a code or tooling change requires it; do not build a second documentation
|
||||
site in this tree.
|
||||
|
||||
Investigation plans, prompts, AI session transcripts, execution logs, temporary
|
||||
reports, and private security material belong outside the checkout, for example
|
||||
in a task-specific directory under `~/tmp/`. Do not recreate
|
||||
`docs/investigations/`, `docs/security/`, or `docs/rebranding.md`, or move their
|
||||
working material to another repository directory. Extract reusable, verified
|
||||
knowledge into the companion site; keep private evidence outside public Git.
|
||||
|
||||
Compatibility pages may continue to describe similarities with upstream, but
|
||||
must label that compatibility as best effort. The supported and tested server
|
||||
for Console and mcli administration is `pgsty/silo`.
|
||||
|
||||
Before removing or moving documentation, check repository links and scripts,
|
||||
the companion site's links and anchors, and references from this guide. Use
|
||||
fixed commit URLs for historical evidence that should survive deletion from
|
||||
the current tree. Run the site's `make check` for site changes and this
|
||||
repository's `make rebrand-guard` for documentation cleanup that changes the
|
||||
identifier inventory. When new public URLs are involved, publish the site
|
||||
before publishing source changes that depend on those URLs.
|
||||
|
||||
Keep changes in the repository that owns the affected surface. Treat every
|
||||
repository commit, tag, release, image, documentation update, and deployment as
|
||||
a separate deliverable. Follow `CONTRIBUTING.md`, including DCO sign-off
|
||||
(`git commit -s`). `CLAUDE.md` imports this guide so both agents use one policy.
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Embed Console 2.7.0 source `26ce922424f1`, including its ten-language UI,
|
||||
rebuilt frontend assets, Swagger array-validation fixes and Redoc routing.
|
||||
Select the published mcli 20261109 source `e1d4cc5abfe0` and silo-pkg
|
||||
v3.16.0 directly in the maintained dependency graph.
|
||||
- Adopt go-jose/v4 v4.1.9, cpuid/v2 v2.4.2, YAML v3.0.6 and gjson v1.20.1,
|
||||
plus Console's runtime/server-middleware v0.33.4. Preserve Go 1.27.3, the
|
||||
verified upstream minio-go commit, networking/crypto security floors and
|
||||
terminal-width/NetBSD compatibility selections.
|
||||
- Update NATS credentials to jwt/v2 v2.8.3 and nkeys v0.4.17, including
|
||||
rejection of identity-point and small-order public keys. Use nats-server
|
||||
v2.15.2 for notification integration tests.
|
||||
- Update Google Cloud auth to v0.24.3 and oauth2adapt to v0.3.1 for REST
|
||||
request validation and AWS FIPS endpoint signing fixes.
|
||||
- Bundle verified mcli 20261109 archives in source container builds and use
|
||||
that release by default in the integration-test client installer.
|
||||
- Prevent replication worker metrics from panicking when the replication
|
||||
pool is absent, while retaining historical averages and peak counts.
|
||||
|
||||
## RELEASE.2026-10-26T00-00-00Z
|
||||
|
||||
Package version: `20261026000000.0.0`; published from source `c3ad836aed0d`.
|
||||
Artifacts are listed in the [GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z).
|
||||
The coordinated [English release notes](https://silo.pgsty.com/blog/release/silo-20261026/)
|
||||
and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261026/)
|
||||
cover all components, upgrade guidance, and validation scope.
|
||||
|
||||
- Fix LDAP login for long normalized DNs without hiding unreadable IAM
|
||||
revocation records. Only length errors for valid keys with an ASCII path
|
||||
component over 255 bytes are treated as absent; long-DN writes still fail.
|
||||
Thanks to [@910patin](https://github.com/910patin) for the original fix and
|
||||
reproduction in [#250](https://github.com/pgsty/silo/pull/250).
|
||||
- Build the server and standalone debugging tools with Go 1.27.3 and refresh
|
||||
Go dependencies, including silo-pkg v3.15.2, mcli 20261026
|
||||
and upstream minio-go `320af050c445` for multipart, read/seek,
|
||||
batch-delete and bucket-replication removal fixes. Retain output-format and NetBSD compatibility selections.
|
||||
- Embed Console v2.5.1 source `fff1618d9abb`, including its GUI update,
|
||||
diagnostic WebSocket closing handshake, complete health-report decoding,
|
||||
group-membership preservation, OIDC group permissions, TLS-proxy session
|
||||
cookies, credential exports and accessibility fixes.
|
||||
- Bundle mcli 20261026 in container builds, the verified client installer and
|
||||
Helm client jobs. Include client identity isolation, resync status and rate
|
||||
fixes, and bounded batch-status fallback. Scripts must handle the additive
|
||||
batch `pending` state and nonzero exits for partial resync mutations.
|
||||
- Read current bucket inventory in account-info responses so Console refreshes
|
||||
immediately reflect successful creation and deletion. This management endpoint
|
||||
now performs a live list instead of returning a cached inventory.
|
||||
- Restore the native built-in policy immediately after resetting an override.
|
||||
Peer and etcd notifications reload committed state, so delayed delete events
|
||||
preserve subsequent edits. Keep deletion revisions, policy-in-use checks and
|
||||
custom-policy deletion behavior; storage failures retain the previous policy.
|
||||
- Reload policies without holding the authentication-cache lock, retain IAM
|
||||
write ordering and publish replacements only after a complete successful load.
|
||||
Propagate cancellation through IAM/KMS and SSE-S3 decryption without shortening
|
||||
the caller's policy-query deadline.
|
||||
- Preserve authoritative copies when a multi-pool delete or bulk cleanup fails,
|
||||
choose an available pool for a new delete marker, and keep tier data referenced
|
||||
by surviving copies. This does not introduce strict cross-pool read recovery.
|
||||
- Preserve remote tier data when lifecycle expiry only adds a delete marker,
|
||||
including non-null versions in suspended buckets. Forward `FULL_OBJECT`
|
||||
checksum type for multipart replicas. Thanks to
|
||||
[@botdeniska](https://github.com/botdeniska) for these lifecycle, replication
|
||||
and multi-pool deletion contributions.
|
||||
- Preserve Kafka's default broker protocol 2.1 and terminal-width behavior;
|
||||
fix legacy DARE v1 key unsealing and verify DARE v1/v2 and MinKMS TLS compatibility.
|
||||
- Forward storage RPC delete options, including the previous data directory
|
||||
needed by overwrite rollback. Preserve existing drive/RPC timeouts, request
|
||||
cancellation, object locking, read-quorum and healing behavior.
|
||||
- Update bundled curl to 8.23.0 and require patched static libraries. Apply
|
||||
[Red Hat's compromised-CA mitigation](https://access.redhat.com/security/vulnerabilities/RHSB-2026-012);
|
||||
the root blocklist also rejects legitimate chains under the affected roots.
|
||||
- Ship Windows platform archives as ZIP; Linux and macOS retain tar.gz.
|
||||
|
||||
Storage timeouts retain their existing meaning: a returned timeout does not
|
||||
prove that the underlying disk operation stopped or that a write did not
|
||||
commit. Late commits or delayed rollback can affect newer writes; stale reads
|
||||
remain possible after partial failures, lease loss or network partitions. This
|
||||
release does not provide distributed fencing or repair previously inconsistent
|
||||
objects.
|
||||
|
||||
These source changes are not included in the published 20261018 artifacts.
|
||||
|
||||
## RELEASE.2026-10-18T00-00-00Z
|
||||
|
||||
Package version: `20261018000000.0.0`; published on 2026-10-18 from source
|
||||
`0cd388c666f8211f552604ae73448a066e9c25ce`. This section covers the changes
|
||||
after the public 20260916 source, `2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
|
||||
|
||||
Full [English release notes](https://silo.pgsty.com/blog/release/silo-20261018/)
|
||||
and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261018/)
|
||||
cover the upgrade contract, evidence and remaining limits.
|
||||
|
||||
- **Object Lock (#234, #235):** require `s3:PutObjectRetention` with request-body
|
||||
conditions before object lookup; weakening active GOVERNANCE additionally
|
||||
requires bypass intent and permission. Protect caller-addressed write versions,
|
||||
directory null versions and persisted multipart targets under the write lock;
|
||||
fail closed on unreadable lock state. See SN-2026-015/016 in the
|
||||
[advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
- **IAM Admin authorization (#242, `995439260`):** enforce explicit self denies,
|
||||
session restrictions and plugin decisions; prevent restricted delegation and
|
||||
non-root mutation of root-derived credentials under built-in authorization.
|
||||
Preserve empty-policy inheritance and damaged-key cleanup. Normalize bulk
|
||||
self-list aliases and allow ordinary policyless self inspection. See
|
||||
[SN-2026-017](https://silo.pgsty.com/blog/security/20261018-release-hardening/#sn-2026-017).
|
||||
- **Scoped STS revocation (`0cd388c66`):** enumerate durable parent/type-selected
|
||||
tokens, propagate each committed tombstone to sibling nodes and sites, and
|
||||
recover missed notifications through existing refresh/healing. Tokens issued
|
||||
only at an isolated site are outside the requesting cluster's snapshot;
|
||||
propagation is eventual and newly issued keys remain valid.
|
||||
- **Pool migration (#248, `60c7179dd`, `1748f62c7`):** prevent stale moved
|
||||
payloads from overwriting newer copies, preserve null/UUID delete markers,
|
||||
recheck source versions and clean only confirmed moved versions under the
|
||||
object lock. Added per-pool reads and per-version cleanup have maintenance
|
||||
costs; earlier data loss is not reconstructed.
|
||||
|
||||
- **LIST recovery (#239, `3dd53eb81`, `2080b3d03`):** confirm eligible uncertain
|
||||
latest-object groups under the appropriate read locks, distinguish absence
|
||||
from failure, preserve directory/common-prefix entries and terminal pagination
|
||||
errors, and make lock backoff cancelable. Resolved entries keep their fast path;
|
||||
version history, quorum and disk formats retain their contracts. The strict
|
||||
sub-5% performance target remains unconfirmed in the recorded comparison.
|
||||
- **Multipart listing (`4733e1423`):** withdraw #243's default persistent scan.
|
||||
Public listing uses node-local cache or exact-key lookup; restart omissions,
|
||||
prefix and cache-pagination limits remain. The retired strict switch is ignored
|
||||
with a startup warning. **Former strict users lose durable listing and strict
|
||||
cancellation.** Keep identity metadata, path/wrong-target safety, default
|
||||
best-effort Abort and peer notification. Diagnostic `multipart-preflight`
|
||||
reports `mode=legacy`; its budgets do not gate public operations. #79 remains
|
||||
open, including delayed creation after cancellation.
|
||||
- **Multipart cleanup (#236, `0f9199e0c`):** parse native timestamps before the
|
||||
Base64 fallback and remove empty hash parents without a second expiry wait.
|
||||
Keep active/unreadable uploads; default expiry/check intervals stay 24h/6h.
|
||||
- **Commit errors (#232):** missing staged `xl.meta` at final commit becomes
|
||||
HTTP 503 `SlowDownWrite`; preserve this classification across storage RPC and
|
||||
healing. Real incomplete bodies retain their existing errors.
|
||||
- **Reused metadata (#233):** clear absent legacy `EcM`/`Signature` and locally
|
||||
reset mutually exclusive scanner response fields without changing disk/RPC formats.
|
||||
- **Checksum and Console errors (#225, #229, #238):** align Content-MD5 and
|
||||
algorithm-specific messages, retaining the generic unknown-algorithm message;
|
||||
embed Console's wrapped-error and `InvalidBucketState` handling.
|
||||
- **Lambda and erasure documentation (#220, `2fde3cf53`):** fix inverted
|
||||
`TargetIDSet.IsEmpty`; document parity/quorum per object and erasure set.
|
||||
- **Coordinated dependencies:** Console v2.4.2 (`ef2dd2c41d21`), mcli 20261018
|
||||
(`fa3b8bf72458`), silo-pkg v3.14.3, upstream minio-go v7.4.1 and Go 1.27.2.
|
||||
Standalone Console/mcli still declare pkg v3.14.2. The Server package update
|
||||
speeds admin-action classification without changing policy behavior. Refresh
|
||||
Go x/*, go-jose v4.1.6, LDAP v3.4.15, compress v1.20.3, msgp v1.6.5, etcd v3.7.2,
|
||||
fwd v1.2.1 and cpuid v2.4.1. OIDC RSA JWKs above 4096 bits are rejected.
|
||||
Keep the NetBSD go-systemd v22.6.0 replacement; parallel SDK GET/Compose remain opt-in.
|
||||
- **Build and delivery:** gate releases on Object Lock regressions; use
|
||||
Git for rebrand scanning without a ripgrep dependency (`7007c164a`). Refresh UBI 9 pins,
|
||||
GoReleaser v2.18.3, nFPM v2.48.0, golangci-lint v2.15.0 and govulncheck v1.9.0.
|
||||
Update bundled client hashes/helper and the Helm client image to mcli 20261018.
|
||||
The chart's Server default stays at the public 20260916 image until publication.
|
||||
|
||||
Upgrade every writer and replication site; follow the
|
||||
[IAM recovery guide](https://silo.pgsty.com/operations/replication/iam-upgrade/).
|
||||
Old strict multipart users must review the capability loss before upgrading.
|
||||
#217's multi-site delete-marker late creation and crash residue remain unresolved.
|
||||
Source tests, draft artifacts, public images and production rollout are separate
|
||||
acceptance boundaries; historical 20261015 test counts do not certify this source.
|
||||
|
||||
## RELEASE.2026-09-16T00-00-00Z
|
||||
|
||||
Package version: `20260916000000.0.0`; published source:
|
||||
`2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
|
||||
|
||||
This release includes signed-header and payload-policy hardening (SN-2026-011/012),
|
||||
durable IAM revocations (SN-2026-013), the bounded Console sharing proxy
|
||||
(SN-2026-014), password-permission separation, multi-pool conditions/deletion/tag
|
||||
repairs, encrypted/federated copy repairs, delete-marker retry fixes, opt-in strict
|
||||
multipart discovery, Go-default TLS key exchange, absolute HTTP/1 header deadlines
|
||||
and synchronized CPU metrics. It embeds Console v2.4.1, mcli 20260916 and pkg v3.14.1.
|
||||
|
||||
The complete release, contributor credits, historical artifact acceptance and
|
||||
upgrade requirements are preserved in the canonical
|
||||
[English](https://silo.pgsty.com/blog/release/silo-20260916/) and
|
||||
[Chinese](https://silo.pgsty.com/zh/blog/release/silo-20260916/) release records.
|
||||
|
||||
## RELEASE.2026-09-03T13-18-01Z
|
||||
|
||||
Published source: `9b11dc9469e650815b775cb47b039610644f5da4`.
|
||||
[Complete release notes](https://silo.pgsty.com/blog/release/silo-20260903/) ·
|
||||
[GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)
|
||||
|
||||
This release ships Go 1.27.1, silo-pkg v3.13.2, upstream minio-go `0e78d3f18efe`,
|
||||
mcli 20260903 and embedded Console source `464a59d73ada` (v2.3.0 version identity).
|
||||
Installing the newer standalone mcli or Console does not replace components
|
||||
inside this existing Server binary or image.
|
||||
|
||||
Earlier releases: [release archive](https://github.com/pgsty/silo/releases).
|
||||
+42
-8
@@ -44,11 +44,29 @@ Before opening a pull request:
|
||||
|
||||
- Add or update tests for changed behavior.
|
||||
- Run `make verifiers`.
|
||||
- If `make rebrand-guard` reports a changed compatibility set, review the
|
||||
listed identifiers; when the change is intended, refresh the baseline with
|
||||
`go run ./buildscripts/rebrand-guard --write` and commit
|
||||
`buildscripts/rebrand-guard/compat-baseline.json`.
|
||||
- Run the smallest relevant package tests, then `make test` when practical.
|
||||
- Run `make build` and confirm the generated executable is `silo`.
|
||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||
`.minio.sys`, ARN, module/import-path, or serialized compatibility name.
|
||||
|
||||
The maintenance branch retains existing storage timeouts, cancellation,
|
||||
object locking, read-quorum and healing behavior. The normal suite checks local
|
||||
and RPC timeout/cancellation, remote rollback options, and readability after a
|
||||
failed partial overwrite. A timeout does not prove that a disk worker stopped
|
||||
or that a write did not commit.
|
||||
|
||||
Stronger mutation lifetime, rollback ownership, generation ordering, strict-read
|
||||
and cross-pool recovery diagnostics are preserved behind `consistency_strict`.
|
||||
Compile them with `go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'`
|
||||
and run individual scenarios explicitly. They describe guarantees outside the
|
||||
maintenance contract and may fail; report their results separately from the
|
||||
normal suite. They must not be used to claim distributed fencing or repair of
|
||||
previously inconsistent objects.
|
||||
|
||||
### Commit changes
|
||||
|
||||
After verification, commit your changes with a concise message and a DCO
|
||||
@@ -73,11 +91,22 @@ test evidence, compatibility notes, and documentation impact. Public product
|
||||
documentation is owned by the separate
|
||||
[`pgsty/silo.pgsty.com`](https://github.com/pgsty/silo.pgsty.com) repository.
|
||||
|
||||
## Contributor recognition
|
||||
|
||||
Every human issue or pull-request author is recognized in [CONTRIBUTORS.md](CONTRIBUTORS.md),
|
||||
including open issues, draft PRs, and PRs closed without merging. Merged fixes,
|
||||
adopted proposals, and reports that lead to fixes receive greater prominence;
|
||||
first participation guides the remaining order. Work incorporated through a
|
||||
later PR retains credit without changing the original PR's recorded status.
|
||||
Security disclosures are credited with the reporter's agreement. DCO sign-off
|
||||
applies to code commits, not to opening an issue.
|
||||
|
||||
## Licensing of Contributions
|
||||
|
||||
Silo is licensed under the [GNU AGPL v3.0 or later](LICENSE). Its core is
|
||||
Copyright (c) MinIO, Inc.; the combined work can never be relicensed, and this
|
||||
fork does not try to.
|
||||
Code contributions to PGSTY SILO (`pgsty/silo`) are accepted under the
|
||||
[GNU AGPL v3.0 or later](LICENSE), the same license as the server. Submit issues
|
||||
and pull requests to this repository's maintainers. No separate Apache-2.0
|
||||
license grant to SILO or upstream MinIO maintainers is required.
|
||||
|
||||
* **No CLA.** We do not ask you to sign a Contributor License Agreement and we
|
||||
do not take your copyright. Contributions are accepted inbound=outbound: you
|
||||
@@ -108,15 +137,20 @@ fork does not try to.
|
||||
`Signed-off-by` trailers) and add your own sign-off as the person passing it
|
||||
along. Never import code from a proprietary distribution.
|
||||
|
||||
* **File headers.** Files derived from upstream keep the original MinIO
|
||||
copyright header unchanged. New files added by this fork use the dual
|
||||
header, followed by the standard AGPL boilerplate:
|
||||
* **File headers.** Preserve existing copyright and license notices in inherited
|
||||
and third-party files. New original files name their actual copyright holders
|
||||
and use AGPL-3.0-or-later. Use a header such as the following, then append the
|
||||
standard AGPL boilerplate:
|
||||
|
||||
```
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
// Copyright (c) 2026 Your Name
|
||||
```
|
||||
|
||||
* **Separately licensed material.** Documentation contributions in `docs/`
|
||||
follow its existing [CC BY 4.0 license](docs/LICENSE). Third-party components
|
||||
and earlier Apache-2.0 contributions retain their original licenses and
|
||||
attribution; this policy does not relicense earlier work.
|
||||
|
||||
* **Squash merges** must keep the `Signed-off-by:` trailers in the resulting
|
||||
commit message.
|
||||
|
||||
|
||||
+214
File diff suppressed because one or more lines are too long
+24
-2
@@ -8,10 +8,30 @@
|
||||
# seeds fresh volumes from the image-layer mountpoint, no entrypoint
|
||||
# exists to repair ownership at runtime, and 0777 is what keeps every
|
||||
# privilege mode working, --user included.
|
||||
FROM busybox:1.37.0 AS prep
|
||||
FROM busybox:1.37.0@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec AS prep
|
||||
RUN mkdir -p /prep/data && chmod 0777 /prep/data
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:latest
|
||||
# Debian's bundle in distroless still trusts the roots the classic image
|
||||
# blocklists. Build the same bundle as Dockerfile.goreleaser so both variants
|
||||
# trust the same roots; it is architecture-neutral, so build it natively.
|
||||
FROM --platform=$BUILDPLATFORM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
|
||||
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
|
||||
RUN dnf -y install ca-certificates && \
|
||||
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
|
||||
blocklist=/etc/pki/ca-trust/source/blocklist; \
|
||||
else \
|
||||
blocklist=/etc/pki/ca-trust/source/blacklist; \
|
||||
fi && \
|
||||
trust extract --overwrite --format=pem-bundle \
|
||||
--filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
|
||||
"$blocklist/Comodo_AAA_Services_root.pem" && \
|
||||
trust extract --overwrite --format=pem-bundle \
|
||||
--filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
|
||||
"$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
|
||||
update-ca-trust && \
|
||||
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:latest@sha256:d75cdd72874d4790092fcb1b058493ecf6bb5bf2b2b897045b00ff01d91843f2
|
||||
|
||||
LABEL org.opencontainers.image.title="Silo" \
|
||||
org.opencontainers.image.description="S3-Interface Libre Object Storage (distroless)" \
|
||||
@@ -28,6 +48,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
HOME=/tmp
|
||||
|
||||
# The only file under /etc/ssl/certs, so replacing it is sufficient.
|
||||
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY --chmod=0755 silo /usr/bin/silo
|
||||
# COPY of a directory copies its contents, not the directory entry, so an
|
||||
# empty /prep/data would arrive as a default root:0755 /data and non-root
|
||||
|
||||
+36
-12
@@ -1,4 +1,15 @@
|
||||
FROM golang:1.26.5-alpine AS build
|
||||
FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS curl-build
|
||||
ARG TARGETARCH
|
||||
COPY dockerscripts/build-static-curl.sh /build/build-static-curl
|
||||
RUN /bin/sh /build/build-static-curl
|
||||
|
||||
# Exercise the exact shipped curl without a dynamic loader or shared libraries.
|
||||
FROM scratch AS curl-runtime
|
||||
COPY --from=curl-build /go/bin/curl /curl
|
||||
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/curl"]
|
||||
|
||||
FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
@@ -6,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-08-06T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=4b488bd30af54ad4214e5b654746677c79cd93dc6cad4be3aa2d09dbb48370ff
|
||||
ARG MC_ARM64_SHA256=83f6fedb16ed9c1e8efa8aea6776203dff132bc474214543d5c0767ed2066c2f
|
||||
ARG MC_VERSION=RELEASE.2026-11-09T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=79e6cf4ad00a9e5eb2e7ceccc6e6496e4af4ec096e59695515ea8f223c287e4a
|
||||
ARG MC_ARM64_SHA256=5473c4b39a31dd66478f5c44686a09f3e9e37861a30f1faf18564fbe66118387
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
@@ -54,20 +65,30 @@ RUN apk add -U --no-cache \
|
||||
echo "No mc or mcli binary found in archive:"; ls -la /tmp/mcli-extract/; exit 1; \
|
||||
fi && \
|
||||
chmod +x /go/bin/mcli && \
|
||||
ln -sf mcli /go/bin/mc
|
||||
ln -sf mcli /go/bin/mc && \
|
||||
mkdir -p /go/share/mcli && \
|
||||
cp /tmp/mcli-extract/LICENSE /tmp/mcli-extract/NOTICE /tmp/mcli-extract/CREDITS /go/share/mcli/
|
||||
|
||||
COPY dockerscripts/download-static-curl.sh /build/download-static-curl
|
||||
RUN chmod +x /build/download-static-curl && \
|
||||
/build/download-static-curl
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest AS certs
|
||||
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
|
||||
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
|
||||
RUN dnf -y install ca-certificates && \
|
||||
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
|
||||
blocklist=/etc/pki/ca-trust/source/blocklist; \
|
||||
else \
|
||||
blocklist=/etc/pki/ca-trust/source/blacklist; \
|
||||
fi && \
|
||||
trust extract --overwrite --format=pem-bundle \
|
||||
--filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
|
||||
"$blocklist/Comodo_AAA_Services_root.pem" && \
|
||||
trust extract --overwrite --format=pem-bundle \
|
||||
--filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
|
||||
"$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
|
||||
update-ca-trust && \
|
||||
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
|
||||
dnf clean all && \
|
||||
rm -rf /var/cache/dnf
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6
|
||||
|
||||
LABEL org.opencontainers.image.title="Silo" \
|
||||
org.opencontainers.image.description="S3-Interface Libre Object Storage" \
|
||||
@@ -88,7 +109,10 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
COPY --from=certs /tmp/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY silo /usr/bin/silo
|
||||
COPY --from=build /go/bin/mcli /usr/bin/mcli
|
||||
COPY --from=build /go/bin/curl* /usr/bin/
|
||||
# mcli's own legal files, from the same verified archive as the binary.
|
||||
COPY --from=build /go/share/mcli /licenses/mcli
|
||||
COPY --from=curl-build /go/bin/curl /usr/bin/curl
|
||||
COPY --from=curl-build /go/share/curl /licenses/curl
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY NOTICE /licenses/NOTICE
|
||||
|
||||
@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
|
||||
GOOS ?= $(shell go env GOOS)
|
||||
GOARCH ?= $(shell go env GOARCH)
|
||||
GOLANGCI_VERSION ?= v2.11.3
|
||||
GOLANGCI_VERSION ?= v2.15.0
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
REPO ?= docker.io/pgsty
|
||||
@@ -48,7 +48,7 @@ credits: ## regenerate CREDITS from the licenses of Go modules linked into the b
|
||||
|
||||
check-gen: ## check for updated autogenerated files
|
||||
@go generate ./... >/dev/null
|
||||
@go mod tidy -compat=1.26
|
||||
@go mod tidy -compat=1.27
|
||||
@env bash $(PWD)/buildscripts/gen-credits.sh
|
||||
@changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum CREDITS); \
|
||||
if [ -n "$$changed" ]; then \
|
||||
@@ -73,6 +73,11 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
|
||||
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
|
||||
|
||||
check: test
|
||||
# -tags kqueue,dev: kqueue selects the kqueue backend of rjeczalik/notify on
|
||||
# BSD and macOS (no-op on Linux); dev came with upstream's 2024-06 change for
|
||||
# development builds and no file in this repository or its dependencies
|
||||
# carries a dev build constraint, so it selects nothing. Kept for the
|
||||
# documented command lines; .github/workflows/go.yml carries the same note.
|
||||
test: verifiers build ## builds Silo, runs linters, tests
|
||||
@echo "Running unit tests"
|
||||
@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue,dev ./...
|
||||
@@ -216,7 +221,7 @@ docker: checks build-debugging ## builds the local Linux Silo container image
|
||||
--ldflags "$(LDFLAGS)" -o "$$context/silo"; \
|
||||
mkdir -p "$$context/dockerscripts"; \
|
||||
cp Dockerfile.goreleaser LICENSE NOTICE CREDITS "$$context/"; \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/download-static-curl.sh \
|
||||
cp dockerscripts/docker-entrypoint.sh dockerscripts/build-static-curl.sh \
|
||||
"$$context/dockerscripts/"; \
|
||||
docker build -q --no-cache --platform linux/$(GOARCH) -t $(TAG) --build-arg TARGETARCH=$(GOARCH) \
|
||||
-f "$$context/Dockerfile.goreleaser" "$$context"
|
||||
|
||||
@@ -6,15 +6,17 @@
|
||||
|
||||
|
||||
<p align="center">
|
||||
<strong>S3-compatible object storage — a MinIO fork maintained by PIGSTY</strong>
|
||||
<strong>S3-compatible object storage — a MinIO fork maintained by PGSTY</strong>
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/">Website</a> ·
|
||||
<a href="https://silo.pgsty.com/docs/">Documentation</a> ·
|
||||
<a href="https://silo.pgsty.com/download/">Download</a> ·
|
||||
<a href="https://silo.pgsty.com/tags/silo/">Release Notes</a> ·
|
||||
<a href="https://silo.pgsty.com/compatibility/server/">Compatibility</a> ·
|
||||
<a href="https://silo.pgsty.com/about/manifesto/">Manifesto</a> ·
|
||||
<a href="SECURITY.md">Security</a> ·
|
||||
<a href="README_ZH.md">中文</a>
|
||||
</p>
|
||||
@@ -22,245 +24,152 @@
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/"><img alt="Website" src="https://img.shields.io/badge/Website-silo.pgsty.com-1d588c"></a>
|
||||
<a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
|
||||
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
|
||||
</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Silo is an independent, community-maintained fork of the open-source MinIO server, published by [Pigsty](https://pigsty.io) from [`pgsty/silo`](https://github.com/pgsty/silo). It is not affiliated with, endorsed by, or sponsored by MinIO, Inc. “MinIO” is used only to identify the upstream project and compatibility lineage.
|
||||
> **PGSTY Silo** (hereinafter “Silo”) is an independent, community-maintained fork of the open-source MinIO server, published by [Pigsty](https://pigsty.io) from [`pgsty/silo`](https://github.com/pgsty/silo). It is not affiliated with, endorsed by, or sponsored by MinIO, Inc. “MinIO” is used only to identify the upstream project and compatibility lineage.
|
||||
|
||||
> [!NOTE]
|
||||
> This repository was renamed from `pgsty/minio` to `pgsty/silo`, and its default branch from `master` to `main`, on 2026-08-06. If you need the artifacts maintained under the original MinIO identity, they are on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in the releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z); those assets and the `docker.io/pgsty/minio` image stay published and unmodified. The rename changed product and artifact names only — `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, and the on-disk format are unchanged.
|
||||
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
|
||||
|
||||
## SILO 20261026000000
|
||||
|
||||
[SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) is available as platform archives, Linux packages, and standard or distroless Docker images for amd64 and arm64. The release tag is `RELEASE.2026-10-26T00-00-00Z`; the package version is `20261026000000.0.0`, built from source `c3ad836aed0d`.
|
||||
|
||||
This release fixes LDAP long-DN login, IAM policy refresh, multi-pool deletion safety, tiered-object expiry, multipart replication checksums, and remote rollback-option propagation. It embeds Console v2.5.1 (`fff1618d9abb`) and uses silo-pkg v3.15.2, Go 1.27.3, and upstream minio-go `v7.4.2-0.20261025165926-320af050c445`. Use mcli 20261026 for administration; the container images bundle it as `mcli`.
|
||||
|
||||
The Server IAM cache repair enables Console's **Reset to default** action. Existing storage timeouts and cancellation behavior remain: a timeout does not prove that a disk operation stopped or a write did not commit. Read the [coordinated release notes](https://silo.pgsty.com/blog/release/silo-20261026/) and [remaining storage limits](CHANGELOG.md#release2026-10-26t00-00-00z) before upgrading. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people behind the code, reports, and documentation.
|
||||
|
||||
## Overview
|
||||
|
||||
Silo maintains one downstream release line derived from the open-source MinIO server. It provides maintained builds and release artifacts for existing MinIO-compatible deployments after upstream community distribution ended. Pigsty uses Silo for object storage, including as an optional PostgreSQL backup repository.
|
||||
PGSTY SILO keeps one maintained release line of the open-source MinIO server alive after upstream ended community distribution: builds, packages, multi-arch images, security fixes, and the full web console. Pigsty runs it in production as its PostgreSQL backup repository.
|
||||
|
||||
The fork follows one rule: **the product and its delivery surfaces are renamed; the protocol and your data are not.** The `silo` executable, packages, service, and container image carry the new name, while the S3 API, `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, reserved routes, and the on-disk format stay unchanged. The differences are recorded in the [compatibility notes](https://silo.pgsty.com/compatibility/server/).
|
||||
It follows one rule — **the product and its delivery surfaces are renamed; the protocol and your data are not.** Everything else lives on [silo.pgsty.com](https://silo.pgsty.com/).
|
||||
|
||||
The official project portal is [silo.pgsty.com](https://silo.pgsty.com/). It brings documentation, downloads, release and security notes, and project background together.
|
||||
**Related:** [`pgsty/mc`](https://github.com/pgsty/mc) client (shipped as `mcli`) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
||||
|
||||
## Find the Right Resource
|
||||
|
||||
| Looking for | Canonical location |
|
||||
| :-- | :-- |
|
||||
| Project overview and navigation | [Silo Website](https://silo.pgsty.com/) |
|
||||
| Installation methods and downloads | [Download & Install](https://silo.pgsty.com/download/) |
|
||||
| Operations, administration, development, and reference | [Documentation](https://silo.pgsty.com/docs/) |
|
||||
| Release notes for this server | [Silo release notes](https://silo.pgsty.com/tags/silo/) |
|
||||
| Differences from upstream MinIO | [Silo vs. MinIO compatibility notes](https://silo.pgsty.com/compatibility/server/) |
|
||||
| Project news and security advisories | [Blog](https://silo.pgsty.com/blog/) · [release](https://silo.pgsty.com/blog/release/) and [security](https://silo.pgsty.com/blog/security/) notes |
|
||||
| Versioned binaries, checksums, and source archives | [GitHub Releases](https://github.com/pgsty/silo/releases) |
|
||||
| Bug reports and feature discussions | [GitHub Issues](https://github.com/pgsty/silo/issues) |
|
||||
| Vulnerability reporting | [`SECURITY.md`](SECURITY.md) and [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) |
|
||||
| License, attribution, and trademark information | [`LICENSE`](LICENSE) · [`NOTICE`](NOTICE) · [`CREDITS`](CREDITS) · portal [license](https://silo.pgsty.com/about/license/), [attribution](https://silo.pgsty.com/about/attribution/), and [trademark](https://silo.pgsty.com/about/trademark/) pages |
|
||||
|
||||
## Related Projects
|
||||
|
||||
| Repository | Description |
|
||||
| :-- | :-- |
|
||||
| [`pgsty/silo`](https://github.com/pgsty/silo) | This repository — the Silo object storage server, an S3-compatible MinIO fork |
|
||||
| [`pgsty/mc`](https://github.com/pgsty/mc) | The Silo command-line client, shipped as `mcli` with the `mc` command name |
|
||||
| [`pgsty/silo-console`](https://github.com/pgsty/silo-console) | Admin web console, embedded in the server build |
|
||||
| [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) | Shared Go packages maintained for the Silo forks |
|
||||
| [`pgsty/pigsty`](https://github.com/pgsty/pigsty) | Pigsty — the PostgreSQL distribution that ships Silo as its object storage |
|
||||
|
||||
## Maintenance Policy
|
||||
|
||||
The active release line covers:
|
||||
|
||||
- build and dependency maintenance;
|
||||
- applicable security fixes and advisories;
|
||||
- focused fixes for reproducible defects;
|
||||
- versioned binaries, packages, checksums, and multi-architecture images;
|
||||
- the web console, client, documentation, and Pigsty integration.
|
||||
|
||||
Changes are kept narrow and tested where practical. Maintenance is best effort; no response, remediation, or release schedule is guaranteed.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- a separate product roadmap, new storage engine, or speculative S3 features;
|
||||
- broad rewrites or changes that materially expand the downstream delta;
|
||||
- historical releases or multiple support branches;
|
||||
- commercial support, SLAs, 24×7 coverage, or SUBNET access;
|
||||
- deployment design, access control, monitoring, backup, or recovery.
|
||||
|
||||
## Governance
|
||||
|
||||
The server, client, console, and shared packages are maintained under one release process: DCO-signed commits, reviewed pull requests, and versioned `RELEASE.YYYY-MM-DDTHH-MM-SSZ` tags whose artifacts ship with checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub build attestations. Each release is announced with a [release note](https://silo.pgsty.com/tags/silo/) on the portal; security handling follows [`SECURITY.md`](SECURITY.md) and the [advisory log](docs/security/advisories.md). Upstream copyright, license, and third-party notices are preserved in [`LICENSE`](LICENSE), [`NOTICE`](NOTICE), and [`CREDITS`](CREDITS).
|
||||
|
||||
## Compatibility
|
||||
|
||||
Silo aims to preserve:
|
||||
|
||||
- the `github.com/minio/minio` module path and `github.com/minio/*` import paths;
|
||||
- MinIO-compatible S3 APIs, wire behavior, `MINIO_*` environment variables, metrics, protocol headers, reserved routes, and storage metadata;
|
||||
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` tags and legacy `minio …` container argv translation.
|
||||
|
||||
Silo-owned delivery surfaces use the `silo` executable, package, service, Helm chart, and `pgsty/silo` container image. Native artifacts intentionally do not install a `minio` binary alias.
|
||||
|
||||
Compatibility is the default constraint. Silo preserves existing wire, client, configuration, and operational behavior whenever doing so remains safe. Compatibility is broken only when necessary to close a major security issue, and the release notes must identify the affected behavior and migration path. Treat each release as a downstream upgrade: pin versions, review the [release notes](https://silo.pgsty.com/tags/silo/), [compatibility notes](https://silo.pgsty.com/compatibility/server/), and [security advisories](docs/security/advisories.md), keep a rollback path, and test before production use.
|
||||
|
||||
## Downloads and Release Artifacts
|
||||
|
||||
Use [Download & Install](https://silo.pgsty.com/download/) to choose an installation method. GitHub Releases remains the source for versioned server binaries, checksums, and source archives.
|
||||
|
||||
| Artifact | Location |
|
||||
| :-- | :-- |
|
||||
| Source | [`github.com/pgsty/silo`](https://github.com/pgsty/silo) |
|
||||
| Container image | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo), multi-arch for `linux/amd64` and `linux/arm64` |
|
||||
| Server binaries and checksums | [GitHub Releases](https://github.com/pgsty/silo/releases) for Linux, macOS, and Windows on `amd64` and `arm64` |
|
||||
| Linux packages | RPM, DEB, and APK artifacts, also distributed through the [Pigsty repository](https://pigsty.io/docs/repo/) |
|
||||
| Client | [`pgsty/mc`](https://github.com/pgsty/mc), bundled in the container as `mcli` with an `mc` compatibility alias |
|
||||
| Console | [`pgsty/silo-console`](https://github.com/pgsty/silo-console), embedded through the compatibility import path `github.com/minio/console` |
|
||||
| Shared library | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.11.0, consumed through a `replace` directive while preserving the `github.com/minio/pkg/v3` import path |
|
||||
|
||||
Each new release publishes per-archive and per-package SPDX JSON SBOMs. The archive and package checksum manifests have detached keyless Sigstore bundles, while GitHub artifact attestations record signed provenance for every downloadable artifact and the multi-architecture container image.
|
||||
|
||||
After downloading an archive and its release files, verify integrity, the
|
||||
published SBOM, the signed manifest, and build provenance independently:
|
||||
|
||||
```bash
|
||||
# Integrity: choose the line for the artifact you downloaded.
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# The archive SBOM is a separate checksummed release artifact.
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz.sbom.json' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# Signature over the archive/SBOM checksum manifest.
|
||||
cosign verify-blob \
|
||||
--bundle silo_<version>_checksums.txt.sigstore.json \
|
||||
--certificate-identity-regexp \
|
||||
'^https://github.com/pgsty/(minio|silo)/\.github/workflows/release\.yml@refs/(tags/RELEASE\..+|heads/(master|main))$' \
|
||||
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
||||
silo_<version>_checksums.txt
|
||||
|
||||
# Signed build provenance (online verification against this repository).
|
||||
gh attestation verify silo_<version>_linux_amd64.tar.gz \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
For packages, use `silo_<version>_packages_checksums.txt` and its adjacent
|
||||
Sigstore bundle with the same identity and issuer constraints. Inspect the
|
||||
verified SPDX JSON SBOM with your preferred SPDX tooling. Verify the
|
||||
multi-architecture container provenance by digest:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<manifest-digest> \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
The platform SBOM attestations are attached to the `amd64` and `arm64` image
|
||||
digests rather than the multi-architecture manifest. Verify each one explicitly:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<platform-digest> \
|
||||
--repo pgsty/silo \
|
||||
--predicate-type https://spdx.dev/Document/v2.3
|
||||
```
|
||||
|
||||
Verification by digest avoids trusting a mutable image tag.
|
||||
|
||||
### Native package migration
|
||||
|
||||
The `silo` RPM, DEB, and APK do not declare `Provides`, `Obsoletes`,
|
||||
`Replaces`, or package-level `Conflicts` against `minio`. They can therefore be
|
||||
installed beside an existing MinIO package without silently replacing it. The
|
||||
two systemd units conflict at runtime, so switch them explicitly rather than
|
||||
starting both.
|
||||
|
||||
Before switching, record the old unit's enabled/active state and `User`/`Group`,
|
||||
and back up `/etc/default/minio`. Silo reads that legacy defaults file first and
|
||||
then `/etc/default/silo`; administrator-set values in the latter take
|
||||
precedence. If the existing data must continue to run under its original
|
||||
UID/GID, create `/etc/systemd/system/silo.service.d/10-legacy-user.conf`:
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
User=<legacy-user>
|
||||
Group=<legacy-group>
|
||||
```
|
||||
|
||||
Run `systemctl daemon-reload`, then disable and stop `minio.service` before
|
||||
enabling and starting `silo.service`. Verify health, S3, Admin API, metrics, and
|
||||
logs before masking or uninstalling the old service. Do not recursively change
|
||||
data ownership as part of the package migration; keep the old package and unit
|
||||
available during the rollback window.
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo Console">
|
||||
</p>
|
||||
|
||||
## Quick Start
|
||||
|
||||
For local evaluation:
|
||||
|
||||
```bash
|
||||
mkdir -p data
|
||||
|
||||
export MINIO_ROOT_USER=minioadmin
|
||||
export MINIO_ROOT_PASSWORD=change-me-long-password
|
||||
|
||||
docker run -d --name silo \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-e MINIO_ROOT_USER \
|
||||
-e MINIO_ROOT_PASSWORD \
|
||||
docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:latest server /data --console-address ":9001"
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
Open the console at <http://localhost:9001>; the S3 API listens on <http://localhost:9000>.
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-login.webp" alt="Silo Console">
|
||||
</p>
|
||||
|
||||
The image includes the compatible client as `mcli`:
|
||||
Console on <http://localhost:9001>, S3 API on <http://localhost:9000>. The image bundles the client as `mcli`:
|
||||
|
||||
```bash
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
docker exec silo mcli mb local/demo
|
||||
docker exec silo mcli ls local
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 minioadmin change-me-long-password
|
||||
docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> For production, pin a release, use unique credentials and TLS, monitor the service, keep independent backups, and test recovery.
|
||||
> For production, pin a release, use unique credentials and TLS, monitor the service, keep independent backups, and test recovery. Start from the [documentation](https://silo.pgsty.com/docs/).
|
||||
|
||||
Build the server from source:
|
||||
## Install
|
||||
|
||||
```bash
|
||||
go build -o silo .
|
||||
./silo --version
|
||||
```
|
||||
| Method | Where |
|
||||
| :-- | :-- |
|
||||
| Container | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo), multi-arch for `linux/amd64` and `linux/arm64` |
|
||||
| Binaries | [GitHub Releases](https://github.com/pgsty/silo/releases) — Linux, macOS, Windows on `amd64` and `arm64` |
|
||||
| Packages | RPM, DEB, and APK, also via the [Pigsty repository](https://pigsty.io/docs/repo/) |
|
||||
| Kubernetes | Helm chart, see [Download & Install](https://silo.pgsty.com/download/) |
|
||||
| Source | `go build -o silo . && ./silo --version` |
|
||||
|
||||
For other installation paths—including native packages, binaries, Podman, Kubernetes, source, and Pigsty Ansible—use [Download & Install](https://silo.pgsty.com/download/). For production deployment and administration, start with the [Silo documentation](https://silo.pgsty.com/docs/). Pigsty users can also use the [Pigsty MinIO module](https://pigsty.io/docs/minio/).
|
||||
Every release ships checksums, SPDX SBOMs, Sigstore-signed manifests, and GitHub build attestations. Installation methods and verification commands are documented at [Download & Install](https://silo.pgsty.com/download/); migrating from upstream MinIO — taking over an existing `minio.service` and its `/etc/default/minio`, and keeping data ownership stable with a `/etc/systemd/system/silo.service.d/10-legacy-user.conf` drop-in — is covered by the [migration guide](https://silo.pgsty.com/compatibility/migration/) and the [binary & service notes](https://silo.pgsty.com/compatibility/binary/).
|
||||
|
||||
## Security
|
||||
## Compatibility
|
||||
|
||||
Security fixes target the active development branch and are recorded in the [advisory log](docs/security/advisories.md) and the portal's [security notes](https://silo.pgsty.com/blog/security/). Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md) and [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md). Report issues that also affect upstream MinIO there as well.
|
||||
Silo preserves S3 and storage-format compatibility, including existing `MINIO_*` variables, `minio_*` metrics, `x-minio-*` headers, `/minio/*` routes, and `.minio.sys` data. CI guards selected compatibility identifiers; release notes document intentional security and behavior changes. Silo-owned delivery surfaces use the `silo` executable, package, service, Helm chart, and container image; no `minio` server binary alias is installed.
|
||||
|
||||
## Contributing
|
||||
The supported release stack is `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`; compatibility with unmodified upstream MinIO/MC is best effort. The Server, Console, and client retain their historical module paths where needed, while maintained code imports `github.com/pgsty/silo-pkg/v3` directly. The SDK `github.com/minio/minio-go/v7` is an explicit upstream dependency. See the current [go.mod](go.mod) for versions and replacements.
|
||||
|
||||
Useful contributions include security and dependency updates, reproducible bug fixes, tests, release automation, packaging, and documentation.
|
||||
Every divergence from upstream is listed in the code-verified [compatibility audit](https://silo.pgsty.com/compatibility/server/). Treat each release as a downstream upgrade: pin versions, read the [release notes](https://silo.pgsty.com/tags/silo/), and keep a rollback path.
|
||||
|
||||
Issues and pull requests should include the affected version, reproduction steps, impact, expected behavior, tests, and compatibility notes. Discuss large changes in an issue first.
|
||||
### TLS and Go upgrades
|
||||
|
||||
There is no CLA: contributions are accepted inbound=outbound under the project license (AGPL-3.0-or-later) and contributors keep their copyright. Every commit must be signed off (`git commit -s`) per the [Developer Certificate of Origin](https://developercertificate.org/); see [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
||||
Server 20260916 restored Go's default key exchange; Server 20261018 retains it with Go 1.27.2.
|
||||
TLS key exchange follows Go's defaults across the S3 listener, node links,
|
||||
replication, identity providers, etcd, and external HTTP services. If an endpoint
|
||||
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
|
||||
for the process; certificate verification remains enabled. This option does not
|
||||
disable ML-DSA signatures or resolve every TLS reset. Prefer updating the
|
||||
incompatible endpoint before removing the temporary setting.
|
||||
If only the new SecP hybrids cause problems, `GODEBUG=tlssecpmlkem=0` disables
|
||||
those groups while retaining X25519MLKEM768.
|
||||
|
||||
For builds targeting Go 1.27, setting either `SSL_CERT_FILE` or `SSL_CERT_DIR`
|
||||
on macOS replaces Keychain trust with on-disk roots and Go's verifier. Stale or
|
||||
incomplete CA paths can break previously trusted connections; unset inherited
|
||||
values to restore Keychain trust. Explicit certificates in the configured `CAs`
|
||||
directory remain additive to the selected root pool.
|
||||
Go 1.27 binaries require macOS 13 or later. See the
|
||||
[Go release notes](https://go.dev/doc/go1.27) and the
|
||||
[Go 1.27 TLS and OIDC discovery guide](https://silo.pgsty.com/blog/design/go127-tls-oidc-discovery/).
|
||||
|
||||
## Documentation ownership
|
||||
|
||||
User documentation is maintained at [silo.pgsty.com](https://silo.pgsty.com/docs/),
|
||||
with source in [pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com).
|
||||
The remaining `docs/` tree contains inherited references, examples, and tooling
|
||||
fixtures. Investigation logs, AI work records, and temporary reports are kept
|
||||
outside this repository; reusable findings belong in the companion site.
|
||||
See [AGENTS.md](AGENTS.md) for repository ownership and maintenance rules.
|
||||
|
||||
## Security & Contributing
|
||||
|
||||
Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); every fix ships with a public [advisory](https://silo.pgsty.com/blog/security/). Contributions are accepted inbound=outbound under AGPL-3.0-or-later with no CLA — only DCO sign-off (`git commit -s`) is required; see [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
||||
|
||||
## Contributors
|
||||
|
||||
<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
|
||||
|
||||
Every human issue or pull-request author is part of the SILO community, including open and unmerged work. We also recognize security reports, including reports sent by email, with the reporter's permission. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
|
||||
|
||||
<a href="CONTRIBUTORS.md">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-light.svg" alt="SILO community contributors">
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
[View contribution notes and actual PR status](CONTRIBUTORS.md).
|
||||
|
||||
## Star History
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-light.svg" alt="SILO GitHub star history">
|
||||
</picture>
|
||||
|
||||
## Background
|
||||
|
||||
This project was created in response to changes in the upstream community distribution and maintenance model. The maintainer’s analysis, alternatives considered, and early maintenance record are documented below:
|
||||
Upstream wound down its community edition: the web console was cut back to a stub, prebuilt community binaries stopped, and the community repository was archived. Silo exists to keep those deployments running. The fork is a means, not an identity — if upstream restores its community edition, we will narrow our scope and offer the fixes back.
|
||||
|
||||
| Essay | Subject |
|
||||
| :-- | :-- |
|
||||
| [MinIO Is Dead](https://silo.pgsty.com/blog/post/minio-is-dead/) | Changes to the upstream project and distribution model |
|
||||
| [MinIO Is Dead, Who Takes Over?](https://silo.pgsty.com/blog/post/minio-alternative/) | Alternatives considered |
|
||||
| [MinIO Is Dead, Long Live MinIO](https://silo.pgsty.com/blog/post/minio-resurrect/) | Establishing the fork and its release pipeline |
|
||||
| [Two months into maintaining a MinIO fork](https://silo.pgsty.com/blog/post/minio-promise-kept/) | Initial security and maintenance work |
|
||||
The [**Manifesto**](https://silo.pgsty.com/about/manifesto/) is the project's public commitment in eleven articles, under one discipline: every article is either something already done with public evidence, or something explicitly refused. In short:
|
||||
|
||||
## License and Trademark
|
||||
- **Compatibility contract** — the protocol and your data do not change, and every release documents its tested rollback target and path.
|
||||
- **The license cannot change** — AGPLv3, no CLA, no copyright aggregation; nobody here, ourselves included, holds enough copyright to relicense on everyone else's behalf.
|
||||
- **The never list**, append-only — no paywalling existing features, no registration wall on downloads, no telemetry (upstream's phone-home paths are removed outright), no CLA, no license change, no trademark enforcement against normal use.
|
||||
- **Security and release discipline** — a public advisory for every fix, and a release every one to two months, at most a quarter apart. Judge both against the public record.
|
||||
|
||||
The server source is distributed under the [GNU Affero General Public License v3.0 or later](LICENSE). This fork derives from [`minio/minio`](https://github.com/minio/minio): [`NOTICE`](NOTICE) retains the upstream product notice, [`CREDITS`](CREDITS) records licenses and notices for included third-party components, and the Git history records downstream modifications.
|
||||
Essays: [MinIO Is Dead](https://silo.pgsty.com/blog/post/minio-is-dead/) · [Who Takes Over?](https://silo.pgsty.com/blog/post/minio-alternative/) · [Long Live MinIO](https://silo.pgsty.com/blog/post/minio-resurrect/) · [Promise Kept](https://silo.pgsty.com/blog/post/minio-promise-kept/)
|
||||
|
||||
MinIO is a trademark of MinIO, Inc. The name is used here only to identify the upstream project and compatibility lineage. Pigsty, Silo, and `pgsty/silo` are independent community efforts and are not affiliated with, endorsed by, or sponsored by MinIO, Inc.
|
||||
## License & Trademark
|
||||
|
||||
The portal separately publishes the project [license summary](https://silo.pgsty.com/about/license/), [documentation attribution](https://silo.pgsty.com/about/attribution/), and [trademark notice](https://silo.pgsty.com/about/trademark/).
|
||||
Silo is [AGPL-3.0-or-later](LICENSE), derived from [`minio/minio`](https://github.com/minio/minio) with upstream copyright and third-party notices preserved in [`NOTICE`](NOTICE) and [`CREDITS`](CREDITS). MinIO is a trademark of MinIO, Inc.; the name is used here only to identify the upstream project and compatibility lineage.
|
||||
|
||||
Details: [license](https://silo.pgsty.com/about/license/) · [attribution](https://silo.pgsty.com/about/attribution/) · [trademark](https://silo.pgsty.com/about/trademark/)
|
||||
|
||||
+94
-197
@@ -1,20 +1,22 @@
|
||||
<h1 align="center">
|
||||
<a href="https://silo.pgsty.com/zh/">
|
||||
<img src=".github/silo-logo.svg" alt="Silo" width="112"><br>
|
||||
<img src=".github/silo-logo.svg" alt="Silo" width="160">
|
||||
</a>
|
||||
</h1>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<strong>S3 兼容对象存储 —— 由 PIGSTY 维护的 MinIO 社区分支</strong>
|
||||
<strong>S3 兼容对象存储 —— 由 PGSTY 维护的 MinIO 社区分支</strong>
|
||||
</p>
|
||||
|
||||
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/zh/">官网</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/docs/">文档</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/download/">下载</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/tags/silo/">版本说明</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/compatibility/server/">兼容性</a> ·
|
||||
<a href="https://silo.pgsty.com/zh/about/manifesto/">宣言</a> ·
|
||||
<a href="SECURITY.md">安全策略</a> ·
|
||||
<a href="README.md">English</a>
|
||||
</p>
|
||||
@@ -22,240 +24,135 @@
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/zh/"><img alt="官网" src="https://img.shields.io/badge/%E5%AE%98%E7%BD%91-silo.pgsty.com%2Fzh-1d588c"></a>
|
||||
<a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
|
||||
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
|
||||
</p>
|
||||
|
||||
> [!IMPORTANT]
|
||||
> Silo 是由 [Pigsty](https://pigsty.cc) 独立维护、从 [`pgsty/silo`](https://github.com/pgsty/silo) 发布的开源 MinIO 社区分支。本项目与 MinIO, Inc. 不存在隶属、背书或赞助关系;文中使用 “MinIO” 仅用于说明上游项目及兼容谱系。
|
||||
> **PGSTY Silo**(以下简称 Silo)是由 [Pigsty](https://pigsty.cc) 独立维护、从 [`pgsty/silo`](https://github.com/pgsty/silo) 发布的开源 MinIO 社区分支。本项目与 MinIO, Inc. 不存在隶属、背书或赞助关系;文中使用 “MinIO” 仅用于说明上游项目及兼容谱系。
|
||||
|
||||
> [!NOTE]
|
||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。如果你需要以原本 MinIO 形态维持的归档构件,它们位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止到 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中;这些资产与 `docker.io/pgsty/minio` 镜像保持已发布状态且不做改动。本次更名只改变产品与交付物名称 —— `MINIO_*` 变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由与磁盘格式均保持不变。
|
||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
|
||||
|
||||
## SILO 20261026000000
|
||||
|
||||
[SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) 已正式发布,提供各平台归档包、Linux 软件包,以及 amd64/arm64 标准版和 distroless Docker 镜像。版本标签为 `RELEASE.2026-10-26T00-00-00Z`,包版本为 `20261026000000.0.0`,构建源码为 `c3ad836aed0d`。
|
||||
|
||||
本版修复 LDAP 长 DN 登录、IAM 策略刷新、多池删除安全、分层对象过期、分片复制校验和及远端回滚参数透传。内置 Console v2.5.1(`fff1618d9abb`),使用 silo-pkg v3.15.2、Go 1.27.3 与上游 minio-go `v7.4.2-0.20261025165926-320af050c445`。配套管理客户端为 mcli 20261026,容器镜像内置该版本的 `mcli`。
|
||||
|
||||
Server IAM 缓存修复使 Console **Reset to default(恢复默认)** 操作即时生效。存储超时与取消行为保持既有语义:超时不代表底层磁盘操作已停止,也不代表写入未提交。升级前请阅读[联合发布说明](https://silo.pgsty.com/zh/blog/release/silo-20261026/)与[仍存的存储限制](CHANGELOG.md#release2026-10-26t00-00-00z)。代码、问题报告和文档贡献者见 [CONTRIBUTORS.md](CONTRIBUTORS.md)。
|
||||
|
||||
## 概述
|
||||
|
||||
Silo 维护一条源自开源 MinIO 服务端的下游版本线,为上游停止社区发行后仍在运行 MinIO 兼容部署的用户提供持续构建与发行产物。Pigsty 使用 Silo 提供对象存储,包括作为可选的 PostgreSQL 备份仓库。
|
||||
上游停止社区发行后,Silo 为开源 MinIO 服务端维护一条持续可用的版本线:构建、软件包、多架构镜像、安全修复与完整 Web 控制台。Pigsty 在生产环境中用它承载 PostgreSQL 备份存储。
|
||||
|
||||
本分支遵循一条原则:**改名的是产品与交付物,不是协议与你的数据。** `silo` 可执行文件、软件包、服务与容器镜像使用新名称,而 S3 API、`MINIO_*` 变量、`minio_*` 指标、`x-minio-*` 头、保留路由与磁盘格式保持不变。具体差异记录在[兼容性说明](https://silo.pgsty.com/zh/compatibility/server/)中。
|
||||
它只遵循一条原则:**改名的是产品与交付物,不是协议与你的数据。** 其余内容都在 [silo.pgsty.com](https://silo.pgsty.com/zh/)。
|
||||
|
||||
项目统一门户为 [silo.pgsty.com](https://silo.pgsty.com/zh/),集中提供文档、下载安装、版本与安全动态及项目背景。
|
||||
**相关项目:**[`pgsty/mc`](https://github.com/pgsty/mc) 客户端(以 `mcli` 发行) · [`pgsty/silo-console`](https://github.com/pgsty/silo-console) · [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) · [`pgsty/pigsty`](https://github.com/pgsty/pigsty)
|
||||
|
||||
## 按需求选择入口
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-metrics-simple.webp" alt="Silo 控制台">
|
||||
</p>
|
||||
|
||||
| 需求 | 权威入口 |
|
||||
| :-- | :-- |
|
||||
| 项目概览与全站导航 | [Silo 官网](https://silo.pgsty.com/zh/) |
|
||||
| 安装方式与软件下载 | [下载与安装](https://silo.pgsty.com/zh/download/) |
|
||||
| 运维、管理、开发与参考指南 | [文档](https://silo.pgsty.com/zh/docs/) |
|
||||
| 本服务端的版本说明 | [Silo 版本说明](https://silo.pgsty.com/zh/tags/silo/) |
|
||||
| 与上游 MinIO 的差异 | [Silo vs. MinIO 兼容性说明](https://silo.pgsty.com/zh/compatibility/server/) |
|
||||
| 项目动态与安全通告 | [博客](https://silo.pgsty.com/zh/blog/) · [版本发布](https://silo.pgsty.com/zh/blog/release/)与[安全通告](https://silo.pgsty.com/zh/blog/security/) |
|
||||
| 带版本的二进制、校验和与源码归档 | [GitHub Releases](https://github.com/pgsty/silo/releases) |
|
||||
| 缺陷报告与功能讨论 | [GitHub Issues](https://github.com/pgsty/silo/issues) |
|
||||
| 私密漏洞报告 | [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) |
|
||||
| 许可证、署名与商标信息 | [`LICENSE`](LICENSE) · [`NOTICE`](NOTICE) · [`CREDITS`](CREDITS) · 门户的[许可证](https://silo.pgsty.com/zh/about/license/)、[署名归属](https://silo.pgsty.com/zh/about/attribution/)与[商标政策](https://silo.pgsty.com/zh/about/trademark/)页面 |
|
||||
|
||||
## 相关项目
|
||||
|
||||
| 仓库 | 说明 |
|
||||
| :-- | :-- |
|
||||
| [`pgsty/silo`](https://github.com/pgsty/silo) | 本仓库 —— Silo 对象存储服务器,S3 兼容的 MinIO fork |
|
||||
| [`pgsty/mc`](https://github.com/pgsty/mc) | Silo 命令行客户端,以 `mcli` 发行、保留 `mc` 命令名 |
|
||||
| [`pgsty/silo-console`](https://github.com/pgsty/silo-console) | Web 管理控制台,内嵌于服务端构建 |
|
||||
| [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) | 为 Silo 系 fork 维护的共享 Go 包 |
|
||||
| [`pgsty/pigsty`](https://github.com/pgsty/pigsty) | Pigsty —— 内置 Silo 作为对象存储的 PostgreSQL 发行版 |
|
||||
|
||||
## 维护政策
|
||||
|
||||
活跃版本线的维护范围包括:
|
||||
|
||||
- 构建与依赖项维护;
|
||||
- 适用的安全修复与公告;
|
||||
- 针对可复现缺陷的范围明确的修复;
|
||||
- 带版本的二进制、软件包、校验和与多架构镜像;
|
||||
- Web Console、客户端、文档与 Pigsty 集成。
|
||||
|
||||
改动保持克制,并在可行时提供测试。所有维护均为尽力而为,不承诺固定的响应、修复或发布时间。
|
||||
|
||||
### 范围之外
|
||||
|
||||
- 独立产品路线图、新存储引擎或假设性的 S3 新特性;
|
||||
- 大规模重写或显著扩大下游差异的改动;
|
||||
- 历史版本或多条支持分支;
|
||||
- 商业支持、SLA、7×24 服务或 SUBNET 服务;
|
||||
- 部署设计、访问控制、监控、备份与恢复。
|
||||
|
||||
## 治理
|
||||
|
||||
服务端、客户端、控制台与共享包在同一套发布流程下共同维护:提交必须签署 DCO、Pull Request 经过评审、以 `RELEASE.YYYY-MM-DDTHH-MM-SSZ` 标签发布,产物附带校验和、SPDX SBOM、Sigstore 签名清单与 GitHub 构建证明。每个版本都会在门户发布[版本说明](https://silo.pgsty.com/zh/tags/silo/);安全问题按照 [`SECURITY.md`](SECURITY.md) 与仓库[安全公告](docs/security/advisories.md)处理。上游版权、许可证与第三方声明完整保留于 [`LICENSE`](LICENSE)、[`NOTICE`](NOTICE) 与 [`CREDITS`](CREDITS)。
|
||||
|
||||
## 兼容策略
|
||||
|
||||
Silo 尽量保留:
|
||||
|
||||
- `github.com/minio/minio` module path 与 `github.com/minio/*` 导入路径;
|
||||
- MinIO 兼容的 S3 API、线协议、`MINIO_*` 环境变量、指标、协议头、保留路由与存储元数据;
|
||||
- `RELEASE.YYYY-MM-DDTHH-MM-SSZ` 标签,以及容器入口对旧式 `minio …` 参数的转换。
|
||||
|
||||
Silo 自有交付面统一使用 `silo` 可执行文件、软件包、服务、Helm Chart 与 `pgsty/silo` 容器镜像;原生交付物不会安装 `minio` 二进制别名。
|
||||
|
||||
兼容性是默认约束。只要不会留下安全问题,Silo 就保留既有的协议、客户端、配置与运维行为;只有在修复重大安全问题确有必要时才会打破兼容,并在版本说明中明确受影响行为与迁移方式。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/)、[兼容性说明](https://silo.pgsty.com/zh/compatibility/server/)与[安全公告](docs/security/advisories.md),保留回滚路径,并在生产使用前完成测试。
|
||||
|
||||
## 下载与发行产物
|
||||
|
||||
请先在[下载与安装](https://silo.pgsty.com/zh/download/)页面选择合适的安装方式;GitHub Releases 仍是带版本服务端二进制、校验和与源码归档的获取位置。
|
||||
|
||||
| 产物 | 位置 |
|
||||
| :-- | :-- |
|
||||
| 源码 | [`github.com/pgsty/silo`](https://github.com/pgsty/silo) |
|
||||
| 容器镜像 | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo),支持 `linux/amd64` 与 `linux/arm64` 多架构清单 |
|
||||
| 服务端二进制与校验和 | [GitHub Releases](https://github.com/pgsty/silo/releases),覆盖 Linux、macOS、Windows 的 `amd64` 与 `arm64` |
|
||||
| Linux 软件包 | RPM、DEB、APK,并通过 [Pigsty 软件仓库](https://pigsty.cc/docs/repo/) 分发 |
|
||||
| 客户端 | [`pgsty/mc`](https://github.com/pgsty/mc),容器内以 `mcli` 提供,并保留 `mc` 兼容别名 |
|
||||
| 管理控制台 | [`pgsty/silo-console`](https://github.com/pgsty/silo-console),通过兼容导入路径 `github.com/minio/console` 嵌入服务端构建 |
|
||||
| 共享库 | [`pgsty/silo-pkg`](https://github.com/pgsty/silo-pkg) v3.11.0,通过 `replace` 指令使用,同时保留 `github.com/minio/pkg/v3` 导入路径 |
|
||||
|
||||
每个新版本都会为各平台归档和软件包发布 SPDX JSON SBOM。归档与软件包的校验和清单分别带有无长期密钥的 Sigstore 签名包;GitHub 制品证明则为全部可下载产物及多架构容器镜像记录已签名的构建来源。
|
||||
|
||||
下载归档及配套文件后,请分别验证完整性、已发布 SBOM、签名清单与构建来源:
|
||||
## 快速上手
|
||||
|
||||
```bash
|
||||
# 完整性:选择与你下载产物相符的一行。
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# 归档 SBOM 是另一个独立校验的 Release 产物。
|
||||
grep -F ' silo_<version>_linux_amd64.tar.gz.sbom.json' \
|
||||
silo_<version>_checksums.txt | sha256sum --check
|
||||
|
||||
# 验证归档/SBOM 校验和清单的签名。
|
||||
cosign verify-blob \
|
||||
--bundle silo_<version>_checksums.txt.sigstore.json \
|
||||
--certificate-identity-regexp \
|
||||
'^https://github.com/pgsty/(minio|silo)/\.github/workflows/release\.yml@refs/(tags/RELEASE\..+|heads/(master|main))$' \
|
||||
--certificate-oidc-issuer https://token.actions.githubusercontent.com \
|
||||
silo_<version>_checksums.txt
|
||||
|
||||
# 在线验证本仓库签发的构建来源。
|
||||
gh attestation verify silo_<version>_linux_amd64.tar.gz \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
软件包使用 `silo_<version>_packages_checksums.txt` 及其相邻的 Sigstore
|
||||
签名包,并使用相同的 identity 与 issuer 约束;请用 SPDX 工具检查已验证的
|
||||
SPDX JSON SBOM。按 digest 验证多架构容器清单的构建来源:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<manifest-digest> \
|
||||
--repo pgsty/silo
|
||||
```
|
||||
|
||||
分架构 SBOM 证明附在 `amd64` 与 `arm64` 平台镜像的 digest 上,而非多架构
|
||||
清单上,需要分别显式验证:
|
||||
|
||||
```bash
|
||||
gh attestation verify \
|
||||
oci://index.docker.io/pgsty/silo@sha256:<platform-digest> \
|
||||
--repo pgsty/silo \
|
||||
--predicate-type https://spdx.dev/Document/v2.3
|
||||
```
|
||||
|
||||
按 digest 验证可避免信任可变镜像标签。
|
||||
|
||||
### 原生软件包迁移
|
||||
|
||||
`silo` RPM、DEB 与 APK 不针对 `minio` 声明 `Provides`、`Obsoletes`、
|
||||
`Replaces` 或包级 `Conflicts`,因此可以与已有 MinIO 软件包并存安装,
|
||||
不会被普通升级静默替换。两个 systemd unit 在运行时互斥,应由管理员显式
|
||||
切换,不能同时启动。
|
||||
|
||||
切换前请记录旧 unit 的 enabled/active 状态与 `User`/`Group`,并备份
|
||||
`/etc/default/minio`。Silo 先读取该旧配置,再读取 `/etc/default/silo`;后者
|
||||
中由管理员设置的同名变量优先。如现有数据必须继续使用原 UID/GID,请创建
|
||||
`/etc/systemd/system/silo.service.d/10-legacy-user.conf`:
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
User=<legacy-user>
|
||||
Group=<legacy-group>
|
||||
```
|
||||
|
||||
执行 `systemctl daemon-reload`,先停用并停止 `minio.service`,再启用并启动
|
||||
`silo.service`。确认健康检查、S3、Admin API、指标与日志后,才 mask 或卸载
|
||||
旧服务。软件包迁移期间不要递归修改数据属主;在回滚观察窗口内保留旧包与
|
||||
旧 unit。
|
||||
|
||||
## 快速开始
|
||||
|
||||
本地体验:
|
||||
|
||||
```bash
|
||||
mkdir -p data
|
||||
|
||||
export MINIO_ROOT_USER=minioadmin
|
||||
export MINIO_ROOT_PASSWORD=change-me-long-password
|
||||
|
||||
docker run -d --name silo \
|
||||
-p 9000:9000 \
|
||||
-p 9001:9001 \
|
||||
-e MINIO_ROOT_USER \
|
||||
-e MINIO_ROOT_PASSWORD \
|
||||
docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:latest server /data --console-address ":9001"
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
管理控制台位于 <http://localhost:9001>,S3 API 位于 <http://localhost:9000>。
|
||||
<p align="center">
|
||||
<img src="https://silo.pgsty.com/images/silo-console/console-login.webp" alt="Silo 控制台">
|
||||
</p>
|
||||
|
||||
镜像内置兼容客户端 `mcli`:
|
||||
控制台位于 <http://localhost:9001>,S3 API 位于 <http://localhost:9000>。镜像内置客户端 `mcli`:
|
||||
|
||||
```bash
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 \
|
||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||
docker exec silo mcli mb local/demo
|
||||
docker exec silo mcli ls local
|
||||
docker exec silo mcli alias set local http://127.0.0.1:9000 minioadmin change-me-long-password
|
||||
docker exec silo mcli mb local/demo && docker exec silo mcli ls local
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 生产环境应锁定版本,使用独立凭据与 TLS,配置监控,保留独立备份,并验证恢复流程。
|
||||
> 生产环境应锁定版本,使用独立凭据与 TLS,配置监控,保留独立备份,并验证恢复流程。请从[文档](https://silo.pgsty.com/zh/docs/)开始。
|
||||
|
||||
从源码构建服务端:
|
||||
## 安装
|
||||
|
||||
```bash
|
||||
go build -o silo .
|
||||
./silo --version
|
||||
```
|
||||
| 方式 | 位置 |
|
||||
| :-- | :-- |
|
||||
| 容器镜像 | [`pgsty/silo`](https://hub.docker.com/r/pgsty/silo),支持 `linux/amd64` 与 `linux/arm64` |
|
||||
| 二进制 | [GitHub Releases](https://github.com/pgsty/silo/releases),覆盖 Linux、macOS、Windows 的 `amd64` 与 `arm64` |
|
||||
| 软件包 | RPM、DEB、APK,也可通过 [Pigsty 软件仓库](https://pigsty.cc/docs/repo/) 安装 |
|
||||
| Kubernetes | Helm Chart,参见[下载与安装](https://silo.pgsty.com/zh/download/) |
|
||||
| 源码构建 | `go build -o silo . && ./silo --version` |
|
||||
|
||||
其他安装方式(包括原生软件包、二进制、Podman、Kubernetes、源码构建与 Pigsty Ansible)请前往[下载与安装](https://silo.pgsty.com/zh/download/);生产部署与管理请从 [Silo 文档](https://silo.pgsty.com/zh/docs/)开始。Pigsty 用户也可以直接使用 [Pigsty MinIO 模块](https://pigsty.cc/docs/minio/)。
|
||||
每个版本都附带校验和、SPDX SBOM、Sigstore 签名清单与 GitHub 构建证明。完整安装方式与验证命令见[下载与安装](https://silo.pgsty.com/zh/download/);从上游 MinIO 迁移 —— 接管既有 `minio.service` 与 `/etc/default/minio`,并用 `/etc/systemd/system/silo.service.d/10-legacy-user.conf` drop-in 保持数据属主不变 —— 见[迁移指南](https://silo.pgsty.com/zh/compatibility/migration/)与[二进制与服务说明](https://silo.pgsty.com/zh/compatibility/binary/)。
|
||||
|
||||
## 安全
|
||||
## 兼容性
|
||||
|
||||
安全修复面向当前活跃开发分支,并记录在仓库[安全公告](docs/security/advisories.md)与门户[安全通告](https://silo.pgsty.com/zh/blog/security/)中。请按照 [`SECURITY.md`](SECURITY.md) 与 [`VULNERABILITY_REPORT.md`](VULNERABILITY_REPORT.md) 私密报告漏洞;同时影响上游 MinIO 的问题也应向上游报告。
|
||||
Silo 保留 S3 与存储格式兼容性,包括既有 `MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/*` 路由与 `.minio.sys` 数据。CI 守卫检查选定的兼容性标识,有意的安全与行为变化在发布说明中记录。Silo 自有交付面使用 `silo` 可执行文件、软件包、服务、Helm Chart 与容器镜像;原生交付物不会安装 `minio` 服务端二进制别名。
|
||||
|
||||
## 贡献
|
||||
正式支持和发布验收的组合为 `pgsty/silo` + `pgsty/silo-console` + `pgsty/mc` + `pgsty/silo-pkg`,对未修改的上游 MinIO/MC 尽最大努力保持兼容。Server、Console 与客户端按需保留历史模块路径,维护源码直接导入 `github.com/pgsty/silo-pkg/v3`;SDK `github.com/minio/minio-go/v7` 是明确保留的上游依赖。具体版本与 replace 以当前 [go.mod](go.mod) 为准。
|
||||
|
||||
欢迎安全与依赖项更新、可复现缺陷修复、测试、发布自动化、打包与文档改进。
|
||||
与上游的全部分歧,以逐项核验代码的[兼容性审计](https://silo.pgsty.com/zh/compatibility/server/)形式维护。每个版本仍应视为下游升级:锁定版本,阅读[版本说明](https://silo.pgsty.com/zh/tags/silo/),并保留回滚路径。
|
||||
|
||||
Issue 与 Pull Request 应说明受影响版本、复现步骤、影响、预期行为、测试与兼容性说明。大型改动请先提交 Issue 讨论。
|
||||
### TLS 与 Go 升级
|
||||
|
||||
本项目不要求签署 CLA:贡献按项目许可证(AGPL-3.0-or-later,inbound=outbound)接收,贡献者保留自己的版权。每个提交都必须按照 [DCO](https://developercertificate.org/) 签署(`git commit -s`),详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||
Server 20260916 已恢复 Go 默认密钥交换策略,20261018 在 Go 1.27.2 下保留该行为。
|
||||
`GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化,
|
||||
以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
|
||||
|
||||
## 文档归属
|
||||
|
||||
用户文档统一维护在 [silo.pgsty.com](https://silo.pgsty.com/zh/docs/),源码位于
|
||||
[pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com)。本仓库保留的 `docs/`
|
||||
主要是继承的参考材料、示例和工具测试夹具。调查日志、AI 工作记录与临时报告放在仓库外;
|
||||
可复用的结论应整理进伴生文档站。仓库职责与维护规则见 [AGENTS.md](AGENTS.md)。
|
||||
|
||||
## 安全与贡献
|
||||
|
||||
请按照 [`SECURITY.md`](SECURITY.md) 私密报告漏洞;每项修复都会发布公开[安全公告](https://silo.pgsty.com/zh/blog/security/)。本项目不要求签署 CLA:贡献按 AGPL-3.0-or-later(inbound=outbound)接收,只需 DCO 签署(`git commit -s`),详见 [`CONTRIBUTING.md`](CONTRIBUTING.md)。
|
||||
|
||||
## 贡献者
|
||||
|
||||
<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
|
||||
|
||||
每位 issue 或 PR 的作者,以及同意公开署名的安全报告者,都是 SILO 社区的一员,包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示,其余参考首次参与时间;金色圆环突出经过审核的显著贡献。
|
||||
|
||||
<a href="CONTRIBUTORS.md">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/contributors-light.svg" alt="SILO 社区贡献者">
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
[查看贡献记录与实际 PR 状态](CONTRIBUTORS.md)。
|
||||
|
||||
## Star History
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-dark.svg">
|
||||
<img src="https://raw.githubusercontent.com/pgsty/silo/codex/repository-cards/star-history-light.svg" alt="SILO GitHub 星标历史">
|
||||
</picture>
|
||||
|
||||
## 背景
|
||||
|
||||
本项目源于上游社区发行与维护模式的变化。维护者对相关变化的分析、替代方案评估与早期维护记录见以下文章:
|
||||
本项目因上游收缩社区版而生:Web 控制台被削减为残桩、社区预编译制品停发、社区仓库被归档。Silo 的存在就是让这些部署继续跑下去。Fork 是手段,不是身份 —— 若上游恢复社区版承诺,我们乐意收缩范围,并把修复回馈上游。
|
||||
|
||||
| 文章 | 主题 |
|
||||
| :-- | :-- |
|
||||
| [MinIO已死](https://silo.pgsty.com/zh/blog/post/minio-is-dead/) | 上游项目与发行模式的变化 |
|
||||
| [MinIO已死,谁能接盘?](https://silo.pgsty.com/zh/blog/post/minio-alternative/) | 可选替代方案评估 |
|
||||
| [MinIO 已死,MinIO 复生](https://silo.pgsty.com/zh/blog/post/minio-resurrect/) | 建立分支及其发行流水线 |
|
||||
| [续命 MinIO:承诺兑现](https://silo.pgsty.com/zh/blog/post/minio-promise-kept/) | 初期安全与维护工作 |
|
||||
[**宣言**](https://silo.pgsty.com/zh/about/manifesto/)是项目的公开承诺,共十一条,通篇遵循一项纪律:**每一条,要么是已经在做且有公开证据的事实,要么是刻意拒绝的承诺。** 摘要:
|
||||
|
||||
- **兼容性合同** —— 协议与数据不改,每个版本都标注经过测试的回滚目标与路径。
|
||||
- **许可证无法变更** —— AGPLv3、无 CLA、不做版权聚合;包括我们自己在内,没有人握有足够版权代表所有贡献者重新授权。
|
||||
- **永不清单**(只增不减)—— 永不将既有功能移入付费墙、永不给下载设注册墙、永不加入遥测(上游回连路径已整体移除)、永不引入 CLA、永不变更许可证、永不以商标追究正常使用。
|
||||
- **安全与发布纪律** —— 每项安全修复配一篇公开公告;通常每一到两个月发布一版,最长不超过一个季度。请拿公开记录检验这两条。
|
||||
|
||||
延伸阅读:[MinIO已死](https://silo.pgsty.com/zh/blog/post/minio-is-dead/) · [谁能接盘?](https://silo.pgsty.com/zh/blog/post/minio-alternative/) · [MinIO 复生](https://silo.pgsty.com/zh/blog/post/minio-resurrect/) · [承诺兑现](https://silo.pgsty.com/zh/blog/post/minio-promise-kept/)
|
||||
|
||||
## 许可证与商标
|
||||
|
||||
服务端源码依据 [GNU Affero 通用公共许可证第 3 版或任何后续版本](LICENSE)分发。本分支衍生自 [`minio/minio`](https://github.com/minio/minio):[`NOTICE`](NOTICE) 保留上游产品声明,[`CREDITS`](CREDITS) 记录所含第三方组件的许可证与声明,Git 历史记录下游修改。
|
||||
Silo 采用 [AGPL-3.0-or-later](LICENSE),衍生自 [`minio/minio`](https://github.com/minio/minio),上游版权与第三方声明完整保留于 [`NOTICE`](NOTICE) 与 [`CREDITS`](CREDITS)。MinIO 是 MinIO, Inc. 的商标,此处使用仅为标识上游项目与兼容谱系。
|
||||
|
||||
MinIO 是 MinIO, Inc. 的商标;本文仅为标识上游项目与兼容谱系而使用该名称。Pigsty、Silo 与 `pgsty/silo` 均为独立社区项目,与 MinIO, Inc. 不存在隶属、背书或赞助关系。
|
||||
|
||||
门户另行发布项目的[许可证摘要](https://silo.pgsty.com/zh/about/license/)、[文档署名归属](https://silo.pgsty.com/zh/about/attribution/)与[商标声明](https://silo.pgsty.com/zh/about/trademark/)。
|
||||
详见:[许可证](https://silo.pgsty.com/zh/about/license/) · [署名归属](https://silo.pgsty.com/zh/about/attribution/) · [商标声明](https://silo.pgsty.com/zh/about/trademark/)
|
||||
|
||||
+33
-5
@@ -7,20 +7,48 @@ Silo-specific fixes or release notes.
|
||||
## Supported Versions
|
||||
|
||||
Security fixes are tracked on the active development branch and summarized in
|
||||
[docs/security/advisories.md](docs/security/advisories.md). Only the current
|
||||
[the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). Only the current
|
||||
Silo release line is supported unless an advisory says otherwise.
|
||||
|
||||
## Inherited Fix Evidence
|
||||
|
||||
The canonical ledger also records security fixes inherited from upstream when
|
||||
they are part of the Silo release baseline. Source and fork commits are linked
|
||||
separately even when the fork preserves the original commit object and SHA.
|
||||
|
||||
- [CVE-2025-62506](https://github.com/advisories/GHSA-jjjj-jwhf-8rgr):
|
||||
upstream [PR #21642](https://github.com/minio/minio/pull/21642) merged as
|
||||
[`minio/minio@c1a49490`](https://github.com/minio/minio/commit/c1a49490c78e9c3ebcad86ba0662319138ace190),
|
||||
inherited unchanged as
|
||||
[`pgsty/silo@c1a49490`](https://github.com/pgsty/silo/commit/c1a49490c78e9c3ebcad86ba0662319138ace190),
|
||||
and is present in every Silo community release beginning with
|
||||
[`RELEASE.2025-12-03T12-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2025-12-03T12-00-00Z).
|
||||
The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212)
|
||||
and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46)
|
||||
regression groups remain part of `go test ./cmd`; see the
|
||||
[canonical ledger](https://silo.pgsty.com/about/security-advisories/#inherited)
|
||||
for the operator-facing record.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
The security contact for Silo is [ron@pgsty.com](mailto:ron@pgsty.com).
|
||||
This address also receives automated security reports for this repository.
|
||||
|
||||
For vulnerabilities in this fork:
|
||||
|
||||
1. Follow the fork-specific expectations in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
2. Prefer this repository's [private GitHub security advisory](https://github.com/pgsty/silo/security/advisories/new) workflow.
|
||||
3. If private reporting is unavailable, contact the maintainers through the
|
||||
repository without publishing exploit details until a private channel is
|
||||
established.
|
||||
3. If you cannot use GitHub private reporting, email [ron@pgsty.com](mailto:ron@pgsty.com)
|
||||
directly. Do not publish vulnerability details or exploit code in public
|
||||
issues or pull requests.
|
||||
4. If you confirm the issue also affects upstream `minio/minio`, report it upstream separately.
|
||||
|
||||
## Scanner Configuration
|
||||
|
||||
The [OSS Scanner build environment](.oss-scanner/Dockerfile) and
|
||||
[scan scope and offline checks](.oss-scanner/threat_model.md) are maintained
|
||||
with the source. Update them when dependencies or security boundaries change.
|
||||
|
||||
## Disclosure Process
|
||||
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [docs/security/advisories.md](docs/security/advisories.md). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
Fork-specific fixes and user-visible upgrade notes are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/). The fork-specific triage and remediation process is described in [VULNERABILITY_REPORT.md](VULNERABILITY_REPORT.md).
|
||||
|
||||
@@ -34,4 +34,4 @@ Based on the report, the Silo maintainers investigate:
|
||||
If the vulnerability exists in this fork itself, the maintainers will, when
|
||||
feasible, fix the issue or implement reasonable countermeasures such that the
|
||||
vulnerability can no longer be exploited. Fork-specific upgrade notes and
|
||||
security advisories are published in `docs/security/advisories.md`.
|
||||
security advisories are published in [the security advisory ledger](https://silo.pgsty.com/about/security-advisories/).
|
||||
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Fail closed before a release job can replace published or finalized assets.
|
||||
# An ordinary Draft is retry state; a finalized Draft contains GPG-derived
|
||||
# materials and must never be replaced by the build lane.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
release_tag="${1:-}"
|
||||
fixture="${2:-}"
|
||||
repository="${GITHUB_REPOSITORY:-pgsty/silo}"
|
||||
require_draft="${REQUIRE_DRAFT:-false}"
|
||||
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "jq is required to inspect GitHub release state" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${release_tag}" =~ ^RELEASE\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}Z$ ]]; then
|
||||
echo "Invalid release tag format: ${release_tag:-<empty>}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "${fixture}" ]; then
|
||||
release_json="$(<"${fixture}")"
|
||||
else
|
||||
error_file="$(mktemp)"
|
||||
trap 'rm -f "${error_file}"' EXIT
|
||||
if ! release_json="$(
|
||||
gh api --paginate "repos/${repository}/releases?per_page=100" --jq '.[]' 2>"${error_file}" |
|
||||
jq --arg tag "${release_tag}" -s '[.[] | select(.tag_name == $tag)]'
|
||||
)"; then
|
||||
cat "${error_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! jq -e 'type == "array" and all(.[]; type == "object" and (.tag_name | type == "string") and (.draft | type == "boolean"))' \
|
||||
<<<"${release_json}" >/dev/null 2>&1; then
|
||||
echo "Invalid release state response for ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! jq -e --arg tag "${release_tag}" 'all(.[]; .tag_name == $tag)' \
|
||||
<<<"${release_json}" >/dev/null 2>&1; then
|
||||
echo "Release state returned a tag other than ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
release_count="$(jq 'length' <<<"${release_json}")"
|
||||
if [ "${release_count}" -eq 0 ]; then
|
||||
if [ "${require_draft}" = "true" ]; then
|
||||
echo "Expected one Draft release for ${release_tag}, found none" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "No existing release for ${release_tag}."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${release_count}" -ne 1 ]; then
|
||||
echo "Refusing to choose among ${release_count} releases for ${release_tag}; clean duplicate Drafts first" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$(jq -r '.[0].draft' <<<"${release_json}")" != "true" ]; then
|
||||
echo "Refusing to overwrite published release ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
finalize_markers="$(jq '[.[0].assets[]? | select(.name | endswith("_packages_provenance.sigstore.json"))] | length' <<<"${release_json}")"
|
||||
if [ "${finalize_markers}" -ne 0 ]; then
|
||||
echo "Refusing to replace finalized Draft ${release_tag}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Existing unfinalized Draft ${release_tag} will be replaced from scratch."
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
checker="${script_dir}/check-release-state.sh"
|
||||
tag="RELEASE.2026-08-29T00-00-00Z"
|
||||
fixture="$(mktemp)"
|
||||
stdout_file="$(mktemp)"
|
||||
stderr_file="$(mktemp)"
|
||||
trap 'rm -f "${fixture}" "${stdout_file}" "${stderr_file}"' EXIT
|
||||
|
||||
expect_success() {
|
||||
if ! "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
cat "${stderr_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
if "${checker}" "$@" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
echo "Expected release-state check to fail: $*" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
printf '[]\n' >"${fixture}"
|
||||
expect_success "${tag}" "${fixture}"
|
||||
grep -qF "No existing release for ${tag}." "${stdout_file}"
|
||||
|
||||
if REQUIRE_DRAFT=true "${checker}" "${tag}" "${fixture}" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
echo "Expected required-Draft check to fail when no release exists" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -qF "Expected one Draft release for ${tag}, found none" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true,"assets":[]}]\n' "${tag}" >"${fixture}"
|
||||
expect_success "${tag}" "${fixture}"
|
||||
grep -qF "Existing unfinalized Draft ${tag} will be replaced from scratch." "${stdout_file}"
|
||||
if ! REQUIRE_DRAFT=true "${checker}" "${tag}" "${fixture}" >"${stdout_file}" 2>"${stderr_file}"; then
|
||||
cat "${stderr_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true,"assets":[{"name":"silo_20260829000000.0.0_packages_provenance.sigstore.json"}]}]\n' "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to replace finalized Draft ${tag}" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":false}]\n' "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to overwrite published release ${tag}" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"%s","draft":true},{"tag_name":"%s","draft":true}]\n' "${tag}" "${tag}" >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Refusing to choose among 2 releases" "${stderr_file}"
|
||||
|
||||
printf '[{"tag_name":"RELEASE.2026-08-28T00-00-00Z","draft":true}]\n' >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "other than ${tag}" "${stderr_file}"
|
||||
|
||||
printf '{not-json}\n' >"${fixture}"
|
||||
expect_failure "${tag}" "${fixture}"
|
||||
grep -qF "Invalid release state response for ${tag}" "${stderr_file}"
|
||||
|
||||
expect_failure "not-a-release-tag" "${fixture}"
|
||||
grep -qF "Invalid release tag format" "${stderr_file}"
|
||||
|
||||
echo "release-state decision tests passed"
|
||||
@@ -7,7 +7,7 @@ _init() {
|
||||
|
||||
## Minimum required versions for build dependencies
|
||||
GIT_VERSION="1.0"
|
||||
GO_VERSION="1.16"
|
||||
GO_VERSION="1.27.3"
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
|
||||
@@ -22,8 +22,8 @@ function start_silo_4drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-08-06T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-11-09T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
@@ -74,7 +74,7 @@ expected=$(awk -v asset="${archive}" '
|
||||
{
|
||||
name=$2
|
||||
sub(/^\*/, "", name)
|
||||
if (name == asset && $1 ~ /^[0-9a-fA-F]{64}$/) print tolower($1)
|
||||
if (name == asset && length($1) == 64 && $1 ~ /^[0-9a-fA-F]+$/) print tolower($1)
|
||||
}
|
||||
' "${tmp_dir}/${checksums}")
|
||||
if ! printf '%s\n' "${expected}" | grep -Eq '^[0-9a-f]{64}$'; then
|
||||
|
||||
@@ -45,8 +45,8 @@ function start_silo_10drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -71,7 +71,7 @@ function start_silo_10drive() {
|
||||
"${PWD}/mc" mb --with-versioning silo/bucket
|
||||
|
||||
export AWS_ACCESS_KEY_ID=silo
|
||||
export AWS_SECRET_ACCESS_KEY=silo123
|
||||
export AWS_SECRET_ACCESS_KEY=silo1234
|
||||
aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json
|
||||
uploadId=$(jq -r '.UploadId' upload-id.json)
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const manifestVersion = 3
|
||||
const manifestVersion = 4
|
||||
|
||||
var (
|
||||
minioImportRE = regexp.MustCompile(`github\.com/minio/[A-Za-z0-9_./-]+`)
|
||||
@@ -44,19 +44,18 @@ var (
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Version int `json:"version"`
|
||||
ModulePath string `json:"module_path"`
|
||||
MinioImports []string `json:"minio_imports"`
|
||||
Environment []string `json:"environment"`
|
||||
Metrics []string `json:"metrics"`
|
||||
Headers []string `json:"headers"`
|
||||
Routes []string `json:"routes"`
|
||||
RouteRoots []string `json:"route_roots"`
|
||||
GridRoutes []string `json:"grid_routes"`
|
||||
StorageMarkers []string `json:"storage_markers"`
|
||||
PolicyValues []string `json:"policy_values"`
|
||||
ExportedSymbols []string `json:"exported_symbols"`
|
||||
BrandAllowlist []string `json:"brand_allowlist"`
|
||||
Version int `json:"version"`
|
||||
ModulePath string `json:"module_path"`
|
||||
MinioImports []string `json:"minio_imports"`
|
||||
Environment []string `json:"environment"`
|
||||
Metrics []string `json:"metrics"`
|
||||
Headers []string `json:"headers"`
|
||||
Routes []string `json:"routes"`
|
||||
RouteRoots []string `json:"route_roots"`
|
||||
GridRoutes []string `json:"grid_routes"`
|
||||
StorageMarkers []string `json:"storage_markers"`
|
||||
PolicyValues []string `json:"policy_values"`
|
||||
BrandAllowlist []string `json:"brand_allowlist"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -101,22 +100,22 @@ func collect(repo string) (manifest, error) {
|
||||
}
|
||||
|
||||
sets := map[string]map[string]struct{}{
|
||||
"imports": {},
|
||||
"env": {},
|
||||
"metrics": {},
|
||||
"headers": {},
|
||||
"routes": {},
|
||||
"roots": {},
|
||||
"grid": {},
|
||||
"storage": {},
|
||||
"policy": {},
|
||||
"exported": {},
|
||||
"brand": {},
|
||||
"imports": {},
|
||||
"env": {},
|
||||
"metrics": {},
|
||||
"headers": {},
|
||||
"routes": {},
|
||||
"roots": {},
|
||||
"grid": {},
|
||||
"storage": {},
|
||||
"policy": {},
|
||||
"brand": {},
|
||||
}
|
||||
modulePath := ""
|
||||
fset := token.NewFileSet()
|
||||
|
||||
for _, rel := range files {
|
||||
// Migration notes and guard fixtures contain archived identifiers.
|
||||
if rel == "SILO_REBRANDING_MIGRATION.md" ||
|
||||
strings.HasPrefix(rel, "buildscripts/rebrand-guard/") ||
|
||||
strings.HasPrefix(rel, "buildscripts/helm-migration-guard/") {
|
||||
@@ -163,17 +162,17 @@ func collect(repo string) (manifest, error) {
|
||||
sets["imports"][value] = struct{}{}
|
||||
}
|
||||
}
|
||||
collectStringMatches(sets["routes"], routeRE, file)
|
||||
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
||||
if rel == "internal/grid/manager.go" {
|
||||
collectStringMatches(sets["grid"], routeRE, file)
|
||||
}
|
||||
if !strings.HasSuffix(rel, "_test.go") {
|
||||
collectExported(sets["exported"], filepath.ToSlash(filepath.Dir(rel)), file)
|
||||
// Test files hold request paths for fixtures, not served routes.
|
||||
collectStringMatches(sets["routes"], routeRE, file)
|
||||
if strings.HasPrefix(rel, "cmd/") || strings.HasPrefix(rel, "internal/") {
|
||||
collectBrandStrings(sets["brand"], rel, file)
|
||||
}
|
||||
}
|
||||
collectNamedStringValues(sets["roots"], rel, file, "minioReservedBucket")
|
||||
if rel == "internal/grid/manager.go" {
|
||||
collectStringMatches(sets["grid"], routeRE, file)
|
||||
}
|
||||
}
|
||||
}
|
||||
// This was a shell-local PID variable in the generated inspect script,
|
||||
@@ -184,19 +183,18 @@ func collect(repo string) (manifest, error) {
|
||||
return manifest{}, errors.New("go.mod module path was not found")
|
||||
}
|
||||
return manifest{
|
||||
Version: manifestVersion,
|
||||
ModulePath: modulePath,
|
||||
MinioImports: sorted(sets["imports"]),
|
||||
Environment: sorted(sets["env"]),
|
||||
Metrics: sorted(sets["metrics"]),
|
||||
Headers: sorted(sets["headers"]),
|
||||
Routes: sorted(sets["routes"]),
|
||||
RouteRoots: sorted(sets["roots"]),
|
||||
GridRoutes: sorted(sets["grid"]),
|
||||
StorageMarkers: sorted(sets["storage"]),
|
||||
PolicyValues: sorted(sets["policy"]),
|
||||
ExportedSymbols: sorted(sets["exported"]),
|
||||
BrandAllowlist: sorted(sets["brand"]),
|
||||
Version: manifestVersion,
|
||||
ModulePath: modulePath,
|
||||
MinioImports: sorted(sets["imports"]),
|
||||
Environment: sorted(sets["env"]),
|
||||
Metrics: sorted(sets["metrics"]),
|
||||
Headers: sorted(sets["headers"]),
|
||||
Routes: sorted(sets["routes"]),
|
||||
RouteRoots: sorted(sets["roots"]),
|
||||
GridRoutes: sorted(sets["grid"]),
|
||||
StorageMarkers: sorted(sets["storage"]),
|
||||
PolicyValues: sorted(sets["policy"]),
|
||||
BrandAllowlist: sorted(sets["brand"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -262,7 +260,7 @@ func collectStringMatches(dst map[string]struct{}, re *regexp.Regexp, file *ast.
|
||||
}
|
||||
|
||||
func trackedFiles(repo string) ([]string, error) {
|
||||
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "--others", "--exclude-standard", "-z")
|
||||
cmd := exec.Command("git", "-C", repo, "ls-files", "--cached", "-z")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git ls-files: %w", err)
|
||||
@@ -286,78 +284,6 @@ func addMatches(dst map[string]struct{}, re *regexp.Regexp, text string, lower b
|
||||
}
|
||||
}
|
||||
|
||||
func collectExported(dst map[string]struct{}, dir string, file *ast.File) {
|
||||
prefix := dir + ":" + file.Name.Name + ":"
|
||||
for _, decl := range file.Decls {
|
||||
switch decl := decl.(type) {
|
||||
case *ast.FuncDecl:
|
||||
if !ast.IsExported(decl.Name.Name) {
|
||||
continue
|
||||
}
|
||||
if decl.Recv == nil {
|
||||
dst[prefix+"func:"+decl.Name.Name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
receiver := receiverName(decl.Recv.List[0].Type)
|
||||
dst[prefix+"method:"+receiver+"."+decl.Name.Name] = struct{}{}
|
||||
case *ast.GenDecl:
|
||||
for _, spec := range decl.Specs {
|
||||
switch spec := spec.(type) {
|
||||
case *ast.TypeSpec:
|
||||
if !ast.IsExported(spec.Name.Name) {
|
||||
continue
|
||||
}
|
||||
dst[prefix+"type:"+spec.Name.Name] = struct{}{}
|
||||
collectExportedFields(dst, prefix, spec.Name.Name, spec.Type)
|
||||
case *ast.ValueSpec:
|
||||
kind := strings.ToLower(decl.Tok.String())
|
||||
for _, name := range spec.Names {
|
||||
if ast.IsExported(name.Name) {
|
||||
dst[prefix+kind+":"+name.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collectExportedFields(dst map[string]struct{}, prefix, typeName string, expr ast.Expr) {
|
||||
var fields *ast.FieldList
|
||||
switch typed := expr.(type) {
|
||||
case *ast.StructType:
|
||||
fields = typed.Fields
|
||||
case *ast.InterfaceType:
|
||||
fields = typed.Methods
|
||||
default:
|
||||
return
|
||||
}
|
||||
for _, field := range fields.List {
|
||||
for _, name := range field.Names {
|
||||
if ast.IsExported(name.Name) {
|
||||
dst[prefix+"field:"+typeName+"."+name.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func receiverName(expr ast.Expr) string {
|
||||
switch expr := expr.(type) {
|
||||
case *ast.Ident:
|
||||
return expr.Name
|
||||
case *ast.StarExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.IndexExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.IndexListExpr:
|
||||
return receiverName(expr.X)
|
||||
case *ast.SelectorExpr:
|
||||
return receiverName(expr.X) + "." + expr.Sel.Name
|
||||
default:
|
||||
return fmt.Sprintf("%T", expr)
|
||||
}
|
||||
}
|
||||
|
||||
func sorted(set map[string]struct{}) []string {
|
||||
values := make([]string, 0, len(set))
|
||||
for value := range set {
|
||||
@@ -409,7 +335,6 @@ func compare(want, got manifest) error {
|
||||
{"grid_routes", want.GridRoutes, got.GridRoutes},
|
||||
{"storage_markers", want.StorageMarkers, got.StorageMarkers},
|
||||
{"policy_values", want.PolicyValues, got.PolicyValues},
|
||||
{"exported_symbols", want.ExportedSymbols, got.ExportedSymbols},
|
||||
{"brand_allowlist", want.BrandAllowlist, got.BrandAllowlist},
|
||||
}
|
||||
for _, check := range checks {
|
||||
@@ -454,10 +379,10 @@ func setDiff(want, got []string) (missing, added []string) {
|
||||
}
|
||||
|
||||
func printSummary(value manifest) {
|
||||
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d exported=%d brand=%d sha256=%s\n",
|
||||
fmt.Printf("compatibility manifest: imports=%d env=%d metrics=%d headers=%d routes=%d roots=%d grid=%d storage=%d policy=%d brand=%d sha256=%s\n",
|
||||
len(value.MinioImports), len(value.Environment), len(value.Metrics), len(value.Headers),
|
||||
len(value.Routes), len(value.RouteRoots), len(value.GridRoutes), len(value.StorageMarkers), len(value.PolicyValues),
|
||||
len(value.ExportedSymbols), len(value.BrandAllowlist), manifestDigest(value))
|
||||
len(value.BrandAllowlist), manifestDigest(value))
|
||||
}
|
||||
|
||||
func manifestDigest(value manifest) string {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
__pycache__/
|
||||
@@ -0,0 +1,163 @@
|
||||
# Copyright (c) 2026 Feng Ruohang
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Pure, self-contained SVG rendering for SILO's README cards."""
|
||||
from datetime import date, timedelta
|
||||
from html import escape
|
||||
import math
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
themes = {
|
||||
'light': dict(bg='#ffffff', wash='#f2f7fc', edge='#d9e3ee', ink='#16222e',
|
||||
muted='#62758a', blue='#1d588c', copper='#b4762e', grid='#e5edf5',
|
||||
line='#2b6ca3', ring='#dce5ef', field='#f7f9fc', label='#3d4e61'),
|
||||
'dark': dict(bg='#101923', wash='#152738', edge='#2b3c50', ink='#e8eef6',
|
||||
muted='#93a3b8', blue='#7fb8e8', copper='#e0a35c', grid='#263749',
|
||||
line='#5da2dd', ring='#3a4e63', field='#0b1119', label='#b6c2d2'),
|
||||
}
|
||||
|
||||
def read_emblem(path):
|
||||
emblem = ET.parse(path).getroot()
|
||||
body = ''.join(ET.tostring(child, encoding='unicode') for child in emblem
|
||||
if child.tag.rsplit('}', 1)[-1] in ('defs', 'g'))
|
||||
return '\n'.join(line.rstrip() for line in body.splitlines()).strip()
|
||||
|
||||
|
||||
def txt(x, y, value, size=14, color=None, weight=400, anchor='start', mono=False, spacing=None):
|
||||
family = 'Menlo,Consolas,monospace' if mono else 'Arial,Helvetica,sans-serif'
|
||||
extra = f' letter-spacing="{spacing}"' if spacing is not None else ''
|
||||
return (f'<text x="{x}" y="{y}" font-family="{family}" font-size="{size}" '
|
||||
f'font-weight="{weight}" fill="{color}" text-anchor="{anchor}"{extra}>'
|
||||
f'{escape(str(value))}</text>')
|
||||
|
||||
|
||||
def start(height, theme, title, description, emblem_body):
|
||||
t = themes[theme]
|
||||
return [f'<svg xmlns="http://www.w3.org/2000/svg" width="1000" height="{height}" '
|
||||
f'viewBox="0 0 1000 {height}" role="img" aria-labelledby="title desc">',
|
||||
f'<title id="title">{escape(title)}</title><desc id="desc">{escape(description)}</desc>',
|
||||
'<defs><linearGradient id="surface" x1="0" y1="1" x2="1" y2="0">'
|
||||
f'<stop offset="0" stop-color="{t["bg"]}"/>'
|
||||
f'<stop offset="1" stop-color="{t["wash"]}"/></linearGradient>'
|
||||
'<linearGradient id="accent" x1="0" y1="0" x2="1" y2="0">'
|
||||
f'<stop offset="0" stop-color="{t["blue"]}"/>'
|
||||
f'<stop offset="1" stop-color="{t["copper"]}"/></linearGradient>'
|
||||
'<linearGradient id="area" x1="0" y1="0" x2="0" y2="1">'
|
||||
f'<stop offset="0" stop-color="{t["line"]}" stop-opacity=".22"/>'
|
||||
f'<stop offset="1" stop-color="{t["line"]}" stop-opacity=".015"/>'
|
||||
'</linearGradient></defs>',
|
||||
f'<rect x=".75" y=".75" width="998.5" height="{height-1.5}" rx="22" '
|
||||
f'fill="url(#surface)" stroke="{t["edge"]}" stroke-width="1.5"/>',
|
||||
f'<svg x="40" y="25" width="25" height="25" viewBox="230 213 570 570">{emblem_body}</svg>']
|
||||
|
||||
|
||||
def heading(parts, t, eyebrow, title, subtitle, value, value_label):
|
||||
parts.extend([
|
||||
txt(76, 43, eyebrow, 11, t['muted'], 600, mono=True, spacing=1.7),
|
||||
txt(40, 94, title, 32, t['ink'], 700),
|
||||
txt(41, 123, subtitle, 14, t['muted']),
|
||||
txt(958, 89, f'{value:,}', 45, t['ink'], 700, anchor='end'),
|
||||
txt(957, 114, value_label, 10, t['muted'], 600, anchor='end', mono=True, spacing=1.5),
|
||||
f'<path d="M40 146 H960" stroke="{t["edge"]}"/>',
|
||||
])
|
||||
|
||||
|
||||
def contributors(theme, people, snapshot, emblem_body):
|
||||
height = 206 + 76 * math.ceil(len(people) / 10)
|
||||
t = themes[theme]
|
||||
parts = start(height, theme, f'SILO community — {len(people)} contributors',
|
||||
f'The existing SILO community roll, including code, proposals and reports across related projects. '
|
||||
f'Gold rings retain the existing significant-contribution designation. Snapshot {snapshot}.', emblem_body)
|
||||
heading(parts, t, 'SILO / COMMUNITY', 'Contributors',
|
||||
'Code, proposals & reports across SILO and related projects', len(people), 'COMMUNITY CONTRIBUTORS')
|
||||
for row in range(math.ceil(len(people) / 10)):
|
||||
group = people[row * 10:(row + 1) * 10]
|
||||
row_width = len(group) * 91
|
||||
for col, person in enumerate(group):
|
||||
x = (1000 - row_width) / 2 + col * 91 + 45.5
|
||||
y = 199 + row * 76
|
||||
identifier = f'avatar-{row}-{col}'
|
||||
featured = bool(person.get('featured'))
|
||||
parts.append(f'<g><title>@{escape(person["handle"])} — {escape(person["what"])}</title>')
|
||||
parts.append(f'<defs><clipPath id="{identifier}"><circle cx="{x}" cy="{y}" r="29"/></clipPath></defs>')
|
||||
if featured:
|
||||
parts.append(f'<circle cx="{x}" cy="{y}" r="34" fill="{t["copper"]}" opacity=".09"/>')
|
||||
if person.get('avatarDataUrl'):
|
||||
parts.append(f'<image x="{x-29}" y="{y-29}" width="58" height="58" '
|
||||
f'clip-path="url(#{identifier})" href="{escape(person["avatarDataUrl"])}"/>')
|
||||
else:
|
||||
parts.append(f'<circle cx="{x}" cy="{y}" r="29" fill="{t["ring"]}"/>')
|
||||
parts.append(txt(x, y + 9, person['handle'][0].upper(), 26, t['ink'], 700, 'middle'))
|
||||
parts.append(f'<circle cx="{x}" cy="{y}" r="30.5" fill="none" '
|
||||
f'stroke="{t["copper"] if featured else t["ring"]}" stroke-width="{2 if featured else 1.25}"/></g>')
|
||||
parts.extend([
|
||||
f'<path d="M40 {height-42} H960" stroke="{t["edge"]}"/>',
|
||||
f'<circle cx="47" cy="{height-21}" r="4" fill="none" stroke="{t["copper"]}" stroke-width="1.5"/>',
|
||||
txt(61, height-17, 'Gold rings mark significant contributions', 12, t['muted']),
|
||||
txt(959, height-17, f'AS OF {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6),
|
||||
'</svg>',
|
||||
])
|
||||
return ''.join(parts)
|
||||
|
||||
|
||||
def stars(theme, history, snapshot, emblem_body):
|
||||
points = history['points']
|
||||
star_count = points[-1]['stars']
|
||||
t = themes[theme]
|
||||
provenance = ('Initial history reconstructed · Daily totals since ' + history['bootstrap']['through'] + ' · UTC'
|
||||
if history['bootstrap']['reconstructed'] else 'Observed daily star totals · UTC')
|
||||
parts = start(558, theme, f'SILO star history — {star_count:,} stars',
|
||||
f'GitHub repository pgsty/silo. {star_count:,} stars as of {snapshot}. ' +
|
||||
provenance, emblem_body)
|
||||
heading(parts, t, 'SILO / GITHUB', 'Star History', 'pgsty/silo', star_count, 'GITHUB STARS')
|
||||
left, right, top, bottom = 76, 958, 177, 440
|
||||
begin = date.fromisoformat(points[0]['date'])
|
||||
end = date.fromisoformat(points[-1]['date'])
|
||||
days = max(1, (end - begin).days)
|
||||
maximum = max(500, math.ceil(max(p['stars'] for p in points) / 500) * 500)
|
||||
tick_step = 10 ** max(0, int(math.log10(maximum)))
|
||||
xy = lambda day, n: (left + (right-left)*(date.fromisoformat(day)-begin).days / days,
|
||||
bottom-(bottom-top)*n/maximum)
|
||||
for value in range(0, maximum+1, tick_step):
|
||||
y = xy(points[0]['date'], value)[1]
|
||||
parts.append(f'<path d="M{left} {y:.2f} H{right}" stroke="{t["grid"]}" stroke-dasharray="4 6"/>')
|
||||
parts.append(txt(left-16, round(y+4, 2), f'{value / 1000:g}k' if value >= 1000 else str(value), 12, t['muted'], anchor='end'))
|
||||
dates = sorted({begin + timedelta(days=round((end-begin).days*i/5)) for i in range(6)})
|
||||
ticks = [(d.isoformat(), d.strftime('%b %Y') if days > 90 else d.strftime('%b %d')) for d in dates]
|
||||
for day, label in ticks:
|
||||
x = xy(day, 0)[0]
|
||||
parts.append(f'<path d="M{x:.2f} {top} V{bottom}" stroke="{t["grid"]}" stroke-opacity=".65"/>')
|
||||
anchor = 'start' if day == points[0]['date'] else 'end' if day == snapshot else 'middle'
|
||||
parts.append(txt(round(x, 2), 466, label, 12, t['muted'], anchor=anchor))
|
||||
coords = [xy(p['date'], p['stars']) for p in points]
|
||||
line = 'M' + ' L'.join(f'{x:.2f} {y:.2f}' for x, y in coords)
|
||||
area = line + f' L{coords[-1][0]:.2f} {bottom} L{left} {bottom} Z'
|
||||
parts.extend([
|
||||
f'<path d="{area}" fill="url(#area)"/>',
|
||||
f'<path d="{line}" fill="none" stroke="url(#accent)" stroke-width="3" '
|
||||
'stroke-linecap="round" stroke-linejoin="round"/>',
|
||||
f'<path d="M{left} {bottom} H{right}" stroke="{t["edge"]}"/>',
|
||||
])
|
||||
x, y = coords[-1]
|
||||
parts.extend([
|
||||
f'<circle cx="{x}" cy="{y}" r="10" fill="{t["copper"]}" opacity=".12"/>',
|
||||
f'<circle cx="{x}" cy="{y}" r="5" fill="{t["copper"]}" stroke="{t["bg"]}" stroke-width="2"/>',
|
||||
f'<path d="M40 491 H960" stroke="{t["edge"]}"/>',
|
||||
txt(40, 516, f'{begin:%b %Y} — {end:%b %Y}'.upper(), 10, t['muted'], 500, mono=True, spacing=.7),
|
||||
txt(959, 516, f'SNAPSHOT {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6),
|
||||
txt(40, 539, provenance, 11, t['muted']),
|
||||
'</svg>',
|
||||
])
|
||||
return ''.join(parts)
|
||||
@@ -0,0 +1 @@
|
||||
PyYAML==6.0.3
|
||||
@@ -0,0 +1,178 @@
|
||||
# Copyright (c) 2026 Feng Ruohang
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Regression checks for historical accuracy, contributor scope and SVG safety."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from urllib.error import URLError
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import render
|
||||
import update
|
||||
|
||||
NS = {'s': 'http://www.w3.org/2000/svg'}
|
||||
PNG = base64.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a7mgAAAAASUVORK5CYII=')
|
||||
|
||||
|
||||
def person(handle='Alice', group='reports', featured=False):
|
||||
return {'handle': handle, 'group': group, 'featured': featured,
|
||||
'what': 'A reviewed contribution', 'firstContribution': '2026-09-01'}
|
||||
|
||||
|
||||
def history():
|
||||
return {'repository': 'pgsty/silo',
|
||||
'bootstrap': {'through': '2026-09-15', 'reconstructed': True},
|
||||
'points': [{'date': '2026-09-14', 'stars': 100}, {'date': '2026-09-15', 'stars': 105}]}
|
||||
|
||||
|
||||
class HistoryTests(unittest.TestCase):
|
||||
def test_new_day_preserves_old_counts_and_unstars(self):
|
||||
before = history()
|
||||
after = update.update_history(before, '2026-09-16', 103)
|
||||
self.assertEqual(after['points'][:-1], before['points'])
|
||||
self.assertEqual(after['points'][-1], {'date': '2026-09-16', 'stars': 103})
|
||||
self.assertEqual(before, history())
|
||||
|
||||
def test_same_day_rerun_replaces_instead_of_appending(self):
|
||||
first = update.update_history(history(), '2026-09-15', 107)
|
||||
self.assertEqual(len(first['points']), 2)
|
||||
self.assertEqual(first, update.update_history(first, '2026-09-15', 107))
|
||||
|
||||
def test_missing_days_are_not_invented(self):
|
||||
result = update.update_history(history(), '2026-09-18', 106)
|
||||
self.assertEqual([p['date'] for p in result['points']], ['2026-09-14', '2026-09-15', '2026-09-18'])
|
||||
|
||||
def test_rejects_wrong_repository_and_corrupt_history(self):
|
||||
cases = []
|
||||
wrong = history(); wrong['repository'] = 'someone/else'; cases.append(wrong)
|
||||
duplicate = history(); duplicate['points'].append(duplicate['points'][-1]); cases.append(duplicate)
|
||||
unordered = history(); unordered['points'].reverse(); cases.append(unordered)
|
||||
negative = history(); negative['points'][0]['stars'] = -1; cases.append(negative)
|
||||
future = history(); future['points'][-1]['date'] = '2026-09-20'; cases.append(future)
|
||||
for case in cases:
|
||||
with self.subTest(case=case), self.assertRaises(ValueError):
|
||||
update.update_history(case, '2026-09-16', 100)
|
||||
|
||||
def test_first_run_has_no_fabricated_history(self):
|
||||
result = update.update_history(None, '2026-09-16', 10)
|
||||
self.assertFalse(result['bootstrap']['reconstructed'])
|
||||
self.assertEqual(result['points'], [{'date': '2026-09-16', 'stars': 10}])
|
||||
|
||||
|
||||
class ContributorTests(unittest.TestCase):
|
||||
def test_retries_truncated_json_before_using_it(self):
|
||||
with patch('update.request', side_effect=[b'{"partial":', b'{"ok":true}']), patch('update.time.sleep'):
|
||||
self.assertEqual(update.GitHub('').get('repos/pgsty/silo'), {'ok': True})
|
||||
|
||||
def test_paginates_past_one_full_page(self):
|
||||
class API(update.GitHub):
|
||||
def __init__(self): self.calls = []
|
||||
def get(self, path):
|
||||
self.calls.append(path)
|
||||
return list(range(100)) if 'page=1&' in path else [100]
|
||||
api = API()
|
||||
self.assertEqual(len(list(api.issues('pgsty/silo'))), 101)
|
||||
self.assertIn('state=all', api.calls[0])
|
||||
self.assertIn('page=2&', api.calls[1])
|
||||
|
||||
def test_bots_deduplication_unmerged_work_and_reviewed_credit(self):
|
||||
def issue(login, kind='issue', user_type='User'):
|
||||
item = {'user': {'login': login, 'type': user_type, 'avatar_url': ''}, 'created_at': '2026-09-02T00:00:00Z'}
|
||||
if kind != 'issue': item['pull_request'] = {'merged_at': None if kind == 'open' else '2026-09-03T00:00:00Z'}
|
||||
return item
|
||||
class API:
|
||||
def issues(self, _repo):
|
||||
return [issue('alice'), issue('Bob', 'open'), issue('Bob', 'merged'),
|
||||
issue('Carol', 'open'), issue('Copilot'), issue('robot', user_type='Bot')]
|
||||
curated = {'repositories': ['pgsty/silo', 'pgsty/mc'], 'bots': ['Copilot'],
|
||||
'people': [person('Alice', featured=True), person('Reporter'), person('Copilot')]}
|
||||
result = update.collect_people(API(), curated)
|
||||
self.assertEqual({p['handle'] for p in result}, {'Alice', 'Bob', 'Carol', 'Reporter'})
|
||||
self.assertEqual(result[0]['handle'], 'Bob')
|
||||
self.assertEqual(result[1]['handle'], 'Carol')
|
||||
self.assertTrue(next(p for p in result if p['handle'] == 'Alice')['featured'])
|
||||
self.assertEqual(next(p for p in result if p['handle'] == 'Bob')['group'], 'code')
|
||||
self.assertFalse(next(p for p in result if p['handle'] == 'Carol')['featured'])
|
||||
|
||||
def test_newer_reviewed_preview_survives_until_site_catches_up(self):
|
||||
remote = {'updated': '2026-09-16T03:00:00+00:00'}
|
||||
cached = {'updated': '2026-09-16T04:00:00+00:00'}
|
||||
self.assertIs(update.select_curated(remote, cached), cached)
|
||||
newer = {'updated': '2026-09-17T03:00:00+00:00'}
|
||||
self.assertIs(update.select_curated(newer, cached), newer)
|
||||
|
||||
def test_avatar_failure_reuses_raster_cache(self):
|
||||
previous = {**person(), 'avatarDataUrl': update.raster_data_url(PNG)}
|
||||
with patch('update.request', side_effect=URLError('unavailable')):
|
||||
result = update.add_avatars(None, [{**person(), 'avatarUrl': 'https://avatars.githubusercontent.com/u/1'}], [previous])
|
||||
self.assertEqual(result[0]['avatarDataUrl'], previous['avatarDataUrl'])
|
||||
with self.assertRaises(ValueError): update.raster_data_url(b'<svg onload="bad()"/>')
|
||||
with self.assertRaises(ValueError): update.cached_avatar({'avatarDataUrl': 'data:image/svg+xml;base64,PHN2Zy8+'})
|
||||
|
||||
def test_fetch_failure_leaves_published_assets_untouched(self):
|
||||
class API:
|
||||
def get(self, path):
|
||||
if path == 'repos/pgsty/silo': return {'full_name': 'pgsty/silo', 'stargazers_count': 106}
|
||||
raise URLError('roster unavailable')
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
out = Path(directory)
|
||||
original = json.dumps(history())
|
||||
(out / 'history.json').write_text(original)
|
||||
(out / 'contributors-light.svg').write_text('previous image')
|
||||
with self.assertRaises(URLError): update.refresh(out, API(), Path('.'))
|
||||
self.assertEqual((out / 'history.json').read_text(), original)
|
||||
self.assertEqual((out / 'contributors-light.svg').read_text(), 'previous image')
|
||||
|
||||
|
||||
class RenderTests(unittest.TestCase):
|
||||
def test_real_emblem_generates_clean_xml(self):
|
||||
emblem = render.read_emblem(Path(__file__).resolve().parents[2] / '.github/silo.svg')
|
||||
svg = render.contributors('light', [person()], '2026-09-16', emblem)
|
||||
ET.fromstring(svg)
|
||||
self.assertTrue(all(line == line.rstrip() for line in svg.splitlines()))
|
||||
|
||||
def test_all_avatars_fit_when_the_roster_grows(self):
|
||||
people = [{**person(f'person-{i}'), 'avatarDataUrl': update.raster_data_url(PNG)} for i in range(151)]
|
||||
for theme in ('light', 'dark'):
|
||||
root = ET.fromstring(render.contributors(theme, people, '2026-09-16', ''))
|
||||
images = root.findall('.//s:image', NS)
|
||||
self.assertEqual(len(images), 151)
|
||||
footer = float(root.attrib['height']) - 42
|
||||
self.assertTrue(all(float(i.attrib['y']) + float(i.attrib['height']) < footer for i in images))
|
||||
self.assertTrue(all(i.attrib['href'].startswith('data:image/png;base64,') for i in images))
|
||||
|
||||
def test_untrusted_text_is_escaped(self):
|
||||
data = [{**person(), 'what': '<script>alert("x")</script> & contributions'}]
|
||||
svg = render.contributors('light', data, '2026-09-16', '')
|
||||
root = ET.fromstring(svg)
|
||||
self.assertEqual(root.findall('.//s:script', NS), [])
|
||||
self.assertIn('<script>', svg)
|
||||
|
||||
def test_single_point_and_decreasing_star_history_render(self):
|
||||
for data in (update.update_history(None, '2026-09-16', 0), update.update_history(history(), '2026-09-16', 90)):
|
||||
for theme in ('light', 'dark'):
|
||||
svg = render.stars(theme, data, '2026-09-16', '')
|
||||
ET.fromstring(svg)
|
||||
self.assertNotIn('nan', svg.lower())
|
||||
self.assertNotIn('inf', svg.lower())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2026 Feng Ruohang
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Refresh the generated-asset checkout; publishing is handled by the workflow."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timezone
|
||||
import json
|
||||
from http.client import IncompleteRead
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import yaml
|
||||
|
||||
import render
|
||||
|
||||
REPOSITORY = 'pgsty/silo'
|
||||
SOURCE = 'repos/pgsty/silo.pgsty.com/contents/data/home/contributors.yaml?ref=main'
|
||||
GROUPS = ('code', 'proposed', 'reports')
|
||||
HANDLE = re.compile(r'[A-Za-z0-9][A-Za-z0-9-]{0,38}\Z')
|
||||
|
||||
|
||||
def request(url, token='', limit=8 * 1024 * 1024):
|
||||
headers = {'User-Agent': 'silo-repository-cards', 'Accept': 'application/vnd.github+json'}
|
||||
if urlparse(url).netloc == 'api.github.com':
|
||||
headers['X-GitHub-Api-Version'] = '2022-11-28'
|
||||
if token:
|
||||
headers['Authorization'] = f'Bearer {token}'
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urlopen(Request(url, headers=headers), timeout=25) as response:
|
||||
data = response.read(limit + 1)
|
||||
if len(data) > limit:
|
||||
raise ValueError('Response exceeds the size limit')
|
||||
expected = response.headers.get('Content-Length')
|
||||
if expected is not None and len(data) != int(expected):
|
||||
raise URLError('Incomplete response body')
|
||||
return data
|
||||
except HTTPError as exc:
|
||||
if exc.code < 500 or attempt == 2:
|
||||
raise
|
||||
except (URLError, TimeoutError, IncompleteRead):
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(attempt + 1)
|
||||
|
||||
|
||||
class GitHub:
|
||||
def __init__(self, token):
|
||||
self.token = token
|
||||
|
||||
def get(self, path):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
return json.loads(request('https://api.github.com/' + path, self.token))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
if attempt == 2:
|
||||
raise ValueError(f'Incomplete or invalid GitHub JSON: {path}') from exc
|
||||
time.sleep(attempt + 1)
|
||||
|
||||
def issues(self, repository):
|
||||
page = 1
|
||||
while True:
|
||||
batch = self.get(f'repos/{repository}/issues?state=all&per_page=100&page={page}&sort=created&direction=asc')
|
||||
if not isinstance(batch, list):
|
||||
raise ValueError(f'Invalid issues response for {repository}')
|
||||
yield from batch
|
||||
if len(batch) < 100:
|
||||
return
|
||||
page += 1
|
||||
|
||||
|
||||
def curated_snapshot(data, revision):
|
||||
updated = str(data['updated'])
|
||||
datetime.fromisoformat(updated)
|
||||
repositories = [item['repo'] for item in data['repositories']]
|
||||
if not repositories or any(not re.fullmatch(r'pgsty/[A-Za-z0-9_.-]+', repo) for repo in repositories):
|
||||
raise ValueError('Invalid contributor repository scope')
|
||||
people = []
|
||||
for group in GROUPS:
|
||||
for entry in data[group]:
|
||||
if not HANDLE.fullmatch(entry['handle']):
|
||||
raise ValueError('Invalid GitHub contributor handle')
|
||||
people.append({
|
||||
'handle': entry['handle'], 'group': group,
|
||||
'featured': bool(entry.get('featured')), 'what': entry['what'],
|
||||
'firstContribution': str(entry.get('firstContribution', '9999-12-31')),
|
||||
})
|
||||
if not people or len({p['handle'].lower() for p in people}) != len(people):
|
||||
raise ValueError('Empty or duplicate contributor roster')
|
||||
return {'updated': updated, 'revision': revision, 'repositories': repositories,
|
||||
'bots': data.get('bots', ['Copilot', 'dependabot[bot]']), 'people': people}
|
||||
|
||||
|
||||
def select_curated(remote, cached):
|
||||
# The initial, approved preview can contain reviewed credit not published by
|
||||
# the companion site yet. Keep that newer snapshot until the site catches up.
|
||||
if cached and datetime.fromisoformat(cached['updated']) > datetime.fromisoformat(remote['updated']):
|
||||
return cached
|
||||
return remote
|
||||
|
||||
|
||||
def collect_people(api, curated):
|
||||
bots = {name.lower() for name in curated['bots']}
|
||||
people = {p['handle'].lower(): dict(p) for p in curated['people']
|
||||
if p['handle'].lower() not in bots and not p['handle'].lower().endswith('[bot]')}
|
||||
order = {p['handle'].lower(): index for index, p in enumerate(curated['people'])}
|
||||
for repository in curated['repositories']:
|
||||
print(f'Reading issue and PR authors: {repository}', flush=True)
|
||||
for issue in api.issues(repository):
|
||||
user = issue.get('user') or {}
|
||||
handle = user.get('login', '')
|
||||
key = handle.lower()
|
||||
if user.get('type') != 'User' or key in bots or key.endswith('[bot]'):
|
||||
continue
|
||||
if not HANDLE.fullmatch(handle):
|
||||
raise ValueError('Invalid issue author')
|
||||
pr = issue.get('pull_request')
|
||||
group = 'code' if pr and pr.get('merged_at') else 'proposed' if pr else 'reports'
|
||||
first = issue['created_at'][:10]
|
||||
date.fromisoformat(first)
|
||||
person = people.setdefault(key, {
|
||||
'handle': handle, 'group': group, 'featured': False,
|
||||
'what': 'Contributed an issue or pull request to SILO and related projects',
|
||||
'firstContribution': first,
|
||||
})
|
||||
person['avatarUrl'] = user.get('avatar_url', '')
|
||||
person['firstContribution'] = min(person['firstContribution'], first)
|
||||
if GROUPS.index(group) < GROUPS.index(person['group']):
|
||||
person['group'] = group
|
||||
if not people:
|
||||
raise ValueError('No human contributors were collected')
|
||||
return sorted(people.values(), key=lambda p: (
|
||||
GROUPS.index(p['group']), not p['featured'],
|
||||
order.get(p['handle'].lower(), len(order)), p['firstContribution'], p['handle'].lower()))
|
||||
|
||||
|
||||
def raster_data_url(data):
|
||||
if data.startswith(b'\x89PNG\r\n\x1a\n'):
|
||||
mime = 'image/png'
|
||||
elif data.startswith(b'\xff\xd8\xff'):
|
||||
mime = 'image/jpeg'
|
||||
elif data.startswith((b'GIF87a', b'GIF89a')):
|
||||
mime = 'image/gif'
|
||||
elif data[:4] == b'RIFF' and data[8:12] == b'WEBP':
|
||||
mime = 'image/webp'
|
||||
else:
|
||||
raise ValueError('Avatar is not a raster image')
|
||||
return f'data:{mime};base64,' + base64.b64encode(data).decode('ascii')
|
||||
|
||||
|
||||
def cached_avatar(person):
|
||||
value = person.get('avatarDataUrl', '')
|
||||
if not value:
|
||||
return ''
|
||||
prefix, encoded = value.split(',', 1)
|
||||
if prefix not in ('data:image/png;base64', 'data:image/jpeg;base64', 'data:image/gif;base64', 'data:image/webp;base64'):
|
||||
raise ValueError('Invalid cached avatar format')
|
||||
raw = base64.b64decode(encoded, validate=True)
|
||||
if len(raw) > 512 * 1024 or raster_data_url(raw) != value:
|
||||
raise ValueError('Invalid cached avatar')
|
||||
return value
|
||||
|
||||
|
||||
def add_avatars(api, people, previous):
|
||||
cached = {p['handle'].lower(): cached_avatar(p) for p in previous}
|
||||
|
||||
def update(person):
|
||||
person = dict(person)
|
||||
try:
|
||||
url = person.pop('avatarUrl', '') or api.get('users/' + person['handle'])['avatar_url']
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme != 'https' or parsed.netloc != 'avatars.githubusercontent.com':
|
||||
raise ValueError('Unexpected avatar host')
|
||||
data = request(url + ('&' if '?' in url else '?') + 's=96', limit=512 * 1024)
|
||||
person['avatarDataUrl'] = raster_data_url(data)
|
||||
except (HTTPError, URLError, TimeoutError, IncompleteRead, ValueError, KeyError) as exc:
|
||||
person.pop('avatarUrl', None)
|
||||
person['avatarDataUrl'] = cached.get(person['handle'].lower(), '')
|
||||
print(f'Avatar fallback for @{person["handle"]}: {type(exc).__name__}', file=sys.stderr)
|
||||
return person
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as pool:
|
||||
return list(pool.map(update, people))
|
||||
|
||||
|
||||
def update_history(history, day, stars):
|
||||
date.fromisoformat(day)
|
||||
if type(stars) is not int or stars < 0:
|
||||
raise ValueError('Invalid repository star count')
|
||||
if history is None:
|
||||
history = {'repository': REPOSITORY, 'bootstrap': {'through': day, 'reconstructed': False}, 'points': []}
|
||||
if history['repository'] != REPOSITORY:
|
||||
raise ValueError('Star history belongs to a different repository')
|
||||
date.fromisoformat(history['bootstrap']['through'])
|
||||
dates = []
|
||||
for point in history['points']:
|
||||
date.fromisoformat(point['date'])
|
||||
if type(point['stars']) is not int or point['stars'] < 0:
|
||||
raise ValueError('Invalid historical star count')
|
||||
dates.append(point['date'])
|
||||
if dates != sorted(set(dates)) or any(d > day for d in dates):
|
||||
raise ValueError('History contains duplicate, unordered, or future dates')
|
||||
# Replace today's observation, preserve previous days, and allow unstars.
|
||||
points = [dict(p) for p in history['points'] if p['date'] != day]
|
||||
points.append({'date': day, 'stars': stars})
|
||||
return {**history, 'points': points}
|
||||
|
||||
|
||||
def read_json(path, default=None):
|
||||
return json.loads(path.read_text()) if path.exists() else default
|
||||
|
||||
|
||||
def refresh(output, api, source_root):
|
||||
day = datetime.now(timezone.utc).date().isoformat()
|
||||
metadata = api.get('repos/' + REPOSITORY)
|
||||
if metadata['full_name'].lower() != REPOSITORY:
|
||||
raise ValueError('Unexpected repository metadata')
|
||||
history = update_history(read_json(output / 'history.json'), day, metadata['stargazers_count'])
|
||||
source = api.get(SOURCE)
|
||||
reviewed = yaml.safe_load(base64.b64decode(source['content'], validate=False))
|
||||
curated = select_curated(curated_snapshot(reviewed, source['sha']), read_json(output / 'curated.json'))
|
||||
people = collect_people(api, curated)
|
||||
previous = read_json(output / 'contributors.json', {}).get('people', [])
|
||||
people = add_avatars(api, people, previous)
|
||||
emblem = render.read_emblem(source_root / '.github/silo.svg')
|
||||
payloads = {}
|
||||
for theme in ('light', 'dark'):
|
||||
payloads[f'contributors-{theme}.svg'] = render.contributors(theme, people, day, emblem) + '\n'
|
||||
payloads[f'star-history-{theme}.svg'] = render.stars(theme, history, day, emblem) + '\n'
|
||||
for svg in payloads.values():
|
||||
ET.fromstring(svg)
|
||||
for name, data in {
|
||||
'history.json': history,
|
||||
'curated.json': curated,
|
||||
'contributors.json': {'repository': REPOSITORY, 'updated': day, 'people': people},
|
||||
}.items():
|
||||
payloads[name] = json.dumps(data, indent=2, ensure_ascii=False) + '\n'
|
||||
payloads['README.md'] = f'''# SILO repository cards
|
||||
|
||||
Generated by [Repository Cards](https://github.com/pgsty/silo/actions/workflows/repository-cards.yml)
|
||||
at 00:00 UTC daily (08:00 Asia/Shanghai). GitHub may queue scheduled runs.
|
||||
|
||||
Snapshot: {day}. {metadata['stargazers_count']:,} stars; {len(people)} community contributors.
|
||||
|
||||
- `contributors-light.svg` / `contributors-dark.svg`: human issue and PR authors across the SILO project scope, plus reviewed acknowledgements. Bots are excluded. Gold rings follow the reviewed companion-site roster; new authors are collected automatically.
|
||||
- `star-history-light.svg` / `star-history-dark.svg`: initial history reconstructed from the then-current stargazers; later points are daily observed totals, including decreases. Missing days are not fabricated.
|
||||
- `curated.json`: a cache of reviewed contributor credit from `pgsty/silo.pgsty.com/data/home/contributors.yaml`. The approved initial preview may be newer than the published site; a newer reviewed snapshot is retained until the site catches up.
|
||||
- `contributors.json`: generated contributor data and embedded raster avatars. Failed avatar refreshes use the previous image, or an initial when no image is available.
|
||||
- `history.json`: persistent daily totals. Keep this file when regenerating images.
|
||||
|
||||
The SVGs are self-contained. Source and instructions live on the default branch;
|
||||
this branch contains generated assets only. Do not merge it into `main`.
|
||||
'''
|
||||
# Collect and validate everything before touching the publication checkout.
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
for filename, text in payloads.items():
|
||||
(output / filename).write_text(text)
|
||||
print(f'{day}: {len(people)} contributors; {metadata["stargazers_count"]:,} stars; {len(history["points"])} history points')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
configured = os.environ.get('GITHUB_REPOSITORY', REPOSITORY)
|
||||
if configured.lower() != REPOSITORY:
|
||||
raise SystemExit('This workflow is scoped to pgsty/silo')
|
||||
refresh(args.output, GitHub(os.environ.get('GH_TOKEN', '')), Path(__file__).resolve().parents[2])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -18,8 +18,8 @@ function start_silo_5drive() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
|
||||
@@ -28,8 +28,8 @@ function verify_rewrite() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ACCESS_KEY=silo
|
||||
export MINIO_SECRET_KEY=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_SECRET_KEY=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -87,7 +87,7 @@ function verify_rewrite() {
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key silo \
|
||||
-secret-key silo123 \
|
||||
-secret-key silo1234 \
|
||||
-endpoint "http://127.0.0.1:${start_port}/" 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
@@ -105,14 +105,14 @@ function verify_rewrite() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo123"
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "silo" "silo1234"
|
||||
sleep 1
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key silo \
|
||||
-secret-key silo123 \
|
||||
-secret-key silo1234 \
|
||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
||||
echo "server1 log:"
|
||||
cat "${WORK_DIR}/server1.log"
|
||||
|
||||
@@ -74,8 +74,8 @@ function test_silo_with_timeout() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MC_HOST_silo="http://silo:silo123@127.0.0.1:${start_port}/"
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MC_HOST_silo="http://silo:silo1234@127.0.0.1:${start_port}/"
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
mkdir ${WORK_DIR}
|
||||
|
||||
@@ -15,10 +15,10 @@ WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
export MINT_MODE=core
|
||||
export MINT_DATA_DIR="$WORK_DIR/data"
|
||||
export SERVER_ENDPOINT="127.0.0.1:9000"
|
||||
export MC_HOST_verify="http://silo:silo123@${SERVER_ENDPOINT}/"
|
||||
export MC_HOST_verify_ipv6="http://silo:silo123@[::1]:9000/"
|
||||
export MC_HOST_verify="http://silo:silo1234@${SERVER_ENDPOINT}/"
|
||||
export MC_HOST_verify_ipv6="http://silo:silo1234@[::1]:9000/"
|
||||
export ACCESS_KEY="silo"
|
||||
export SECRET_KEY="silo123"
|
||||
export SECRET_KEY="silo1234"
|
||||
export ENABLE_HTTPS=0
|
||||
export GO111MODULE=on
|
||||
export GOGC=25
|
||||
@@ -225,7 +225,7 @@ function __init__() {
|
||||
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-verified-fixture.sh" \
|
||||
https://raw.githubusercontent.com/pgsty/mc/4c4dcc4b55baf238cd0c81030d77945b3828f157/functional-tests.sh \
|
||||
@@ -282,7 +282,6 @@ function main() {
|
||||
purge "$WORK_DIR"
|
||||
}
|
||||
|
||||
(__init__ "$@" && main "$@")
|
||||
rv=$?
|
||||
purge "$WORK_DIR"
|
||||
exit "$rv"
|
||||
trap 'purge "$WORK_DIR"' EXIT
|
||||
__init__ "$@"
|
||||
main "$@"
|
||||
|
||||
@@ -15,7 +15,7 @@ SILO=("$PWD/silo" --config-dir "$SILO_CONFIG_DIR" server)
|
||||
|
||||
function start_silo_3_node() {
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -37,7 +37,7 @@ function start_silo_3_node() {
|
||||
pid3=$!
|
||||
disown $pid3
|
||||
|
||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
||||
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||
|
||||
timeout 15m /tmp/mc ready mysilo || fail
|
||||
|
||||
@@ -116,7 +116,7 @@ function __init__() {
|
||||
mkdir -p "$SILO_CONFIG_DIR"
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
if [ ! -f /tmp/mc ]; then
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||
|
||||
@@ -17,7 +17,7 @@ function start_silo() {
|
||||
start_port=$1
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
||||
unset MINIO_CI_CD
|
||||
unset CI
|
||||
|
||||
@@ -20,7 +20,7 @@ function start_silo_3_node() {
|
||||
done
|
||||
|
||||
export MINIO_ROOT_USER=silo
|
||||
export MINIO_ROOT_PASSWORD=silo123
|
||||
export MINIO_ROOT_PASSWORD=silo1234
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
export MINIO_CI_CD=1
|
||||
|
||||
@@ -46,7 +46,7 @@ function start_silo_3_node() {
|
||||
pid3=$!
|
||||
disown $pid3
|
||||
|
||||
export MC_HOST_mysilo="http://silo:silo123@127.0.0.1:$((start_port + 1))"
|
||||
export MC_HOST_mysilo="http://silo:silo1234@127.0.0.1:$((start_port + 1))"
|
||||
timeout 15m /tmp/mc ready mysilo || fail
|
||||
|
||||
[ ${first_time} -eq 0 ] && upload_objects
|
||||
@@ -117,7 +117,7 @@ function __init__() {
|
||||
mkdir -p "$SILO_CONFIG_DIR"
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo123"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
echo '{"version": "3", "credential": {"accessKey": "silo", "secretKey": "silo1234"}, "region": "us-east-1"}' >"$SILO_CONFIG_DIR/config.json"
|
||||
|
||||
if [ ! -f /tmp/mc ]; then
|
||||
"$(git rev-parse --show-toplevel)/buildscripts/install-mcli.sh" /tmp/mc
|
||||
|
||||
@@ -113,7 +113,7 @@ helm_run template my-release "${new_chart}" \
|
||||
go run ./buildscripts/helm-migration-guard "${old_render}" "${new_render}"
|
||||
|
||||
helm_run package "${new_chart}" --destination "${output_dir}" >/dev/null
|
||||
test -s "${work_dir}/silo-7.0.1.tgz"
|
||||
test -s "${work_dir}/silo-7.0.3.tgz"
|
||||
if find "${work_dir}" -maxdepth 1 -type f -name 'minio-*.tgz' | grep -q .; then
|
||||
echo "Helm packaging emitted a legacy MinIO chart name" >&2
|
||||
exit 1
|
||||
|
||||
@@ -11,6 +11,33 @@ fail() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Git is already a build prerequisite. --no-index also scans untracked files
|
||||
# and exported source trees. As before, honor .gitignore only in a worktree.
|
||||
search_ignore=--no-exclude-standard
|
||||
if git rev-parse --show-toplevel >/dev/null 2>&1; then
|
||||
search_ignore=--exclude-standard
|
||||
fi
|
||||
search_errors="$(mktemp)"
|
||||
trap 'rm -f "${search_errors}"' EXIT
|
||||
|
||||
search_text() {
|
||||
local status=0
|
||||
git --no-pager grep --no-index "${search_ignore}" --no-color -nIHE "$@" 2>"${search_errors}" || status=$?
|
||||
# Git can return 1 for both no matches and a file read error. Check stderr
|
||||
# as well, before callers filter matches through their allowlists.
|
||||
if [ "${status}" -gt 1 ] || [ -s "${search_errors}" ]; then
|
||||
cat "${search_errors}" >&2
|
||||
fail "could not scan source files"
|
||||
fi
|
||||
}
|
||||
|
||||
# Keep hidden files and build output out of the repository-wide scans.
|
||||
source_excludes=(
|
||||
':(exclude,glob)**/.*'
|
||||
':(exclude,glob)**/.*/**'
|
||||
':(exclude)dist/**'
|
||||
)
|
||||
|
||||
require_file() {
|
||||
[ -f "$1" ] || fail "missing required file: $1"
|
||||
}
|
||||
@@ -36,7 +63,7 @@ for file in \
|
||||
buildscripts/verify-helm-migration.sh \
|
||||
Dockerfile.goreleaser \
|
||||
Dockerfile.distroless \
|
||||
dockerscripts/download-static-curl.sh \
|
||||
dockerscripts/build-static-curl.sh \
|
||||
dockerscripts/docker-entrypoint.sh \
|
||||
helm/silo/Chart.yaml \
|
||||
helm/silo/values.yaml \
|
||||
@@ -101,7 +128,7 @@ require_text Dockerfile.goreleaser "Published checksum drift"
|
||||
require_text Dockerfile.distroless 'COPY --chmod=0755 silo /usr/bin/silo'
|
||||
require_text Dockerfile.distroless 'ENTRYPOINT ["/usr/bin/silo"]'
|
||||
require_text Dockerfile.distroless '"/usr/bin/silo", "healthcheck", "ready"'
|
||||
require_text dockerscripts/download-static-curl.sh "sha256sum -c"
|
||||
require_text dockerscripts/build-static-curl.sh "sha256sum -c"
|
||||
require_text helm/silo/Chart.yaml "name: silo"
|
||||
require_text helm/silo/values.yaml "repository: pgsty/silo"
|
||||
require_text helm/silo/templates/deployment.yaml "/usr/bin/docker-entrypoint.sh silo server"
|
||||
@@ -152,21 +179,27 @@ if grep -Ev '^[[:space:]]*(#|$)' silo.env | grep -q '='; then
|
||||
fail "silo.env must not contain active assignments that shadow /etc/default/minio"
|
||||
fi
|
||||
|
||||
if rg -n 'pgsty/minio:' .github/workflows Dockerfile.goreleaser helm/silo; then
|
||||
delivery_hits="$(search_text 'pgsty/minio:' -- .github/workflows Dockerfile.goreleaser helm/silo \
|
||||
':(exclude,glob).github/workflows/**/.*' ':(exclude,glob).github/workflows/**/.*/**' \
|
||||
':(exclude,glob)helm/silo/**/.*' ':(exclude,glob)helm/silo/**/.*/**')"
|
||||
if [ -n "${delivery_hits}" ]; then
|
||||
printf '%s\n' "${delivery_hits}" >&2
|
||||
fail "an active delivery surface still publishes the frozen pgsty/minio image"
|
||||
fi
|
||||
|
||||
# The repository and its default branch are pgsty/silo and main. The invariant
|
||||
# is that the old name is never a live target, not that it is never spoken: the
|
||||
# READMEs have to name it to explain the rename and to point at the archived
|
||||
# artifacts, which is the opposite of stranding a reader on it.
|
||||
# artifacts, which is the opposite of stranding a reader on it. CONTRIBUTORS.md
|
||||
# also quotes historical issue titles.
|
||||
#
|
||||
# So two rules. First, no live URL may resolve to the old repository anywhere,
|
||||
# READMEs included.
|
||||
stale_repo_url="$(rg -n -e 'github\.com/pgsty/minio' -e 'hub\.docker\.com/r/pgsty/minio' \
|
||||
--glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
# READMEs and CONTRIBUTORS.md included.
|
||||
old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
|
||||
stale_repo_url="$(search_text -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" -- . \
|
||||
"${source_excludes[@]}" ':(exclude)SILO_REBRANDING_MIGRATION.md' \
|
||||
':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
|
||||
stale_repo_url="$(printf '%s\n' "${stale_repo_url}" |
|
||||
sed 's#^\./##' | grep -v '^buildscripts/verify-rebrand\.sh:' || true)"
|
||||
if [ -n "${stale_repo_url}" ]; then
|
||||
printf '%s\n' "${stale_repo_url}" >&2
|
||||
@@ -175,35 +208,35 @@ fi
|
||||
|
||||
# Second, the bare name may only appear where it is deliberate: the pinned
|
||||
# pre-rebrand image digest in the upgrade test, the two guards that refuse a
|
||||
# legacy image, and the two READMEs that document the rename and the archived
|
||||
# minio branch.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md):'
|
||||
stale_repo="$(rg -n 'pgsty/minio' --glob '!.git/**' --glob '!dist/**' \
|
||||
--glob '!SILO_REBRANDING_MIGRATION.md' \
|
||||
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . |
|
||||
# legacy image, the two READMEs that document the rename and the archived
|
||||
# minio branch, and historical issue titles in CONTRIBUTORS.md.
|
||||
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
|
||||
stale_repo="$(search_text "${old_repo_pattern}" -- . "${source_excludes[@]}" \
|
||||
':(exclude)SILO_REBRANDING_MIGRATION.md' ':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
|
||||
stale_repo="$(printf '%s\n' "${stale_repo}" |
|
||||
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
|
||||
if [ -n "${stale_repo}" ]; then
|
||||
printf '%s\n' "${stale_repo}" >&2
|
||||
fail "a source reference still names the pre-rename pgsty/minio repository"
|
||||
fi
|
||||
|
||||
stale_branch="$(rg -n 'pgsty/silo/(blob/|tree/|raw/)?master' \
|
||||
--glob '!.git/**' --glob '!dist/**' . || true)"
|
||||
stale_branch="$(search_text 'pgsty/silo/(blob/|tree/|raw/)?master' -- . "${source_excludes[@]}")"
|
||||
if [ -n "${stale_branch}" ]; then
|
||||
printf '%s\n' "${stale_branch}" >&2
|
||||
fail "a link still targets the retired master branch; raw and Actions URLs do not follow a branch rename"
|
||||
fi
|
||||
|
||||
for workflow in .github/workflows/go.yml .github/workflows/vulncheck.yml; do
|
||||
if rg -q '^\s+- master$' "${workflow}"; then
|
||||
stale_filter="$(search_text '^[[:space:]]+- master$' -- "${workflow}")"
|
||||
if [ -n "${stale_filter}" ]; then
|
||||
fail "${workflow} still filters on master and would go silently dormant on main"
|
||||
fi
|
||||
require_text "${workflow}" " - main"
|
||||
done
|
||||
|
||||
network_hits="$(rg -n --glob '*.go' --glob '!**/*_test.go' \
|
||||
'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' \
|
||||
cmd internal || true)"
|
||||
network_hits="$(search_text \
|
||||
'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' -- \
|
||||
':(glob)cmd/**/*.go' ':(glob)internal/**/*.go' ':(exclude,glob)**/*_test.go' "${source_excludes[@]}")"
|
||||
network_hits="$(printf '%s\n' "${network_hits}" | grep -Ev '^[^:]+:[0-9]+:[[:space:]]*//' || true)"
|
||||
if [ -n "${network_hits}" ]; then
|
||||
printf '%s\n' "${network_hits}" >&2
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type abandonedCleanupCommitDisk struct {
|
||||
StorageAPI
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d abandonedCleanupCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
var success int64
|
||||
gate := &scopeStagedContext{
|
||||
Context: ctx, meta: filepath.Join(d.Endpoint().Path, dv, dp, fi.DataDir, "part.1"),
|
||||
entered: d.entered, release: d.release,
|
||||
tracker: healthDiskCtxValue{lastSuccess: &success},
|
||||
}
|
||||
return d.StorageAPI.RenameData(gate, sv, sp, fi, dv, dp, o)
|
||||
}
|
||||
|
||||
// A lost namespace lease can admit cleanup while a disk transaction is between
|
||||
// moving its external shards and replacing xl.meta. Those shards are live work,
|
||||
// even though the old metadata does not refer to them yet. Cleanup must wait.
|
||||
func TestMutationAbandonedCleanupWaitsForCommit(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-during-commit"
|
||||
body := strings.Repeat("new payload", 200000)
|
||||
mutationPut(t, z, bucket, object, 0, strings.Repeat("old payload", 200000), time.Now().Add(-time.Minute))
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = abandonedCleanupCommitDisk{d, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
|
||||
committed <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before shard barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not move all external shards")
|
||||
}
|
||||
}
|
||||
t.Log("All eight new external shard directories exist; old xl.meta is still active")
|
||||
cleaned := make(chan error, len(disks))
|
||||
for _, d := range disks {
|
||||
go func(d StorageAPI) { cleaned <- d.CleanAbandonedData(context.Background(), bucket, object) }(d)
|
||||
}
|
||||
early := 0
|
||||
timer := time.NewTimer(150 * time.Millisecond)
|
||||
wait:
|
||||
for {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
early++
|
||||
t.Errorf("abandoned-data cleanup overtook unfinished commit: %v", err)
|
||||
if early == len(disks) {
|
||||
break wait
|
||||
}
|
||||
case <-timer.C:
|
||||
break wait
|
||||
}
|
||||
}
|
||||
timer.Stop()
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err != nil {
|
||||
t.Fatalf("PUT failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not finish")
|
||||
}
|
||||
for i := early; i < len(disks); i++ {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup did not finish")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("acknowledged PUT unreadable after abandoned-data cleanup: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("acknowledged external shards lost: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup can also arrive after a disk has committed but before its delayed
|
||||
// reply lets the coordinator decide to undo a failed quorum. The previous
|
||||
// generation's backup is still owned by that unfinished transaction.
|
||||
func TestMutationAbandonedCleanupPreservesUndoBackup(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-before-undo"
|
||||
body := strings.Repeat("last acknowledged", 150000)
|
||||
base := time.Now().Add(-time.Minute)
|
||||
mutationPut(t, z, bucket, object, 0, body, base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = delayedRenameResult{d, i >= 4, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
next := strings.Repeat("failed update", 180000)
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(next), int64(len(next)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Second)})
|
||||
committed <- err
|
||||
}()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before reply barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("disk commit did not reach reply barrier")
|
||||
}
|
||||
}
|
||||
for _, d := range disks[:4] {
|
||||
if err := d.CleanAbandonedData(t.Context(), bucket, object); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err == nil {
|
||||
t.Fatal("four-of-eight commit unexpectedly reached write quorum")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("failed PUT did not finish undo")
|
||||
}
|
||||
swap(disks)
|
||||
for i, d := range disks {
|
||||
fi, err := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !fi.ModTime.Equal(base) {
|
||||
t.Errorf("disk%d lost last acknowledged generation after failed PUT: mtime=%v err=%v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("last acknowledged value unreadable after cleanup and undo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("last acknowledged value lost after cleanup and undo: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -25,7 +25,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Data types used for returning dummy access control
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
)
|
||||
|
||||
func TestAccountInfoReflectsBucketMutations(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
bed, err := prepareAdminErasureTestBed(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer bed.TearDown()
|
||||
previousCache := listBucketsCache
|
||||
defer func() { listBucketsCache = previousCache }()
|
||||
|
||||
// Hold a real pre-mutation snapshot past the assertions so the regression
|
||||
// cannot disappear merely because a slow test outlasted the cache TTL.
|
||||
freezeListing := func() {
|
||||
t.Helper()
|
||||
buckets, err := bed.objLayer.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listBucketsCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) ([]BucketInfo, error) {
|
||||
return buckets, nil
|
||||
})
|
||||
if _, err := listBucketsCache.GetWithCtx(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const bucket = "account-info-freshness"
|
||||
assertPresent := func(want bool) {
|
||||
t.Helper()
|
||||
req, err := buildAdminRequest(nil, http.MethodGet, "/accountinfo", 0, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
bed.router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("account info: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var info madmin.AccountInfo
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &info); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range info.Buckets {
|
||||
if item.Name == bucket {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found != want {
|
||||
t.Errorf("account info bucket present = %v, want %v", found, want)
|
||||
}
|
||||
}
|
||||
|
||||
freezeListing()
|
||||
if err := bed.objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertPresent(true)
|
||||
freezeListing()
|
||||
if err := bed.objLayer.DeleteBucket(ctx, bucket, DeleteBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertPresent(false)
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
)
|
||||
|
||||
func corsAdminRequest(t *testing.T, cred auth.Credentials, method, path string, body []byte) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
router := mux.NewRouter()
|
||||
registerAdminRouter(router, true)
|
||||
req, err := newTestSignedRequestV4(method, adminPathPrefix+adminAPIVersionPrefix+path,
|
||||
int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("admin %s: %d: %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func corsImportReport(t *testing.T, rec *httptest.ResponseRecorder) madmin.BucketMetaImportErrs {
|
||||
t.Helper()
|
||||
var rpt madmin.BucketMetaImportErrs
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rpt); err != nil {
|
||||
t.Fatalf("import report %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
return rpt
|
||||
}
|
||||
|
||||
func corsZip(t *testing.T, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, data := range entries {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// corsCorruptedZip builds an archive holding a stored (uncompressed) cors.xml
|
||||
// whose payload is altered after the checksum is computed, plus the given
|
||||
// companion entries. The altered document stays well formed, so only the zip
|
||||
// checksum tells the two apart.
|
||||
func corsCorruptedZip(t *testing.T, name string, doc []byte, others map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for other, data := range others {
|
||||
ow, err := zw.Create(other)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = ow.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := buf.Bytes()
|
||||
at := bytes.Index(raw, []byte("app.example.com"))
|
||||
if at < 0 {
|
||||
t.Fatalf("stored CORS payload not found in archive")
|
||||
}
|
||||
raw[at] = 'A'
|
||||
return raw
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSRoundTrip covers the export/import round trip for
|
||||
// per-bucket CORS, per-file error reporting for an invalid document, and that
|
||||
// an archive without cors.xml leaves an existing configuration alone.
|
||||
func TestAdminBucketMetadataCORSRoundTrip(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, corsXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Export must carry the stored document verbatim.
|
||||
rec := corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
archive := rec.Body.Bytes()
|
||||
zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var exported []byte
|
||||
for _, f := range zr.File {
|
||||
if f.Name != bucket+"/"+bucketCorsConfig {
|
||||
continue
|
||||
}
|
||||
r, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exported, err = io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(exported, corsXML) {
|
||||
t.Fatalf("%s: exported CORS = %q, want %q", instanceType, exported, corsXML)
|
||||
}
|
||||
|
||||
// Drop the configuration: the archive must then omit the entry.
|
||||
if _, err = updateLocalBucketCORSMetadata(t.Context(), obj, bucket, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err == nil {
|
||||
t.Fatalf("%s: CORS still present before restore", instanceType)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
empty := rec.Body.Bytes()
|
||||
zr, err = zip.NewReader(bytes.NewReader(empty), int64(len(empty)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range zr.File {
|
||||
if f.Name == bucket+"/"+bucketCorsConfig {
|
||||
t.Fatalf("%s: export emitted %s for a bucket without CORS", instanceType, f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", archive)
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: restored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
created, err := globalBucketMetadataSys.CreatedAt(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !storedAt.After(created) {
|
||||
t.Fatalf("%s: restored CORS timestamp %v is not after bucket creation %v", instanceType, storedAt, created)
|
||||
}
|
||||
|
||||
// An archive without cors.xml must not remove the configuration.
|
||||
corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/quota.json": []byte(`{"quota":0}`)}))
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: import without cors.xml changed CORS: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A bucket the import itself creates must still land above its own
|
||||
// creation time, otherwise CORS replication would drop the restore.
|
||||
fresh := "cors-import-created-bucket"
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{fresh + "/" + bucketCorsConfig: corsXML}))
|
||||
if st := corsImportReport(t, rec).Buckets[fresh]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: fresh bucket import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
freshStored, freshAt, err := globalBucketMetadataSys.GetCorsConfigXML(fresh)
|
||||
if err != nil || !bytes.Equal(freshStored, corsXML) {
|
||||
t.Fatalf("%s: fresh bucket CORS = %q, err = %v", instanceType, freshStored, err)
|
||||
}
|
||||
freshCreated, err := globalBucketMetadataSys.CreatedAt(fresh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !freshAt.After(freshCreated) {
|
||||
t.Fatalf("%s: fresh bucket CORS timestamp %v is not after creation %v", instanceType, freshAt, freshCreated)
|
||||
}
|
||||
|
||||
// An invalid document must fail loudly for that bucket and change nothing.
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/" + bucketCorsConfig: []byte("<CORSConfiguration><CORSRule>")}))
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; st.Cors.Err == "" {
|
||||
t.Fatalf("%s: invalid CORS import reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: invalid CORS import changed stored config: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A well formed document carried by a corrupt zip entry must be
|
||||
// rejected too, leaving the stored document and its timestamp alone
|
||||
// while the other configs in the same archive still apply.
|
||||
_, corsAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsCorruptedZip(t, bucket+"/"+bucketCorsConfig, corsXML,
|
||||
map[string][]byte{bucket + "/quota.json": []byte(`{"quota":4096,"quotatype":"hard"}`)}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if st.Cors.Err == "" {
|
||||
t.Fatalf("%s: corrupt CORS entry reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if !st.Quota.IsSet || st.Quota.Err != "" {
|
||||
t.Fatalf("%s: corrupt CORS entry blocked the neighboring quota: %+v", instanceType, st.Quota)
|
||||
}
|
||||
stored, storedAt, err = globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) || !storedAt.Equal(corsAt) {
|
||||
t.Fatalf("%s: corrupt CORS entry changed stored config: %q at %v (was %v), err = %v", instanceType, stored, storedAt, corsAt, err)
|
||||
}
|
||||
quota, _, err := globalBucketMetadataSys.GetQuotaConfig(t.Context(), bucket)
|
||||
if err != nil || quota == nil || quota.Quota != 4096 {
|
||||
t.Fatalf("%s: neighboring quota not applied: %+v, err = %v", instanceType, quota, err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
|
||||
// corsPeerStub is a stand-in site-replication peer. It records every
|
||||
// SRBucketMeta it is asked to apply and answers with status.
|
||||
func corsPeerStub(t *testing.T, applied chan<- madmin.SRBucketMeta, status int) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut && applied != nil {
|
||||
var item madmin.SRBucketMeta
|
||||
if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
|
||||
t.Errorf("decode peer apply: %v", err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
applied <- item
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure pins that an
|
||||
// imported CORS document reaches the reachable peers even when the shared
|
||||
// bucket metadata hook failed against an unreachable one, and that both
|
||||
// failures are still reported for the bucket.
|
||||
func TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
ctx := t.Context()
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
|
||||
healthyApplies := make(chan madmin.SRBucketMeta, 4)
|
||||
healthy := corsPeerStub(t, healthyApplies, http.StatusOK)
|
||||
defer healthy.Close()
|
||||
broken := corsPeerStub(t, nil, http.StatusBadRequest)
|
||||
defer broken.Close()
|
||||
|
||||
// With site replication on, admin requests resolve their token signing
|
||||
// key through the site replicator account, so it has to exist.
|
||||
serviceCred, err := auth.CreateCredentials(siteReplicatorSvcAcc, "cors-import-service-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = cred.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
globalSiteReplicatorCred.Set(serviceCred.SecretKey)
|
||||
defer globalSiteReplicatorCred.Set("")
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{
|
||||
Name: "cors-import-test",
|
||||
ServiceAccountAccessKey: serviceCred.AccessKey,
|
||||
Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
|
||||
"peer-healthy": {Name: "healthy", DeploymentID: "peer-healthy", Endpoint: healthy.URL},
|
||||
"peer-broken": {Name: "broken", DeploymentID: "peer-broken", Endpoint: broken.URL},
|
||||
},
|
||||
}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
defer func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
|
||||
globalSiteReplicationSys.Unlock()
|
||||
}()
|
||||
|
||||
rec := corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{
|
||||
bucket + "/" + bucketCorsConfig: corsXML,
|
||||
bucket + "/quota.json": []byte(`{"quota":8192,"quotatype":"hard"}`),
|
||||
}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: stored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// The reachable peer must have been told about the CORS document,
|
||||
// carrying exactly the timestamp that was saved locally.
|
||||
var corsSeen, sharedSeen bool
|
||||
for range 2 {
|
||||
select {
|
||||
case item := <-healthyApplies:
|
||||
if item.Type != madmin.SRBucketMetaTypeCorsConfig {
|
||||
sharedSeen = item.Bucket == bucket && item.Quota != nil
|
||||
continue
|
||||
}
|
||||
if item.Bucket != bucket || item.Cors == nil || !item.UpdatedAt.Equal(storedAt) {
|
||||
t.Fatalf("%s: peer CORS event = %#v, want %s at %v", instanceType, item, bucket, storedAt)
|
||||
}
|
||||
payload, decErr := base64.StdEncoding.Strict().DecodeString(*item.Cors)
|
||||
if decErr != nil || !bytes.Equal(payload, corsXML) {
|
||||
t.Fatalf("%s: peer CORS payload = %q, err = %v", instanceType, payload, decErr)
|
||||
}
|
||||
corsSeen = true
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("%s: healthy peer received no further events (shared=%v cors=%v)", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
}
|
||||
if !sharedSeen || !corsSeen {
|
||||
t.Fatalf("%s: healthy peer events shared=%v cors=%v, want both", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
|
||||
// Both hook failures against the unreachable peer stay reported.
|
||||
if got := strings.Count(st.Err, "->broken:"); got != 2 {
|
||||
t.Fatalf("%s: bucket error mentions the broken peer %d times, want 2: %q", instanceType, got, st.Err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
+205
-19
@@ -41,7 +41,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -76,7 +76,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
|
||||
quotaConfig, err := parseBucketQuota(bucket, data)
|
||||
_, err = parseBucketQuota(bucket, data)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -94,9 +94,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
Quota: data,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
|
||||
bucketMeta.Quota = nil
|
||||
}
|
||||
|
||||
// Call site replication hook.
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
|
||||
@@ -417,6 +414,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketLifecycleConfig,
|
||||
bucketSSEConfig,
|
||||
bucketTaggingConfig,
|
||||
bucketCorsConfig,
|
||||
bucketQuotaConfigFile,
|
||||
objectLockConfig,
|
||||
bucketVersioningConfig,
|
||||
@@ -437,7 +435,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
configData, err := json.Marshal(config)
|
||||
configData, err := canonicalBucketPolicy(config)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
@@ -517,6 +515,19 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketCorsConfig:
|
||||
// Export the stored document verbatim: GetBucketCors returns
|
||||
// the bytes exactly as they were PUT, so the archive must
|
||||
// round-trip them unchanged.
|
||||
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
continue
|
||||
}
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case objectLockConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -589,6 +600,50 @@ type importMetaReport struct {
|
||||
madmin.BucketMetaImportErrs
|
||||
}
|
||||
|
||||
type importMetadataFields map[string]struct{}
|
||||
|
||||
func (f importMetadataFields) add(configFile string) {
|
||||
f[configFile] = struct{}{}
|
||||
}
|
||||
|
||||
func applyImportedBucketMetadata(dst *BucketMetadata, src BucketMetadata, fields importMetadataFields) {
|
||||
for configFile := range fields {
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
dst.PolicyConfigJSON = bytes.Clone(src.PolicyConfigJSON)
|
||||
dst.PolicyConfigUpdatedAt = src.PolicyConfigUpdatedAt
|
||||
case bucketNotificationConfig:
|
||||
dst.NotificationConfigXML = bytes.Clone(src.NotificationConfigXML)
|
||||
dst.NotificationConfigUpdatedAt = src.NotificationConfigUpdatedAt
|
||||
case bucketLifecycleConfig:
|
||||
dst.LifecycleConfigXML = bytes.Clone(src.LifecycleConfigXML)
|
||||
dst.LifecycleConfigUpdatedAt = src.LifecycleConfigUpdatedAt
|
||||
case bucketSSEConfig:
|
||||
dst.EncryptionConfigXML = bytes.Clone(src.EncryptionConfigXML)
|
||||
dst.EncryptionConfigUpdatedAt = src.EncryptionConfigUpdatedAt
|
||||
case bucketTaggingConfig:
|
||||
dst.TaggingConfigXML = bytes.Clone(src.TaggingConfigXML)
|
||||
dst.TaggingConfigUpdatedAt = src.TaggingConfigUpdatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
dst.QuotaConfigJSON = bytes.Clone(src.QuotaConfigJSON)
|
||||
dst.QuotaConfigUpdatedAt = src.QuotaConfigUpdatedAt
|
||||
case bucketCorsConfig:
|
||||
// The import stamps its fields before creating any missing bucket,
|
||||
// and a CORS event stamped before bucket creation is discarded as
|
||||
// belonging to an older incarnation, so the imported document takes
|
||||
// the same monotonic timestamp a local PutBucketCors would assign.
|
||||
dst.CorsConfigUpdatedAt = localCORSUpdatedAt(*dst, src.CorsConfigUpdatedAt)
|
||||
dst.CorsConfigXML = bytes.Clone(src.CorsConfigXML)
|
||||
case objectLockConfig:
|
||||
dst.ObjectLockConfigXML = bytes.Clone(src.ObjectLockConfigXML)
|
||||
dst.ObjectLockConfigUpdatedAt = src.ObjectLockConfigUpdatedAt
|
||||
case bucketVersioningConfig:
|
||||
dst.VersioningConfigXML = bytes.Clone(src.VersioningConfigXML)
|
||||
dst.VersioningConfigUpdatedAt = src.VersioningConfigUpdatedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st := i.Buckets[bucket]
|
||||
var errMsg string
|
||||
@@ -608,6 +663,8 @@ func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketQuotaConfigFile:
|
||||
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketCorsConfig:
|
||||
st.Cors = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case objectLockConfig:
|
||||
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketVersioningConfig:
|
||||
@@ -649,6 +706,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
||||
importedFields := make(map[string]importMetadataFields, len(zr.File))
|
||||
blockedBuckets := make(map[string]struct{})
|
||||
markImported := func(bucket, configFile string) {
|
||||
fields := importedFields[bucket]
|
||||
if fields == nil {
|
||||
fields = make(importMetadataFields)
|
||||
importedFields[bucket] = fields
|
||||
}
|
||||
fields.add(configFile)
|
||||
}
|
||||
|
||||
updatedAt := UTCNow()
|
||||
|
||||
@@ -664,6 +731,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketMap[bucket] = &meta
|
||||
} else if err != errConfigNotFound {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
blockedBuckets[bucket] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -675,6 +743,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == objectLockConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -708,6 +779,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].ObjectLockConfigXML = configData
|
||||
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -720,6 +792,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == bucketVersioningConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -764,6 +839,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].VersioningConfigXML = configData
|
||||
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -781,6 +857,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
|
||||
// create bucket if it does not exist yet.
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
@@ -813,6 +892,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].NotificationConfigXML = configData
|
||||
bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketPolicyConfig:
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
@@ -839,7 +919,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(bucketPolicy)
|
||||
configData, err := canonicalBucketPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
@@ -847,6 +927,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].PolicyConfigJSON = configData
|
||||
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketLifecycleConfig:
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||
@@ -879,6 +960,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].LifecycleConfigXML = configData
|
||||
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketSSEConfig:
|
||||
// Parse bucket encryption xml
|
||||
@@ -917,6 +999,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].EncryptionConfigXML = configData
|
||||
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketTaggingConfig:
|
||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||
@@ -933,6 +1016,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].TaggingConfigXML = configData
|
||||
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketQuotaConfigFile:
|
||||
data, err := io.ReadAll(reader)
|
||||
@@ -949,6 +1033,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].QuotaConfigJSON = data
|
||||
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketCorsConfig:
|
||||
if sz > maxBucketCorsSize {
|
||||
rpt.SetStatus(bucket, fileName, errors.New(ErrEntityTooLarge.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
// Read one byte past the declared size: stopping exactly at sz
|
||||
// leaves archive/zip short of EOF, so it never verifies the entry
|
||||
// checksum and a corrupt entry carrying well formed XML would be
|
||||
// stored as a valid document. The extra byte also lets the reader
|
||||
// reject an entry longer than it declares.
|
||||
corsData, err := io.ReadAll(io.LimitReader(reader, sz+1))
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err = validateCORSReplicationPayload(corsData); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
bucketMap[bucket].CorsConfigXML = corsData
|
||||
bucketMap[bucket].CorsConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -962,22 +1073,97 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
for bucket, meta := range bucketMap {
|
||||
err := globalBucketMetadataSys.save(ctx, *meta)
|
||||
fields := importedFields[bucket]
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
var merged BucketMetadata
|
||||
var commitAt time.Time
|
||||
err := func() error {
|
||||
lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureBucketMetadataCreated(lockCtx, objectAPI, &merged); err != nil {
|
||||
return err
|
||||
}
|
||||
commitAt = UTCNow()
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
if _, ok := fields[file]; ok {
|
||||
commitAt = localBucketConfigUpdatedAt(merged, file, commitAt)
|
||||
}
|
||||
}
|
||||
applyImportedBucketMetadata(&merged, *meta, fields)
|
||||
for _, file := range replicatedBucketConfigs {
|
||||
if _, ok := fields[file]; !ok {
|
||||
continue
|
||||
}
|
||||
data, at := replicatedBucketConfig(&merged, file)
|
||||
payload, _, err := bucketConfigPayload(bucket, file, *data, len(merged.ObjectLockConfigXML) != 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*data, *at = payload, commitAt
|
||||
}
|
||||
return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, &merged)
|
||||
}()
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket,
|
||||
Quota: meta.QuotaConfigJSON,
|
||||
Policy: meta.PolicyConfigJSON,
|
||||
Versioning: enc(meta.VersioningConfigXML),
|
||||
Tags: enc(meta.TaggingConfigXML),
|
||||
ObjectLockConfig: enc(meta.ObjectLockConfigXML),
|
||||
SSEConfig: enc(meta.EncryptionConfigXML),
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
*meta = merged
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
|
||||
hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: commitAt}
|
||||
var hookNeeded bool
|
||||
if _, ok := fields[bucketQuotaConfigFile]; ok {
|
||||
hook.Quota = meta.QuotaConfigJSON
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok {
|
||||
hook.Policy = meta.PolicyConfigJSON
|
||||
hookNeeded = hookNeeded || len(hook.Policy) != 0
|
||||
}
|
||||
if _, ok := fields[bucketVersioningConfig]; ok {
|
||||
hook.Versioning = enc(meta.VersioningConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketTaggingConfig]; ok {
|
||||
hook.Tags = enc(meta.TaggingConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[objectLockConfig]; ok {
|
||||
hook.ObjectLockConfig = enc(meta.ObjectLockConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketSSEConfig]; ok {
|
||||
hook.SSEConfig = enc(meta.EncryptionConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if hookNeeded {
|
||||
err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok && len(meta.PolicyConfigJSON) == 0 {
|
||||
// An omitted bulk Policy cannot express deletion.
|
||||
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypePolicy, Bucket: bucket, UpdatedAt: commitAt,
|
||||
}))
|
||||
}
|
||||
if _, ok := fields[bucketCorsConfig]; ok {
|
||||
// CORS carries its own timestamp, so it replicates through the
|
||||
// dedicated event rather than the shared bucket metadata hook. It
|
||||
// is announced even when the shared hook failed: the document is
|
||||
// already committed locally, and a peer that is unreachable for
|
||||
// one config must not withhold CORS from the reachable ones.
|
||||
if corsEvent, live := newBucketCORSReplicationEvent(bucket, *meta); live {
|
||||
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, corsEvent))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// validateAdminReq will validate request against and return whether it is allowed.
|
||||
|
||||
@@ -37,7 +37,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
||||
|
||||
@@ -32,8 +32,8 @@ import (
|
||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||
|
||||
@@ -28,8 +28,8 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||
@@ -249,8 +249,10 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
|
||||
|
||||
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||
// it is associated with the LDAP/OIDC user properly.
|
||||
// The new credential's policy comes from the request, not the caller.
|
||||
for k, v := range cred.Claims {
|
||||
if k == expClaim {
|
||||
switch k {
|
||||
case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
|
||||
continue
|
||||
}
|
||||
opts.claims[k] = v
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const dummyRoleARN = "dummy-internal"
|
||||
|
||||
@@ -27,8 +27,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -373,7 +373,9 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
|
||||
// Cancel any ongoing rebalance operation
|
||||
globalNotificationSys.StopRebalance(r.Context())
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
|
||||
if err := pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt); !errors.Is(err, errRebalanceNotStarted) {
|
||||
adminLogIf(ctx, err)
|
||||
}
|
||||
globalNotificationSys.LoadRebalanceMeta(ctx, false)
|
||||
}
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
||||
@@ -255,9 +255,11 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
case madmin.SRBucketMetaTypeTags:
|
||||
err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeObjectLockConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
|
||||
err = globalSiteReplicationSys.peerBucketObjectLockConfigItem(ctx, item)
|
||||
case madmin.SRBucketMetaTypeSSEConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeCorsConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, item.Bucket, item.Cors, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaLCConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,556 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func iamAdminFixture(t *testing.T) (context.Context, *IAMSys) {
|
||||
t.Helper()
|
||||
ctx, sys, obj := prepareIAMRevisionFixture(t)
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = obj
|
||||
globalObjLayerMutex.Unlock()
|
||||
return ctx, sys
|
||||
}
|
||||
|
||||
func iamAdminUser(ctx context.Context, t *testing.T, sys *IAMSys, name, policyName string) auth.Credentials {
|
||||
t.Helper()
|
||||
cred := auth.Credentials{AccessKey: name, SecretKey: "review-user-valid-secret"}
|
||||
_, err := sys.CreateUser(ctx, name, madmin.AddOrUpdateUserReq{SecretKey: cred.SecretKey, Status: madmin.AccountEnabled})
|
||||
mustIAM(t, err)
|
||||
if policyName != "" {
|
||||
_, err = sys.PolicyDBSet(ctx, name, policyName, regUser, false)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
func iamAdminServiceAccount(ctx context.Context, t *testing.T, sys *IAMSys, parent, key, document string) auth.Credentials {
|
||||
t.Helper()
|
||||
var sp *policy.Policy
|
||||
var err error
|
||||
if document != "" {
|
||||
sp, err = policy.ParseConfig(strings.NewReader(document))
|
||||
mustIAM(t, err)
|
||||
}
|
||||
cred, _, err := sys.NewServiceAccount(ctx, parent, nil, newServiceAccountOpts{accessKey: key, secretKey: "review-service-valid-secret", sessionPolicy: sp})
|
||||
mustIAM(t, err)
|
||||
return cred
|
||||
}
|
||||
|
||||
func iamAdminRequest(t *testing.T, cred auth.Credentials, method, target string, payload any) *http.Request {
|
||||
t.Helper()
|
||||
var body []byte
|
||||
if payload != nil {
|
||||
raw, err := json.Marshal(payload)
|
||||
mustIAM(t, err)
|
||||
body, err = madmin.EncryptData(cred.SecretKey, raw)
|
||||
mustIAM(t, err)
|
||||
}
|
||||
headers := map[string]string{}
|
||||
if cred.IsTemp() && !cred.IsServiceAccount() {
|
||||
headers["X-Amz-Security-Token"] = cred.SessionToken
|
||||
}
|
||||
req, err := newTestSignedRequestV4(method, "http://localhost"+target, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
|
||||
mustIAM(t, err)
|
||||
mustIAM(t, req.ParseForm())
|
||||
return req
|
||||
}
|
||||
|
||||
func iamAdminCall(t *testing.T, cred auth.Credentials, method, target string, payload any, vars map[string]string, handler http.HandlerFunc) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := iamAdminRequest(t, cred, method, target, payload)
|
||||
req = mux.SetURLVars(req, vars)
|
||||
rec := httptest.NewRecorder()
|
||||
req = req.WithContext(newContext(req, rec, "IAMAdminAuthorization"))
|
||||
handler(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func assertIAMAdminStatus(t *testing.T, operation string, rec *httptest.ResponseRecorder, want int) {
|
||||
t.Helper()
|
||||
if rec.Code != want {
|
||||
t.Errorf("%s status=%d want=%d body=%s", operation, rec.Code, want, rec.Body.String())
|
||||
return
|
||||
}
|
||||
if want == http.StatusForbidden || want == http.StatusNotFound {
|
||||
var response APIErrorResponse
|
||||
mustIAM(t, json.Unmarshal(rec.Body.Bytes(), &response))
|
||||
wantCode := errorCodes.ToAPIErr(ErrAccessDenied).Code
|
||||
if want == http.StatusNotFound {
|
||||
wantCode = errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound).Code
|
||||
}
|
||||
if response.Code != wantCode {
|
||||
t.Errorf("%s error code=%s want=%s", operation, response.Code, wantCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func iamAdminPlugin(t *testing.T, allow bool) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"result": allow})
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
u, err := xnet.ParseHTTPURL(server.URL)
|
||||
mustIAM(t, err)
|
||||
previous := newGlobalAuthZPluginFn()
|
||||
setGlobalAuthZPlugin(polplugin.New(polplugin.Args{URL: u, CloseRespFn: func(r io.ReadCloser) { r.Close() }}))
|
||||
t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
|
||||
}
|
||||
|
||||
func TestIAMAdminRootEmptyPolicy(t *testing.T) {
|
||||
for _, doc := range []string{"", "null", "{}", `{"Statement":null}`} {
|
||||
t.Run("policy="+doc, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-empty", doc)
|
||||
sibling := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "review-root-sibling", "")
|
||||
req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
checked, owner, code := checkKeyValid(req, caller.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("checkKeyValid: %v", code)
|
||||
}
|
||||
active, _ := isAllowedBySessionPolicyForServiceAccount(policy.Args{Claims: checked.Claims})
|
||||
t.Logf("owner=%v parsed_active_session_policy=%v", owner, active)
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-root-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 200)
|
||||
assertIAMAdminStatus(t, "update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewName: "review"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+sibling.AccessKey, nil, map[string]string{"accessKey": sibling.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminEmptyPolicyDelegation(t *testing.T) {
|
||||
for _, parentType := range []string{"root", "user"} {
|
||||
for _, document := range []string{"null", "{}", `{"Statement":null}`} {
|
||||
t.Run(parentType+"/"+document, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := globalActiveCred
|
||||
if parentType == "user" {
|
||||
parent = iamAdminUser(ctx, t, sys, "empty-policy-parent", "readwrite")
|
||||
}
|
||||
sp, err := policy.ParseConfig(strings.NewReader(document))
|
||||
mustIAM(t, err)
|
||||
caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "empty-policy-caller", secretKey: "empty-policy-caller-secret", sessionPolicy: sp,
|
||||
claims: map[string]any{"identity-attribute": "preserved"},
|
||||
})
|
||||
mustIAM(t, err)
|
||||
const limitedPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:ListServiceAccounts"]}]}`
|
||||
for _, tc := range []struct {
|
||||
policy string
|
||||
listStatus int
|
||||
allowObject bool
|
||||
}{
|
||||
{"", http.StatusOK, true},
|
||||
{"null", http.StatusOK, true},
|
||||
{"{}", http.StatusOK, true},
|
||||
{`{"Statement":null}`, http.StatusOK, true},
|
||||
{limitedPolicy, http.StatusOK, false},
|
||||
{`{"Version":"2012-10-17"}`, http.StatusForbidden, false},
|
||||
{`{"Statement":[]}`, http.StatusForbidden, false},
|
||||
} {
|
||||
expires := UTCNow().Add(time.Hour)
|
||||
req := madmin.AddServiceAccountReq{Expiration: &expires}
|
||||
if tc.policy != "" {
|
||||
req.Policy = json.RawMessage(tc.policy)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", req, nil, api.AddServiceAccount)
|
||||
assertIAMAdminStatus(t, "create expiring child", rec, http.StatusOK)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var response madmin.AddServiceAccountResp
|
||||
mustIAM(t, json.Unmarshal(data, &response))
|
||||
child, ok := sys.store.GetUser(response.Credentials.AccessKey)
|
||||
if !ok {
|
||||
t.Fatal("new service account missing")
|
||||
}
|
||||
// Exercise the issued key through authentication and authorization.
|
||||
assertIAMAdminStatus(t, "use expiring child policy="+tc.policy, iamAdminCall(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), tc.listStatus)
|
||||
signed := iamAdminRequest(t, child.Credentials, "GET", "/example/object", nil)
|
||||
checked, owner, code := checkKeyValid(signed, child.Credentials.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("child authentication: %v", code)
|
||||
}
|
||||
if checked.Claims["identity-attribute"] != "preserved" {
|
||||
t.Fatal("creation lost the caller's identity context")
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: checked.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.GetObjectAction, BucketName: "example", ObjectName: "object"})
|
||||
if allowed != tc.allowObject {
|
||||
t.Errorf("GetObject allowed=%v with requested policy=%s", allowed, tc.policy)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminCorruptServiceCleanup(t *testing.T) {
|
||||
for _, status := range []string{auth.AccountOn, auth.AccountOff, "expired"} {
|
||||
t.Run(status, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "review-admin", "consoleAdmin")
|
||||
parent := iamAdminUser(ctx, t, sys, "review-parent", "readwrite")
|
||||
svc := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-corrupt", "")
|
||||
if status == auth.AccountOff {
|
||||
_, err := sys.UpdateServiceAccount(ctx, svc.AccessKey, updateServiceAccountOpts{status: status})
|
||||
mustIAM(t, err)
|
||||
}
|
||||
cache := sys.store.lock()
|
||||
u := cache.iamUsersMap[svc.AccessKey]
|
||||
u.Credentials.SessionToken = "broken-session-token"
|
||||
if status == "expired" {
|
||||
u.Credentials.Expiration = UTCNow().Add(-time.Minute)
|
||||
}
|
||||
cache.iamUsersMap[svc.AccessKey] = u
|
||||
sys.store.unlock()
|
||||
_, _, err := sys.GetServiceAccount(ctx, svc.AccessKey)
|
||||
if err == nil {
|
||||
t.Fatal("fault injection failed")
|
||||
}
|
||||
raw, exists := sys.store.GetUser(svc.AccessKey)
|
||||
_, loginOK := sys.GetUser(ctx, svc.AccessKey)
|
||||
t.Logf("metadata_exists=%v parent_known=%v GetUser_ok=%v", exists, raw.Credentials.ParentUser == parent.AccessKey, loginOK)
|
||||
api := adminAPIHandlers{}
|
||||
outsider := iamAdminUser(ctx, t, sys, "cleanup-outsider", "readwrite")
|
||||
assertIAMAdminStatus(t, "unauthorized cleanup", iamAdminCall(t, outsider, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "admin cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
if _, ok := sys.store.GetUser(svc.AccessKey); ok {
|
||||
t.Fatal("deleted service account remains in the IAM cache")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminPolicylessSelf(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "review-policyless", "")
|
||||
svc := iamAdminServiceAccount(ctx, t, sys, caller.AccessKey, "rv-policyless-child", "")
|
||||
other := iamAdminUser(ctx, t, sys, "bulk-other", "readwrite")
|
||||
_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "bulk-other-child", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 200)
|
||||
rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk)
|
||||
assertIAMAdminStatus(t, "list bulk self", rec, 200)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var listed map[string]madmin.ListAccessKeysResp
|
||||
mustIAM(t, json.Unmarshal(data, &listed))
|
||||
entry, ok := listed[caller.AccessKey]
|
||||
if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 1 || entry.ServiceAccounts[0].AccessKey != svc.AccessKey || len(entry.STSKeys) != 0 {
|
||||
t.Fatalf("policyless bulk self returned unexpected identities: %+v", listed)
|
||||
}
|
||||
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoServiceAccount), 200)
|
||||
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoAccessKey), 200)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 204)
|
||||
assertIAMAdminStatus(t, "list another user", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+globalActiveCred.AccessKey, nil, nil, api.ListServiceAccounts), 403)
|
||||
assertIAMAdminStatus(t, "revoke another user", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
}
|
||||
|
||||
func TestIAMAdminExpiringInheritedPolicy(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "review-legacy-parent", "consoleAdmin")
|
||||
document := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
|
||||
// Model a credential persisted by the old handler: it copied its caller's
|
||||
// policy claims but set the newly created service's type to inherited.
|
||||
expires := UTCNow().Add(time.Hour)
|
||||
caller, _, err := sys.NewServiceAccount(ctx, parent.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "rv-exp-inherited", secretKey: "review-service-valid-secret", expiration: &expires,
|
||||
claims: map[string]any{policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(document))},
|
||||
})
|
||||
mustIAM(t, err)
|
||||
req := iamAdminRequest(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
checked, owner, code := checkKeyValid(req, caller.AccessKey)
|
||||
if code != ErrNone {
|
||||
t.Fatalf("checkKeyValid: %v", code)
|
||||
}
|
||||
args := policy.Args{AccountName: caller.AccessKey, Claims: checked.Claims, IsOwner: owner, Action: policy.CreateServiceAccountAdminAction, DenyOnly: true}
|
||||
if !sys.IsAllowed(args) {
|
||||
t.Fatal("fixture cannot create")
|
||||
}
|
||||
args.Action = policy.CreateUserAdminAction
|
||||
args.DenyOnly = false
|
||||
if sys.IsAllowed(args) {
|
||||
t.Fatal("fixture not constrained")
|
||||
}
|
||||
t.Logf("service=%v temp=%v inherited=%v; create allowed, create-user denied", checked.IsServiceAccount(), checked.IsTemp(), checked.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType)
|
||||
api := adminAPIHandlers{}
|
||||
rec := iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-escaped", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount)
|
||||
if rec.Code == 200 {
|
||||
child, ok := sys.GetUser(ctx, "review-escaped")
|
||||
if !ok {
|
||||
t.Fatal("new key missing")
|
||||
}
|
||||
childReq := iamAdminRequest(t, child.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
childChecked, childOwner, childCode := checkKeyValid(childReq, child.Credentials.AccessKey)
|
||||
if childCode != ErrNone {
|
||||
t.Fatalf("child auth=%v", childCode)
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: childChecked.AccessKey, Claims: childChecked.Claims, IsOwner: childOwner, Action: policy.CreateUserAdminAction})
|
||||
t.Errorf("constrained inherited service created a new key; new key CreateUser allowed=%v", allowed)
|
||||
} else {
|
||||
assertIAMAdminStatus(t, "reject derived creation", rec, 403)
|
||||
}
|
||||
updated := iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+caller.AccessKey, madmin.UpdateServiceAccountReq{NewName: "metadata-only"}, map[string]string{"accessKey": caller.AccessKey}, api.UpdateServiceAccount)
|
||||
if updated.Code == 204 {
|
||||
after, ok := sys.GetUser(ctx, caller.AccessKey)
|
||||
if !ok {
|
||||
t.Fatal("updated key missing")
|
||||
}
|
||||
afterReq := iamAdminRequest(t, after.Credentials, "GET", "/minio/admin/v3/list-service-accounts", nil)
|
||||
afterChecked, afterOwner, afterCode := checkKeyValid(afterReq, caller.AccessKey)
|
||||
if afterCode != ErrNone {
|
||||
t.Fatalf("updated auth=%v", afterCode)
|
||||
}
|
||||
allowed := sys.IsAllowed(policy.Args{AccountName: caller.AccessKey, Claims: afterChecked.Claims, IsOwner: afterOwner, Action: policy.CreateUserAdminAction})
|
||||
t.Errorf("constrained inherited service updated itself; CreateUser allowed after update=%v", allowed)
|
||||
} else {
|
||||
assertIAMAdminStatus(t, "reject inherited self update", updated, 403)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminSessionPolicyCannotDelegate(t *testing.T) {
|
||||
for _, temp := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "service", true: "sts"}[temp], func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "rv-parent", "consoleAdmin")
|
||||
doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:CreateServiceAccount","admin:UpdateServiceAccount"]}]}`
|
||||
var caller auth.Credentials
|
||||
if !temp {
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "review-restricted", doc)
|
||||
} else {
|
||||
var err error
|
||||
caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(doc))}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
caller.ParentUser = parent.AccessKey
|
||||
_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
|
||||
mustIAM(t, err)
|
||||
}
|
||||
sibling := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-sibling", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "restrict create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "restrict LDAP create", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/idp/ldap/add-service-account", madmin.AddServiceAccountReq{AccessKey: "review-bad-new", SecretKey: "review-service-valid-secret"}, nil, api.AddServiceAccountLDAP), 403)
|
||||
assertIAMAdminStatus(t, "restrict update", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+sibling.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": sibling.AccessKey}, api.UpdateServiceAccount), 403)
|
||||
if _, ok := sys.store.GetUser("review-bad-new"); ok {
|
||||
t.Fatal("denied creation persisted a credential")
|
||||
}
|
||||
u, ok := sys.store.GetUser(sibling.AccessKey)
|
||||
if !ok || u.Credentials.SecretKey != sibling.SecretKey {
|
||||
t.Fatal("denied update changed the target credential")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminSelfDenials(t *testing.T) {
|
||||
for _, mode := range []string{"parent_deny", "group_deny", "session_implicit_deny", "session_explicit_deny", "plugin_deny"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "rv-self-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-target", "")
|
||||
caller := parent
|
||||
doc := `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["admin:ListServiceAccounts","admin:RemoveServiceAccount"]}]}`
|
||||
switch mode {
|
||||
case "parent_deny", "group_deny":
|
||||
p, err := policy.ParseConfig(strings.NewReader(doc))
|
||||
mustIAM(t, err)
|
||||
_, err = sys.SetPolicy(ctx, "review-self-deny", *p)
|
||||
mustIAM(t, err)
|
||||
if mode == "group_deny" {
|
||||
_, err = sys.AddUsersToGroup(ctx, "self-deny-group", []string{parent.AccessKey})
|
||||
mustIAM(t, err)
|
||||
_, err = sys.PolicyDBSet(ctx, "self-deny-group", "review-self-deny", regUser, true)
|
||||
} else {
|
||||
_, err = sys.PolicyDBSet(ctx, parent.AccessKey, "readwrite,review-self-deny", regUser, false)
|
||||
}
|
||||
mustIAM(t, err)
|
||||
case "session_implicit_deny":
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::example/*"]}]}`)
|
||||
case "session_explicit_deny":
|
||||
caller = iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "rv-self-caller", doc)
|
||||
case "plugin_deny":
|
||||
iamAdminPlugin(t, false)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 403)
|
||||
assertIAMAdminStatus(t, "list bulk self", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk), 403)
|
||||
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoAccessKey), 403)
|
||||
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "revoke", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminNonRootCannotManageRoot(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "rv-console-admin", "consoleAdmin")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "rv-root-target", "")
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "create root", iamAdminCall(t, caller, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{AccessKey: "rv-new-root", SecretKey: "review-service-valid-secret", TargetUser: globalActiveCred.AccessKey}, nil, api.AddServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "update root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewSecretKey: "review-known-new-secret"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 403)
|
||||
assertIAMAdminStatus(t, "delete root", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "revoke root", iamAdminCall(t, caller, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true&user="+globalActiveCred.AccessKey, nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
// Restricted root-derived credentials and the site replicator are not root
|
||||
// administrators, even when their effective policy allows these actions.
|
||||
limited := iamAdminServiceAccount(ctx, t, sys, globalActiveCred.AccessKey, "limited-root", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:*"]}]}`)
|
||||
assertIAMAdminStatus(t, "limited root delete", iamAdminCall(t, limited, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 404)
|
||||
assertIAMAdminStatus(t, "limited root revoke", iamAdminCall(t, limited, "POST", "/minio/admin/v3/revoke-tokens/builtin?fullRevoke=true", nil, map[string]string{"userProvider": madmin.BuiltinProvider}, api.RevokeTokens), 403)
|
||||
replicator, _, err := sys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{accessKey: siteReplicatorSvcAcc, secretKey: "replicator-valid-secret", allowSiteReplicatorAccount: true})
|
||||
mustIAM(t, err)
|
||||
assertIAMAdminStatus(t, "replicator create root", iamAdminCall(t, replicator, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 403)
|
||||
|
||||
// An external plugin remains authoritative, including for root targets.
|
||||
iamAdminPlugin(t, true)
|
||||
assertIAMAdminStatus(t, "plugin update root", iamAdminCall(t, limited, "POST", "/minio/admin/v3/update-service-account?accessKey="+target.AccessKey, madmin.UpdateServiceAccountReq{NewName: "plugin-authorized"}, map[string]string{"accessKey": target.AccessKey}, api.UpdateServiceAccount), 204)
|
||||
assertIAMAdminStatus(t, "plugin create root", iamAdminCall(t, limited, "PUT", "/minio/admin/v3/add-service-account", madmin.AddServiceAccountReq{}, nil, api.AddServiceAccount), 200)
|
||||
}
|
||||
|
||||
func TestIAMAdminDeleteCorruptServiceReplicates(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
caller := iamAdminUser(ctx, t, sys, "cleanup-admin", "consoleAdmin")
|
||||
parent := iamAdminUser(ctx, t, sys, "cleanup-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "cleanup-target", "")
|
||||
cache := sys.store.lock()
|
||||
u := cache.iamUsersMap[target.AccessKey]
|
||||
u.Credentials.SessionToken = "broken-session-token"
|
||||
u.Credentials.Status = auth.AccountOff
|
||||
cache.iamUsersMap[target.AccessKey] = u
|
||||
sys.store.unlock()
|
||||
|
||||
var deletions atomic.Int32
|
||||
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/minio/health/live" {
|
||||
return
|
||||
}
|
||||
var batch iamRevisionBatch
|
||||
if err := json.NewDecoder(r.Body).Decode(&batch); err != nil {
|
||||
t.Error(err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if len(batch.Items) != 1 || batch.Items[0].SvcAccChange == nil || batch.Items[0].SvcAccChange.Delete == nil || batch.Items[0].SvcAccChange.Delete.AccessKey != target.AccessKey {
|
||||
t.Error("replication did not carry the service account deletion")
|
||||
} else {
|
||||
deletions.Add(1)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(iamRevisionResponse{iamRevisionStatus: iamRevisionStatus{Version: iamRevisionProtocol, Node: "peer", Instance: "peer-boot", Digest: "deleted"}})
|
||||
}))
|
||||
t.Cleanup(peer.Close)
|
||||
globalSiteReplicatorCred.RLock()
|
||||
previousSecret := globalSiteReplicatorCred.secretKey
|
||||
globalSiteReplicatorCred.RUnlock()
|
||||
globalSiteReplicatorCred.Set(caller.SecretKey)
|
||||
t.Cleanup(func() { globalSiteReplicatorCred.Set(previousSecret) })
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{ServiceAccountAccessKey: caller.AccessKey, Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {DeploymentID: globalDeploymentID(), Name: "local"},
|
||||
"peer": {DeploymentID: "peer", Name: "peer", Endpoint: peer.URL},
|
||||
}}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled = false
|
||||
globalSiteReplicationSys.Unlock()
|
||||
})
|
||||
api := adminAPIHandlers{}
|
||||
assertIAMAdminStatus(t, "replicated cleanup", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 204)
|
||||
if deletions.Load() != 1 {
|
||||
t.Fatalf("expected one immediate deletion notification, got %d", deletions.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIAMAdminListSelfAliases(t *testing.T) {
|
||||
for _, kind := range []string{"service", "sts"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
ctx, sys := iamAdminFixture(t)
|
||||
parent := iamAdminUser(ctx, t, sys, "alias-parent", "readwrite")
|
||||
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-target", "")
|
||||
serviceCaller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "")
|
||||
caller := serviceCaller
|
||||
other := iamAdminUser(ctx, t, sys, "alias-other", "readwrite")
|
||||
_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "alias-other-child", "")
|
||||
if kind == "sts" {
|
||||
var err error
|
||||
caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
caller.ParentUser = parent.AccessKey
|
||||
_, err = sys.SetTempUser(ctx, caller.AccessKey, caller, "")
|
||||
mustIAM(t, err)
|
||||
}
|
||||
api := adminAPIHandlers{}
|
||||
for _, alias := range []string{"", caller.AccessKey, parent.AccessKey} {
|
||||
rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts?user="+alias, nil, nil, api.ListServiceAccounts)
|
||||
assertIAMAdminStatus(t, "list self alias", rec, 200)
|
||||
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
|
||||
mustIAM(t, err)
|
||||
var response madmin.ListServiceAccountsResp
|
||||
mustIAM(t, json.Unmarshal(data, &response))
|
||||
found := false
|
||||
for _, account := range response.Accounts {
|
||||
found = found || account.AccessKey == target.AccessKey
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("self alias %q did not list the parent's service accounts", alias)
|
||||
}
|
||||
|
||||
query := url.Values{"listType": {madmin.AccessKeyListAll}}
|
||||
if alias != "" {
|
||||
query.Set("users", alias)
|
||||
}
|
||||
bulk := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?"+query.Encode(), nil, nil, api.ListAccessKeysBulk)
|
||||
assertIAMAdminStatus(t, "list bulk self alias", bulk, 200)
|
||||
data, err = madmin.DecryptData(caller.SecretKey, bulk.Body)
|
||||
mustIAM(t, err)
|
||||
var listed map[string]madmin.ListAccessKeysResp
|
||||
mustIAM(t, json.Unmarshal(data, &listed))
|
||||
entry, ok := listed[parent.AccessKey]
|
||||
if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 2 {
|
||||
t.Fatalf("bulk self alias %q returned unexpected identities: %+v", alias, listed)
|
||||
}
|
||||
want := map[string]bool{target.AccessKey: true, serviceCaller.AccessKey: true}
|
||||
for _, account := range entry.ServiceAccounts {
|
||||
if !want[account.AccessKey] {
|
||||
t.Fatalf("bulk self alias %q returned unexpected or duplicate service account %s", alias, account.AccessKey)
|
||||
}
|
||||
delete(want, account.AccessKey)
|
||||
}
|
||||
if kind == "sts" {
|
||||
if len(entry.STSKeys) != 1 || entry.STSKeys[0].AccessKey != caller.AccessKey {
|
||||
t.Fatalf("bulk self alias %q returned unexpected STS accounts: %+v", alias, entry.STSKeys)
|
||||
}
|
||||
} else if len(entry.STSKeys) != 0 {
|
||||
t.Fatalf("service caller returned unexpected STS accounts: %+v", entry.STSKeys)
|
||||
}
|
||||
if bytes.Contains(data, []byte(serviceCaller.SecretKey)) || bytes.Contains(data, []byte(`"secretKey"`)) || bytes.Contains(data, []byte(`"sessionToken"`)) {
|
||||
t.Fatal("bulk listing returned credential secrets")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||
|
||||
+211
-108
@@ -40,8 +40,8 @@ import (
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
@@ -355,18 +355,25 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
|
||||
func setGroupStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.GroupStatus(status) == madmin.GroupDisabled {
|
||||
return policy.DisableGroupAdminAction
|
||||
}
|
||||
return policy.EnableGroupAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
group := vars["group"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, setGroupStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
err error
|
||||
updatedAt time.Time
|
||||
@@ -398,18 +405,25 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
|
||||
func setUserStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.AccountStatus(status) == madmin.AccountDisabled {
|
||||
return policy.DisableUserAdminAction
|
||||
}
|
||||
return policy.EnableUserAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, setUserStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// you cannot enable or disable yourself.
|
||||
if accessKey == creds.AccessKey {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
@@ -488,11 +502,15 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
checkDenyOnly := accessKey == cred.AccessKey
|
||||
action := policy.Action(policy.CreateUserAdminAction)
|
||||
if checkDenyOnly {
|
||||
action = policy.ChangeMyPasswordAdminAction
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateUserAdminAction,
|
||||
Action: action,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -604,7 +622,11 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
|
||||
policiesNames = policySet.ToSlice()
|
||||
}
|
||||
|
||||
stsAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
stsAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
policyJSON, err := json.MarshalIndent(stsAccountPolicy, "", " ")
|
||||
@@ -696,8 +718,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||
// it is associated with the LDAP/OIDC user properly.
|
||||
// The new credential's policy comes from the request, not the caller.
|
||||
for k, v := range cred.Claims {
|
||||
if k == expClaim {
|
||||
switch k {
|
||||
case expClaim, policy.SessionPolicyName, sessionPolicyNameExtracted, iamPolicyClaimNameSA():
|
||||
continue
|
||||
}
|
||||
opts.claims[k] = v
|
||||
@@ -837,10 +861,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// Permission checks:
|
||||
//
|
||||
// 1. Any type of account (i.e. access keys (previously/still called service
|
||||
// accounts), STS accounts, internal IDP accounts, etc) with the
|
||||
// policy.UpdateServiceAccountAdminAction permission can update any service
|
||||
// account.
|
||||
// 1. Updating requires policy.UpdateServiceAccountAdminAction. Under built-in
|
||||
// IAM, session-limited callers cannot update credentials, and only unrestricted
|
||||
// root credentials can manage root's service accounts.
|
||||
//
|
||||
// 2. We would like to let a user update their own access keys, however it
|
||||
// is currently blocked pending a re-design. Users are still able to delete
|
||||
@@ -856,10 +879,15 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
|
||||
(svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(updateReq.NewPolicy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -933,23 +961,21 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: requestUser == svcAccount.ParentUser,
|
||||
}) {
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
|
||||
if requestUser != svcAccount.ParentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// if session policy is nil or empty, then it is implied policy
|
||||
@@ -965,7 +991,11 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
svcAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
|
||||
@@ -1021,29 +1051,27 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
var targetAccount string
|
||||
|
||||
// If listing is requested for a specific user (who is not the request
|
||||
// sender), check that the user has permissions.
|
||||
selfUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfUser = cred.ParentUser
|
||||
}
|
||||
user := r.Form.Get("user")
|
||||
if user != "" && user != cred.AccessKey {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
targetAccount = user
|
||||
} else {
|
||||
targetAccount = cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
targetAccount = cred.ParentUser
|
||||
}
|
||||
selfOnly := user == "" || user == cred.AccessKey || user == selfUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: selfOnly,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
targetAccount := user
|
||||
if selfOnly {
|
||||
targetAccount = selfUser
|
||||
}
|
||||
|
||||
serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, targetAccount)
|
||||
@@ -1092,7 +1120,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
if objectAPI == nil || globalNotificationSys == nil || !globalIAMSys.Initialized() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1113,35 +1141,41 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
// We do not care if service account is readable or not at this point,
|
||||
// since this is a delete call we shall allow it to be deleted if possible.
|
||||
svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
|
||||
if errors.Is(err, errNoSuchServiceAccount) {
|
||||
// Deletion needs the stored identity, not a valid token or session policy.
|
||||
// Keep disabled, expired and damaged service accounts removable, and retain
|
||||
// their parent for authorization and the site replication hook below.
|
||||
u, ok := globalIAMSys.store.GetUser(serviceAccount)
|
||||
if !ok || !u.Credentials.IsServiceAccount() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
svcAccount := u.Credentials
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
isSelf := parentUser == svcAccount.ParentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
|
||||
if !adminPrivilege {
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
if svcAccount.ParentUser != "" && parentUser != svcAccount.ParentUser {
|
||||
// The service account belongs to another user but return not
|
||||
// found error to mitigate brute force attacks. or the
|
||||
// serviceAccount doesn't exist.
|
||||
DenyOnly: isSelf,
|
||||
}) {
|
||||
if !isSelf {
|
||||
// Hide the existence of another user's service account.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && svcAccount.ParentUser == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true); err != nil {
|
||||
@@ -1211,7 +1245,7 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
@@ -1224,12 +1258,12 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
if selfOnly && len(users) == 0 {
|
||||
if selfOnly {
|
||||
selfUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
selfUser = cred.ParentUser
|
||||
}
|
||||
users = append(users, selfUser)
|
||||
users = []string{selfUser}
|
||||
}
|
||||
|
||||
var checkedUserList []string
|
||||
@@ -1421,7 +1455,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return buckets[i].Name < buckets[j].Name
|
||||
})
|
||||
} else {
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
|
||||
// Console refreshes this inventory immediately after bucket mutations.
|
||||
// A stale cached list can otherwise hide a creation or retain a deletion
|
||||
// indefinitely in the client's snapshot.
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1437,6 +1474,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn()
|
||||
policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
|
||||
var effectivePolicy policy.Policy
|
||||
var policyErr error
|
||||
|
||||
var buf []byte
|
||||
switch {
|
||||
@@ -1460,10 +1498,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
policySlice := newMappedPolicy(policy).toSlice()
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
|
||||
effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policySlice...)
|
||||
|
||||
case hasPolicyClaim:
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
|
||||
effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
|
||||
|
||||
default:
|
||||
policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...)
|
||||
@@ -1472,7 +1510,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
|
||||
effectivePolicy, policyErr = globalIAMSys.GetCombinedPolicy(policies...)
|
||||
}
|
||||
if policyErr != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, policyErr), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
|
||||
@@ -1729,7 +1771,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
|
||||
iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -2028,26 +2070,30 @@ func (a adminAPIHandlers) RevokeTokens(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
isSelfTarget := isTokenSelfRevoke || user == parentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
|
||||
if !adminPrivilege || isTokenSelfRevoke {
|
||||
parentUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
parentUser = cred.ParentUser
|
||||
}
|
||||
if !isTokenSelfRevoke && user != parentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
DenyOnly: isSelfTarget,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
if isSelfTarget {
|
||||
user = parentUser
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && user == globalActiveCred.AccessKey && !canManageRootCredentials(cred) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Infer token revoke type from the request if requestor is STS.
|
||||
if isTokenSelfRevoke && tokenRevokeType == "" && !fullRevoke {
|
||||
@@ -2094,29 +2140,22 @@ func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request)
|
||||
u, ok := globalIAMSys.GetUser(ctx, accessKey)
|
||||
targetCred := u.Credentials
|
||||
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
isSelf := ok && targetCred.ParentUser != "" && requestUser == targetCred.ParentUser
|
||||
if !isAllowedServiceAccountAction(cred, policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: isSelf,
|
||||
}) {
|
||||
// If requested user does not exist and requestor is not allowed to list service accounts, return access denied.
|
||||
if !ok {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
requestUser := cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
requestUser = cred.ParentUser
|
||||
}
|
||||
|
||||
if requestUser != targetCred.ParentUser {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !ok {
|
||||
@@ -2163,7 +2202,11 @@ func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
svcAccountPolicy, err = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
|
||||
@@ -2890,6 +2933,52 @@ func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasActiveSessionPolicy follows the same credential dispatch as IAMSys.IsAllowed.
|
||||
// Expiring service accounts also satisfy IsTemp and are evaluated as STS there.
|
||||
// Callers use this only after authorization under built-in IAM.
|
||||
func hasActiveSessionPolicy(cred auth.Credentials) bool {
|
||||
args := policy.Args{Claims: cred.Claims}
|
||||
if cred.IsTemp() {
|
||||
hasSP, _ := isAllowedBySessionPolicy(args)
|
||||
return hasSP
|
||||
}
|
||||
if cred.IsServiceAccount() {
|
||||
if cred.Claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
|
||||
return false
|
||||
}
|
||||
hasSP, _ := isAllowedBySessionPolicyForServiceAccount(args)
|
||||
return hasSP
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// canManageRootCredentials is an additional guard after built-in IAM authorization.
|
||||
// The signature's owner flag is insufficient: historical empty session policies
|
||||
// clear it even though the service account still inherits root's permissions.
|
||||
func canManageRootCredentials(cred auth.Credentials) bool {
|
||||
return cred.AccessKey == globalActiveCred.AccessKey ||
|
||||
(cred.ParentUser == globalActiveCred.AccessKey && cred.AccessKey != siteReplicatorSvcAcc && !hasActiveSessionPolicy(cred))
|
||||
}
|
||||
|
||||
// isAllowedServiceAccountAction preserves regular users' self inspection and
|
||||
// cleanup even without a policy attachment, while enforcing explicit denies.
|
||||
// Derived credentials and external authorization plugins use full IAM evaluation.
|
||||
func isAllowedServiceAccountAction(cred auth.Credentials, args policy.Args) bool {
|
||||
if args.DenyOnly && !args.IsOwner && !cred.IsTemp() && !cred.IsServiceAccount() && newGlobalAuthZPluginFn() == nil {
|
||||
policies, err := globalIAMSys.PolicyDBGet(args.AccountName, args.Groups...)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
combined, err := globalIAMSys.GetCombinedPolicy(policies...)
|
||||
if err != nil {
|
||||
iamLogIf(GlobalContext, err)
|
||||
return false
|
||||
}
|
||||
return combined.IsAllowed(args)
|
||||
}
|
||||
return globalIAMSys.IsAllowed(args)
|
||||
}
|
||||
|
||||
func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
|
||||
ctx := r.Context()
|
||||
|
||||
@@ -2978,13 +3067,27 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
}) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
effectiveTarget := targetUser
|
||||
if (targetUser == cred.AccessKey || targetUser == cred.ParentUser) && cred.ParentUser != "" {
|
||||
effectiveTarget = cred.ParentUser
|
||||
}
|
||||
if newGlobalAuthZPluginFn() == nil && (hasActiveSessionPolicy(cred) ||
|
||||
(effectiveTarget == globalActiveCred.AccessKey && !canManageRootCredentials(cred))) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy))
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
// Empty service policies mean inheritance, including for expiring keys.
|
||||
// Normalize the replication request as well as the local credential.
|
||||
if sp.Version == "" && sp.Statements == nil && sp.ID == "" {
|
||||
sp = nil
|
||||
createReq.Policy = nil
|
||||
}
|
||||
}
|
||||
|
||||
opts.sessionPolicy = sp
|
||||
|
||||
@@ -40,13 +40,54 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
testDefaultTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
func TestSetUserStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.AccountEnabled), want: policy.EnableUserAdminAction},
|
||||
{name: "disable", status: string(madmin.AccountDisabled), want: policy.DisableUserAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableUserAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setUserStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setUserStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetGroupStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.GroupEnabled), want: policy.EnableGroupAdminAction},
|
||||
{name: "disable", status: string(madmin.GroupDisabled), want: policy.DisableGroupAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableGroupAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setGroupStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setGroupStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// API suite container for IAM
|
||||
type TestSuiteIAM struct {
|
||||
TestSuiteCommon
|
||||
@@ -202,8 +243,12 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
||||
|
||||
suite.SetUpSuite(c)
|
||||
suite.TestUserCreate(c)
|
||||
suite.TestUserPasswordActionAuthorization(c)
|
||||
suite.TestUserStatusActionAuthorization(c)
|
||||
suite.TestGroupStatusActionAuthorization(c)
|
||||
suite.TestUserPolicyEscalationBug(c)
|
||||
suite.TestPolicyCreate(c)
|
||||
suite.TestServiceAccountBareARNPolicyRejected(c)
|
||||
suite.TestCannedPolicies(c)
|
||||
suite.TestGroupAddRemove(c)
|
||||
suite.TestServiceAccountOpsByAdmin(c)
|
||||
@@ -312,6 +357,284 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserPasswordActionAuthorization(c *check) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
statements string
|
||||
self bool
|
||||
other bool
|
||||
}{
|
||||
{"readonly", "", true, false},
|
||||
{"consolereadonly", "", true, false},
|
||||
{"password grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"}`, true, false},
|
||||
{"legacy CreateUser deny", `{"Effect":"Deny","Action":"admin:CreateUser","Resource":"arn:aws:s3:::*"}`, true, false},
|
||||
{"password deny", `{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||
{"user admin", `{"Effect":"Allow","Action":"admin:CreateUser"}`, true, true},
|
||||
{"user admin with password deny", `{"Effect":"Allow","Action":"admin:CreateUser"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, true},
|
||||
{"password deny overrides grant", `{"Effect":"Allow","Action":"admin:ChangeMyPassword"},{"Effect":"Deny","Action":"admin:ChangeMyPassword"}`, false, false},
|
||||
{"wildcard deny", `{"Effect":"Deny","Action":"admin:*"}`, false, false},
|
||||
} {
|
||||
c.Run(tt.name, func(t *testing.T) {
|
||||
c := &check{t, s.serverType}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
var users []string
|
||||
policyName := tt.name
|
||||
defer func() {
|
||||
for _, user := range users {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("remove test user: %v", err)
|
||||
}
|
||||
}
|
||||
if tt.statements != "" {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("remove test policy: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("create test user: %v", err)
|
||||
}
|
||||
users = append(users, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
client := func(accessKey, secretKey string) *madmin.AdminClient {
|
||||
adm, err := madmin.New(s.endpoint, accessKey, secretKey, s.secure)
|
||||
if err != nil {
|
||||
c.Fatal(err)
|
||||
}
|
||||
adm.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return adm
|
||||
}
|
||||
if tt.statements != "" {
|
||||
policyName = getRandomBucketName()
|
||||
doc := []byte(`{"Version":"2012-10-17","Statement":[` + tt.statements + `]}`)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, doc); err != nil {
|
||||
c.Fatalf("save test policy: %v", err)
|
||||
}
|
||||
}
|
||||
accessKey, secretKey := createUser()
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
User: accessKey, Policies: []string{policyName},
|
||||
}); err != nil {
|
||||
c.Fatalf("attach test policy: %v", err)
|
||||
}
|
||||
adm := client(accessKey, secretKey)
|
||||
_, newSecretKey := mustGenerateCredentials(c)
|
||||
err := adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled)
|
||||
if tt.self {
|
||||
if err != nil {
|
||||
c.Fatalf("change own password: %v", err)
|
||||
}
|
||||
if _, err = adm.AccountInfo(ctx, madmin.AccountOpts{}); err == nil {
|
||||
c.Fatal("old password still authenticates")
|
||||
}
|
||||
adm = client(accessKey, newSecretKey)
|
||||
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("self password change: expected AccessDenied, got %v", err)
|
||||
}
|
||||
if _, err := adm.AccountInfo(ctx, madmin.AccountOpts{}); err != nil {
|
||||
c.Fatalf("current password no longer authenticates: %v", err)
|
||||
}
|
||||
target, _ := createUser()
|
||||
newUser, newUserSecret := mustGenerateCredentials(c)
|
||||
for _, key := range []string{target, newUser} {
|
||||
err := adm.SetUser(ctx, key, newUserSecret, madmin.AccountEnabled)
|
||||
if tt.other {
|
||||
if err != nil {
|
||||
c.Fatalf("create or update another user: %v", err)
|
||||
}
|
||||
if key == newUser {
|
||||
users = append(users, newUser)
|
||||
}
|
||||
} else if err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("create or update another user: expected AccessDenied, got %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
defer func() {
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
disableClient := createStatusClient(policy.DisableUserAdminAction)
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err != nil {
|
||||
c.Fatalf("DisableUser-only client could not disable a user: %v", err)
|
||||
}
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableUser-only client unexpectedly enabled a user: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableUserAdminAction)
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("EnableUser-only client could not enable a user: %v", err)
|
||||
}
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableUser-only client unexpectedly disabled a user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestGroupStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
group := getRandomBucketName()
|
||||
var groupCreated bool
|
||||
defer func() {
|
||||
if groupCreated {
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: createdUsers[:1],
|
||||
IsRemove: true,
|
||||
}); err != nil {
|
||||
c.Errorf("unable to remove group member: %v", err)
|
||||
}
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{Group: group, IsRemove: true}); err != nil {
|
||||
c.Errorf("unable to remove test group: %v", err)
|
||||
}
|
||||
}
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: []string{targetAccessKey},
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to create test group: %v", err)
|
||||
}
|
||||
groupCreated = true
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add group status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach group status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create group status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
disableClient := createStatusClient(policy.DisableGroupAdminAction)
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err != nil {
|
||||
c.Fatalf("DisableGroup-only client could not disable a group: %v", err)
|
||||
}
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableGroup-only client unexpectedly enabled a group: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableGroupAdminAction)
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err != nil {
|
||||
c.Fatalf("EnableGroup-only client could not enable a group: %v", err)
|
||||
}
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableGroup-only client unexpectedly disabled a group: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
@@ -600,6 +923,20 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
c.Fatalf("invalid policy creation success")
|
||||
}
|
||||
|
||||
for i, resource := range []string{"arn:aws:s3:::", "*arn:aws:s3:::"} {
|
||||
barePolicyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["%s"]
|
||||
}]
|
||||
}`, resource)
|
||||
if err = s.adm.AddCannedPolicy(ctx, fmt.Sprintf("%s-bare-%d", policy, i), barePolicyBytes); err == nil {
|
||||
c.Fatalf("bare ARN policy creation succeeded for %q", resource)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Create a user, associate policy and verify access
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
|
||||
@@ -653,6 +990,51 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestServiceAccountBareARNPolicyRejected(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
barePolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"NotResource": ["arn:aws:s3:::"]
|
||||
}]
|
||||
}`)
|
||||
if _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account creation accepted a bare ARN policy")
|
||||
}
|
||||
|
||||
validPolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}]
|
||||
}`)
|
||||
credentials, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: validPolicy,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("service account creation rejected an explicit resource: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = s.adm.DeleteServiceAccount(ctx, credentials.AccessKey)
|
||||
}()
|
||||
|
||||
if err = s.adm.UpdateServiceAccount(ctx, credentials.AccessKey, madmin.UpdateServiceAccountReq{
|
||||
NewPolicy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account update accepted a bare ARN policy")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
@@ -665,6 +1047,7 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
defaultPolicies := []string{
|
||||
"readwrite",
|
||||
"readonly",
|
||||
"consolereadonly",
|
||||
"writeonly",
|
||||
"diagnostics",
|
||||
"consoleAdmin",
|
||||
|
||||
@@ -60,8 +60,8 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/secure-io/sio-go"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
@@ -163,6 +163,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
|
||||
// StorageInfo operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/multipart-preflight").HandlerFunc(adminMiddleware(adminAPI.MultipartPreflightHandler, traceAllFlag))
|
||||
// DataUsageInfo operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
|
||||
// Metrics operation
|
||||
@@ -388,6 +389,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
|
||||
adminRouter.Methods(http.MethodGet, http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-revisions").HandlerFunc(adminMiddleware(adminAPI.SRPeerIAMRevisions))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of SILO Object Storage.
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/hash"
|
||||
)
|
||||
|
||||
func TestChecksumMismatchAPIMessage(t *testing.T) {
|
||||
for _, algorithm := range []string{"CRC32", "CRC32C", "SHA1", "SHA256", "CRC64NVME", ""} {
|
||||
t.Run(algorithm, func(t *testing.T) {
|
||||
mismatch := hash.ChecksumMismatch{Algorithm: algorithm, Want: "expected", Got: "actual"}
|
||||
want := errorCodes.ToAPIErr(ErrContentChecksumMismatch).Description
|
||||
if algorithm != "" {
|
||||
want = fmt.Sprintf("The %s you specified did not match the calculated checksum.", algorithm)
|
||||
}
|
||||
for _, err := range []error{mismatch, fmt.Errorf("upload: %w", mismatch)} {
|
||||
got := toAPIError(t.Context(), err)
|
||||
if got.HTTPStatusCode != http.StatusBadRequest || got.Code != "XAmzContentChecksumMismatch" || got.Description != want {
|
||||
t.Fatalf("toAPIError(%v) = %+v, want 400 XAmzContentChecksumMismatch: %s", err, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+49
-3
@@ -48,7 +48,7 @@ import (
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// APIError structure
|
||||
@@ -450,6 +450,9 @@ const (
|
||||
ErrAdminNoSecretKey
|
||||
|
||||
ErrIAMNotInitialized
|
||||
ErrMultipartListingLegacy // Reserved: legacy uploads no longer block listing.
|
||||
ErrMultipartListingIdentity
|
||||
ErrSlowDown
|
||||
|
||||
apiErrCodeEnd // This is used only for the testing code
|
||||
)
|
||||
@@ -547,7 +550,7 @@ var errorCodes = errorCodeMap{
|
||||
},
|
||||
ErrBadDigest: {
|
||||
Code: "BadDigest",
|
||||
Description: "The Content-Md5 you specified did not match what we received.",
|
||||
Description: "The Content-MD5 you specified did not match what we received.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrEntityTooSmall: {
|
||||
@@ -1336,6 +1339,21 @@ var errorCodes = errorCodeMap{
|
||||
Description: "IAM sub-system not initialized yet, please try again.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrMultipartListingLegacy: {
|
||||
Code: "MultipartListingNotReady",
|
||||
Description: "Legacy multipart uploads prevent a complete listing. Upgrade all writers, drain old uploads and run the multipart preflight check.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrMultipartListingIdentity: {
|
||||
Code: "MultipartListingMetadataInvalid",
|
||||
Description: "Multipart upload metadata is inconsistent. Run the multipart preflight check to locate the affected storage set.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrSlowDown: {
|
||||
Code: "SlowDown",
|
||||
Description: "Please reduce your request rate",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrBucketMetadataNotInitialized: {
|
||||
Code: "XMinioBucketMetadataNotInitialized",
|
||||
Description: "Bucket metadata not initialized yet, please try again.",
|
||||
@@ -1523,10 +1541,14 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Your Host header is malformed.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// The stored object cannot be served: a server-side data condition, not a
|
||||
// successful partial read. Upstream maps it to http.StatusPartialContent
|
||||
// (since ca6b4773e, 2017), which lets SDKs accept the XML error document
|
||||
// as object content; SILO deliberately diverges and returns 500.
|
||||
ErrObjectTampered: {
|
||||
Code: "XMinioObjectTampered",
|
||||
Description: errObjectTampered.Error(),
|
||||
HTTPStatusCode: http.StatusPartialContent,
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
},
|
||||
|
||||
ErrSiteReplicationInvalidRequest: {
|
||||
@@ -2169,6 +2191,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
err = unwrapAll(err)
|
||||
|
||||
switch err {
|
||||
case errMultipartListingIdentity:
|
||||
apiErr = ErrMultipartListingIdentity
|
||||
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
||||
apiErr = ErrBadDigest
|
||||
case errMissingPartChecksum:
|
||||
apiErr = ErrInvalidRequest
|
||||
case errInvalidArgument:
|
||||
apiErr = ErrAdminInvalidArgument
|
||||
case errNoSuchPolicy:
|
||||
@@ -2295,6 +2323,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
}
|
||||
|
||||
switch err.(type) {
|
||||
case SlowDown:
|
||||
apiErr = ErrSlowDown
|
||||
case StorageFull:
|
||||
apiErr = ErrStorageFull
|
||||
case hash.BadDigest:
|
||||
@@ -2465,6 +2495,22 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
}
|
||||
|
||||
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
|
||||
switch {
|
||||
case errors.Is(err, errCompleteMultipartChecksumMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumMismatch.Error()+": ")
|
||||
case errors.Is(err, errCompleteMultipartChecksumTypeMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumTypeMismatch.Error()+": ")
|
||||
case errors.Is(err, errMissingPartChecksum):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errMissingPartChecksum.Error()+": ")
|
||||
}
|
||||
// PutObject/UploadPart checksum mismatches name the algorithm in the
|
||||
// message, matching S3 (e.g. "The CRC32 you specified did not match the
|
||||
// calculated checksum."). Falls through to the generic table description
|
||||
// when the algorithm is unknown.
|
||||
var checksumMismatch hash.ChecksumMismatch
|
||||
if errors.As(err, &checksumMismatch) && checksumMismatch.Algorithm != "" {
|
||||
apiErr.Description = fmt.Sprintf("The %s you specified did not match the calculated checksum.", checksumMismatch.Algorithm)
|
||||
}
|
||||
switch apiErr.Code {
|
||||
case "NotImplemented":
|
||||
apiErr = APIError{
|
||||
|
||||
@@ -39,6 +39,10 @@ var toAPIErrorTests = []struct {
|
||||
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
||||
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||
{err: errCompleteMultipartChecksumMismatch, errCode: ErrBadDigest},
|
||||
{err: errCompleteMultipartChecksumTypeMismatch, errCode: ErrBadDigest},
|
||||
{err: errMissingPartChecksum, errCode: ErrInvalidRequest},
|
||||
{err: hash.ChecksumMismatch{}, errCode: ErrContentChecksumMismatch},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
|
||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||
|
||||
+4
-1
@@ -212,7 +212,10 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
||||
}
|
||||
|
||||
if rs == nil && opts.PartNumber > 0 {
|
||||
rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
rs, err = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// For providing ranged content
|
||||
|
||||
+36
-11
@@ -27,7 +27,6 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
@@ -35,14 +34,14 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xxml "github.com/minio/xxml"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
|
||||
maxDeleteList = 1000 // Limit number of objects deleted in a delete call.
|
||||
maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
|
||||
maxUploadsList = 1000 // Limit number of uploads in a listUploadsResponse.
|
||||
maxPartsList = 10000 // Limit number of parts in a listPartsResponse.
|
||||
)
|
||||
|
||||
@@ -380,6 +379,13 @@ type CopyObjectResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumType string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object
|
||||
@@ -387,6 +393,12 @@ type CopyObjectPartResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object part.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// Initiator inherit from Owner struct, fields are same
|
||||
@@ -416,6 +428,7 @@ type CompleteMultipartUploadResponse struct {
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumType string `xml:"ChecksumType,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
@@ -763,19 +776,30 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
||||
|
||||
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
||||
|
||||
// generates CopyObjectResponse from etag and lastModified time.
|
||||
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
||||
// generates CopyObjectResponse from the committed object information.
|
||||
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
|
||||
return CopyObjectResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(oi.ModTime.UTC()),
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
}
|
||||
}
|
||||
|
||||
// generates CopyObjectPartResponse from etag and lastModified time.
|
||||
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
|
||||
// generates CopyObjectPartResponse from the uploaded part information.
|
||||
func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse {
|
||||
return CopyObjectPartResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + partInfo.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()),
|
||||
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -797,6 +821,7 @@ func generateCompleteMultipartUploadResponse(bucket, key, location string, oi Ob
|
||||
Key: key,
|
||||
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
|
||||
+127
-10
@@ -18,16 +18,23 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
consoleapi "github.com/minio/console/api"
|
||||
bktcors "github.com/minio/minio/internal/bucket/cors"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/rs/cors"
|
||||
)
|
||||
|
||||
type bucketCorsAppliedKey struct{}
|
||||
|
||||
func newHTTPServerFn() *xhttp.Server {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
@@ -111,11 +118,6 @@ var rejectedBucketAPIs = []rejectedAPI{
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"inventory", ""},
|
||||
},
|
||||
{
|
||||
api: "cors",
|
||||
methods: []string{http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"cors", ""},
|
||||
},
|
||||
{
|
||||
api: "metrics",
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
@@ -459,15 +461,15 @@ func registerAPIRouter(router *mux.Router) {
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
|
||||
Queries("acl", "")
|
||||
// GetBucketCors - this is a dummy call.
|
||||
// GetBucketCors
|
||||
router.Methods(http.MethodGet).
|
||||
HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// PutBucketCors - this is a dummy call.
|
||||
// PutBucketCors
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// DeleteBucketCors - this is a dummy call.
|
||||
// DeleteBucketCors
|
||||
router.Methods(http.MethodDelete).
|
||||
HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
@@ -648,6 +650,94 @@ func registerAPIRouter(router *mux.Router) {
|
||||
apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
|
||||
}
|
||||
|
||||
// applyBucketCors applies a bucket's CORS configuration to the request.
|
||||
// For an OPTIONS preflight it writes the full CORS response and returns true
|
||||
// (request is complete). For an actual request it adds the applicable
|
||||
// Access-Control-* response headers and returns false so the request
|
||||
// continues down the handler chain. If no rule matches a preflight it writes
|
||||
// 403 and returns true. A matched actual request is marked in its context so
|
||||
// inner legacy middleware does not rewrite an explicitly allowed null origin.
|
||||
func applyBucketCors(w http.ResponseWriter, r *http.Request, cfg *bktcors.Config) (handled bool) {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
return false // not a CORS request
|
||||
}
|
||||
h := w.Header()
|
||||
h.Add("Vary", "Origin")
|
||||
|
||||
isPreflight := r.Method == http.MethodOptions &&
|
||||
r.Header.Get("Access-Control-Request-Method") != ""
|
||||
|
||||
if isPreflight {
|
||||
method := r.Header.Get("Access-Control-Request-Method")
|
||||
reqHeaders := splitAndTrim(r.Header.Get("Access-Control-Request-Headers"))
|
||||
// A preflight response depends on all three request headers that
|
||||
// determine the outcome, including when the request is rejected.
|
||||
h.Add("Vary", "Access-Control-Request-Method")
|
||||
h.Add("Vary", "Access-Control-Request-Headers")
|
||||
rule, allowedOrigin, allowedHeaders, maxAgeSeconds, ok := cfg.MatchPreflight(origin, method, reqHeaders)
|
||||
if !ok {
|
||||
writeResponse(w, http.StatusForbidden, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
h.Set("Access-Control-Allow-Methods", strings.Join(rule.AllowedMethods, ", "))
|
||||
if len(allowedHeaders) > 0 {
|
||||
h.Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", "))
|
||||
}
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
if maxAgeSeconds != nil {
|
||||
h.Set("Access-Control-Max-Age", strconv.Itoa(*maxAgeSeconds))
|
||||
}
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
|
||||
// Actual request: attach headers if the origin+method match.
|
||||
rule, allowedOrigin, ok := cfg.MatchRule(origin, r.Method)
|
||||
if !ok {
|
||||
return false // no matching rule → no CORS headers, continue normally
|
||||
}
|
||||
*r = *r.WithContext(context.WithValue(r.Context(), bucketCorsAppliedKey{}, struct{}{}))
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func bucketCorsWasApplied(r *http.Request) bool {
|
||||
_, ok := r.Context().Value(bucketCorsAppliedKey{}).(struct{})
|
||||
return ok
|
||||
}
|
||||
|
||||
func setBucketCorsOriginHeaders(h http.Header, allowedOrigin, requestOrigin string) {
|
||||
if allowedOrigin == "*" {
|
||||
h.Set("Access-Control-Allow-Origin", "*")
|
||||
h.Del("Access-Control-Allow-Credentials")
|
||||
return
|
||||
}
|
||||
h.Set("Access-Control-Allow-Origin", requestOrigin)
|
||||
h.Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
// splitAndTrim splits a comma-separated header list into trimmed, non-empty values.
|
||||
func splitAndTrim(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
out := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// corsHandler handler for CORS (Cross Origin Resource Sharing)
|
||||
func corsHandler(handler http.Handler) http.Handler {
|
||||
commonS3Headers := []string{
|
||||
@@ -693,5 +783,32 @@ func corsHandler(handler http.Handler) http.Handler {
|
||||
ExposedHeaders: commonS3Headers,
|
||||
AllowCredentials: true,
|
||||
}
|
||||
return cors.New(opts).Handler(handler)
|
||||
globalCors := cors.New(opts).Handler(handler)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Origin") != "" {
|
||||
if bucket, _ := request2BucketObjectName(r); bucket != "" && globalBucketMetadataSys != nil {
|
||||
// Resident-only lookup: this runs before authentication with a
|
||||
// client-supplied path segment as the bucket name, so it must
|
||||
// never load or cache metadata. While startup loading is still
|
||||
// running, for a real bucket whose metadata failed to load, and
|
||||
// for a bucket whose stored CORS document failed to parse, the
|
||||
// request gets no CORS headers; any other non-resident name falls
|
||||
// back to the global policy below.
|
||||
cfg, _, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket)
|
||||
if err == nil && cfg != nil {
|
||||
if applyBucketCors(w, r, cfg) {
|
||||
return
|
||||
}
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
internalLogOnceIf(r.Context(), err, "bucket-cors-metadata")
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
globalCors.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
+70
-87
@@ -28,20 +28,17 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/etag"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xjwt "github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mcontext"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Verify if request has JWT.
|
||||
@@ -363,17 +360,6 @@ func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, a
|
||||
return authorizeRequestWithTags(ctx, r, action, "", requestTags)
|
||||
}
|
||||
|
||||
// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
|
||||
// passes versionID additionally.
|
||||
func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
logger.GetReqInfo(ctx).VersionID = versionID
|
||||
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
if logger.GetReqInfo(ctx) == nil {
|
||||
bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
|
||||
@@ -439,6 +425,23 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
return authorizeRequestWithExistingTags(ctx, r, action, "")
|
||||
}
|
||||
|
||||
func deleteObjectAction(versionID string) policy.Action {
|
||||
if versionID != "" {
|
||||
return policy.DeleteObjectVersionAction
|
||||
}
|
||||
return policy.DeleteObjectAction
|
||||
}
|
||||
|
||||
func actionUsesObjectVersion(action policy.Action) bool {
|
||||
switch action {
|
||||
case policy.DeleteObjectAction, policy.DeleteObjectVersionAction,
|
||||
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func authorizeRequestWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string) (s3Err APIErrorCode) {
|
||||
return authorizeRequestWithTags(ctx, r, action, existingTags, nil)
|
||||
}
|
||||
@@ -457,7 +460,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
versionID := reqInfo.VersionID
|
||||
conditionValuesForAuth := func(locationConstraint string, credentials auth.Credentials) map[string][]string {
|
||||
values := getConditionValuesWithTags(r, locationConstraint, credentials, existingTags, requestTags)
|
||||
if action == policy.DeleteObjectAction {
|
||||
if actionUsesObjectVersion(action) {
|
||||
// DeleteObjects carries the effective version ID in each XML object,
|
||||
// not in the request query. Keep authorization scoped to that entry.
|
||||
if versionID == "" {
|
||||
@@ -503,21 +506,6 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if action == policy.DeleteObjectAction && versionID != "" {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.Action(policy.DeleteObjectVersionAction),
|
||||
BucketName: bucket,
|
||||
ConditionValues: conditionValuesForAuth("", cred),
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) { // Request is not allowed if Deny action on DeleteObjectVersionAction
|
||||
return ErrAccessDenied
|
||||
}
|
||||
}
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
@@ -553,6 +541,40 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// authorizeReplicationDelete preserves the established target-credential
|
||||
// contract for trusted replication: DeleteObject and ReplicateDelete must be
|
||||
// allowed, while an explicit DeleteObjectVersion deny still blocks a named
|
||||
// version. Ordinary S3 requests never use this compatibility path.
|
||||
func authorizeReplicationDelete(ctx context.Context, r *http.Request) APIErrorCode {
|
||||
if s3Err := authorizeRequest(ctx, r, policy.DeleteObjectAction); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if reqInfo.VersionID == "" {
|
||||
return ErrNone
|
||||
}
|
||||
cred := reqInfo.Cred
|
||||
values := getConditionValuesWithTags(r, "", cred, "", nil)
|
||||
values["versionid"] = []string{reqInfo.VersionID}
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.DeleteObjectVersionAction,
|
||||
BucketName: reqInfo.BucketName,
|
||||
ConditionValues: values,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
IsOwner: reqInfo.Owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
@@ -607,18 +629,11 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
return ErrInvalidDigest
|
||||
}
|
||||
|
||||
// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
|
||||
// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
|
||||
// Honor the selected header/query checksum, including the header fallback
|
||||
// for a presigned request. STS separately hashes its body for its signature.
|
||||
var contentSHA256 []byte
|
||||
if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
|
||||
if sha256Sum, ok := r.Form[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
|
||||
contentSHA256, err = hex.DecodeString(sha256Sum[0])
|
||||
if err != nil {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
}
|
||||
} else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok {
|
||||
contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256))
|
||||
if !skipContentSha256Cksum(r) {
|
||||
contentSHA256, err = hex.DecodeString(getContentSha256Cksum(r, serviceS3))
|
||||
if err != nil || len(contentSHA256) == 0 {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
@@ -715,48 +730,6 @@ func setAuthMiddleware(h http.Handler) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) {
|
||||
var retSet bool
|
||||
if cred.AccessKey == "" {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
conditions := getConditionValues(r, "", cred)
|
||||
conditions["object-lock-mode"] = []string{string(retMode)}
|
||||
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
|
||||
if retDays > 0 {
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
||||
}
|
||||
if retMode == objectlock.RetGovernance && byPassSet {
|
||||
byPassSet = globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.BypassGovernanceRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ObjectName: objectName,
|
||||
ConditionValues: conditions,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
}
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.PutObjectRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: conditions,
|
||||
ObjectName: objectName,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
retSet = true
|
||||
}
|
||||
if byPassSet || retSet {
|
||||
return ErrNone
|
||||
}
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// isPutActionAllowed - check if PUT operation is allowed on the resource, this
|
||||
// call verifies bucket policies and IAM policies, supports multi user
|
||||
// checks etc.
|
||||
@@ -786,10 +759,20 @@ func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, buck
|
||||
return s3Err
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
reqInfo.Lock()
|
||||
reqInfo.Cred = cred
|
||||
reqInfo.Owner = owner
|
||||
reqInfo.Region = region
|
||||
reqInfo.Unlock()
|
||||
|
||||
return isPutActionAllowedWithCred(bucketName, objectName, r, action, requestTags, cred, owner)
|
||||
}
|
||||
|
||||
func isPutActionAllowedWithCred(bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string, cred auth.Credentials, owner bool) APIErrorCode {
|
||||
// Do not check for PutObjectRetentionAction permission,
|
||||
// if mode and retain until date are not set.
|
||||
// Can happen when bucket has default lock config set
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
type nullReader struct{}
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// healTask represents what to heal along with options
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// A pull into a suspended bucket replaces the null version. Its policy
|
||||
// decision must name that version even when the source is unversioned S3.
|
||||
func TestBatchReviewPullImplicitNullVersionAuthorization(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
|
||||
mustIAM(t, err)
|
||||
data := []byte("original null version")
|
||||
before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: true})
|
||||
mustIAM(t, err)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "null-deny", batchAuthzFullAccess+`,
|
||||
{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"null"}}}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "null-deny-user", "null-deny")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
job := batchPullJob(f.bucket)
|
||||
job.Source.Type = BatchJobReplicateResourceS3
|
||||
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
|
||||
Name: before.Name, ModTime: UTCNow(), ETag: "batchetag", UserDefined: map[string]string{},
|
||||
}, false, sub)
|
||||
if !isBatchJobAuthzError(err) {
|
||||
t.Errorf("null version policy must refuse the write, got %v", err)
|
||||
}
|
||||
after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: nullVersionID})
|
||||
if gerr != nil || before.ETag != after.ETag {
|
||||
t.Errorf("denied null version must remain unchanged: read=%v unchanged=%v", gerr, before.ETag == after.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
type batchReviewPrefixWalkFailure struct{ ObjectLayer }
|
||||
|
||||
func (a batchReviewPrefixWalkFailure) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
|
||||
if prefix != "a/" {
|
||||
return a.ObjectLayer.Walk(ctx, bucket, prefix, results, opts)
|
||||
}
|
||||
go func() {
|
||||
defer close(results)
|
||||
results <- itemOrErr[ObjectInfo]{Err: errors.New("test: incomplete first prefix")}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
type batchReviewWalkCutoff struct {
|
||||
ObjectLayer
|
||||
cutoff string
|
||||
}
|
||||
|
||||
func (a batchReviewWalkCutoff) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
|
||||
inner := make(chan itemOrErr[ObjectInfo], 1)
|
||||
if err := a.ObjectLayer.Walk(ctx, bucket, prefix, inner, opts); err != nil {
|
||||
close(results)
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
defer close(results)
|
||||
stopped := false
|
||||
for result := range inner {
|
||||
if stopped {
|
||||
continue
|
||||
}
|
||||
if result.Err == nil && result.Item.Name >= a.cutoff {
|
||||
stopped = true
|
||||
result = itemOrErr[ObjectInfo]{Err: errors.New("test: listing stopped before remaining child keys")}
|
||||
}
|
||||
select {
|
||||
case results <- result:
|
||||
case <-ctx.Done():
|
||||
stopped = true
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Listing a directory prefix returns the directory object before its child
|
||||
// keys. A durable cursor for that directory must preserve unfinished children.
|
||||
func TestBatchReviewDirectoryCursorKeepsChildrenPending(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
for range 12 {
|
||||
batchPutVersion(t, f.obj, f.bucket, "a/", nil)
|
||||
}
|
||||
var pending string
|
||||
for i := 0; i < 13; i++ {
|
||||
older := f.putVersions(t, fmt.Sprintf("a/key-%02d", i))
|
||||
if i == 11 {
|
||||
pending = older
|
||||
}
|
||||
}
|
||||
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
|
||||
apiVersion: v1
|
||||
bucket: %s
|
||||
prefix: ["a/"]
|
||||
rules:
|
||||
- type: object
|
||||
purge:
|
||||
retainVersions: 1
|
||||
`, f.bucket))
|
||||
if job == nil {
|
||||
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
_ = job.Expire.Start(f.ctx, batchReviewWalkCutoff{ObjectLayer: f.obj, cutoff: "a/key-11"}, *job)
|
||||
ri := f.jobInfo(t, job)
|
||||
if ri.Object != "a/" {
|
||||
t.Fatalf("control: the completed directory batch must be the cursor, got %q", ri.Object)
|
||||
}
|
||||
mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
|
||||
if f.versionExists(t, "a/key-11", pending) {
|
||||
t.Error("a directory cursor must not suppress the unvisited child key on resume")
|
||||
}
|
||||
}
|
||||
|
||||
// Later prefixes must not advance the completion cursor past an earlier
|
||||
// prefix whose listing failed. This state is produced by Start itself.
|
||||
func TestBatchReviewExpireFailedPrefixKeepsWorkPending(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
older := f.putVersions(t, "a/todo")
|
||||
for i := 0; i < 13; i++ {
|
||||
f.putVersions(t, fmt.Sprintf("z/key-%02d", i))
|
||||
}
|
||||
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
|
||||
apiVersion: v1
|
||||
bucket: %s
|
||||
prefix: ["z/", "a/"]
|
||||
rules:
|
||||
- type: object
|
||||
purge:
|
||||
retainVersions: 1
|
||||
`, f.bucket))
|
||||
if job == nil {
|
||||
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
_ = job.Expire.Start(f.ctx, batchReviewPrefixWalkFailure{f.obj}, *job)
|
||||
ri := f.jobInfo(t, job)
|
||||
if ri.Object >= "a/todo" {
|
||||
t.Errorf("cursor must not cross an incomplete prefix, got %q", ri.Object)
|
||||
}
|
||||
mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
|
||||
if f.versionExists(t, "a/todo", older) {
|
||||
t.Error("a fresh Start must revisit the key omitted by the failed listing")
|
||||
}
|
||||
}
|
||||
|
||||
// A repeated marker replication error must remain a failure on the second
|
||||
// attempt, including after the retry loop resets its counters.
|
||||
func TestBatchReviewMarkerRetryRemainsFailed(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchPutVersion(t, f.obj, f.bucket, "marker", nil)
|
||||
_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
|
||||
mustIAM(t, err)
|
||||
oldTransport := globalRemoteTargetTransport
|
||||
globalRemoteTargetTransport = batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
|
||||
if r.URL.Query().Has("location") {
|
||||
code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
|
||||
}
|
||||
return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
t.Cleanup(func() { globalRemoteTargetTransport = oldTransport })
|
||||
job := BatchJobRequest{
|
||||
ID: "replicate-marker-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
|
||||
Replicate: &BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
|
||||
},
|
||||
}
|
||||
job.Replicate.Flags.Retry.Attempts = 2
|
||||
job.Replicate.Flags.Retry.Delay = time.Millisecond
|
||||
mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
|
||||
ri := f.jobInfo(t, &job)
|
||||
if ri.Complete || !ri.Failed || ri.DeleteMarkersFailed != 1 {
|
||||
t.Errorf("both marker attempts failed: complete=%v failed=%v markerFailures=%d attempts=%d", ri.Complete, ri.Failed, ri.DeleteMarkersFailed, ri.RetryAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
// A current parent mapping suppresses the OpenID policy-name fallback.
|
||||
// Background-condition validation must inspect the same effective policies.
|
||||
func TestBatchReviewMappedPolicySuppressesClaimFallback(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "unused-claim-policy", `{"Effect":"Deny","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
|
||||
parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "mapped-full")
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
|
||||
parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), iamPolicyClaimNameOpenID(): "unused-claim-policy",
|
||||
}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent.AccessKey
|
||||
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
|
||||
mustIAM(t, err)
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
p, err := sub.principal(f.ctx)
|
||||
mustIAM(t, err)
|
||||
req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "mapped/key", versionID: mustGetUUID()}
|
||||
if !f.sys.isAllowedLocally(policy.Args{AccountName: p.accessKey, Claims: p.claims, Action: req.action, BucketName: req.bucket, ObjectName: req.object, ConditionValues: p.conditionValues(req)}) {
|
||||
t.Fatal("control: the current mapped policy must allow this read")
|
||||
}
|
||||
if err := sub.authorize(f.ctx, req); err != nil {
|
||||
t.Errorf("an unused fallback policy must not refuse a mapped-policy grant: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchReviewBindingAndExpiringServiceControls(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
root := batchRootSubmitter(t)
|
||||
priorRoot := globalActiveCred
|
||||
globalActiveCred.SecretKey = "different-test-root-secret"
|
||||
_, err := root.principal(f.ctx)
|
||||
globalActiveCred = priorRoot
|
||||
if !isBatchJobAuthzError(err) {
|
||||
t.Errorf("root-secret change must invalidate the binding: %v", err)
|
||||
}
|
||||
missing := *root
|
||||
missing.Binding = nil
|
||||
if _, err := missing.principal(f.ctx); !isBatchJobAuthzError(err) {
|
||||
t.Errorf("missing fingerprint must refuse the principal: %v", err)
|
||||
}
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "service-parent-full", batchAuthzFullAccess)
|
||||
parent := iamAdminUser(f.ctx, t, f.sys, "service-parent", "service-parent-full")
|
||||
sp, err := policy.ParseConfig(strings.NewReader(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}]}`))
|
||||
mustIAM(t, err)
|
||||
expires := UTCNow().Add(time.Hour)
|
||||
cred, _, err := f.sys.NewServiceAccount(f.ctx, parent.AccessKey, nil, newServiceAccountOpts{sessionPolicy: sp, expiration: &expires})
|
||||
mustIAM(t, err)
|
||||
stored, ok := f.sys.GetUser(f.ctx, cred.AccessKey)
|
||||
if !ok {
|
||||
t.Fatal("created service account must resolve")
|
||||
}
|
||||
cred = stored.Credentials
|
||||
if !cred.IsTemp() || !cred.IsServiceAccount() {
|
||||
t.Fatal("fixture must be an expiring service account")
|
||||
}
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "allowed"}); err != nil {
|
||||
t.Errorf("expiring service account must retain its allowed read: %v", err)
|
||||
}
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectAction, bucket: f.bucket, object: "allowed"}); !isBatchJobAuthzError(err) {
|
||||
t.Errorf("expiring service account must remain session-policy restricted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// An interrupted cross-pool copy can leave the same version in both pools.
|
||||
// A newer legal-hold field in either copy must prevent key rotation.
|
||||
func TestBatchReviewRotationUsesAuthoritativePoolHold(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const name = "rotation/split-hold"
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("batch-rotation-key")
|
||||
t.Cleanup(func() { GlobalKMS = previousKMS })
|
||||
dek, err := GlobalKMS.GenerateKey(t.Context(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{bucket: bucket + "/" + name}})
|
||||
mustIAM(t, err)
|
||||
objectKey := crypto.GenerateKey(dek.Plaintext, nil)
|
||||
sealed := objectKey.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), bucket, name)
|
||||
first := crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed)
|
||||
first[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "OFF"
|
||||
first[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
|
||||
second := maps.Clone(first)
|
||||
second[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = "ON"
|
||||
second[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("protected payload"), objectKey))
|
||||
mustIAM(t, err)
|
||||
before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
|
||||
putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
|
||||
logical, err := z.replicaObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if !enforceRetentionForDeletion(t.Context(), logical) {
|
||||
t.Fatal("control: the authoritative logical version must be held")
|
||||
}
|
||||
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
|
||||
err = r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
|
||||
if !isErrObjectLocked(err) {
|
||||
t.Errorf("rotation must refuse the authoritative legal hold, got %v", err)
|
||||
}
|
||||
after, gerr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, gerr)
|
||||
if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
|
||||
t.Error("held logical version must not have its key rewritten")
|
||||
}
|
||||
}
|
||||
|
||||
// An encryption condition must describe what rotation actually commits,
|
||||
// even when the requested encryption type differs from the existing type.
|
||||
func TestBatchReviewRotationEncryptionConditionMatchesResult(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("required-key")
|
||||
t.Cleanup(func() { GlobalKMS = previousKMS })
|
||||
const name = "rotation/encryption-condition"
|
||||
meta := make(map[string]string)
|
||||
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, name, meta, nil)
|
||||
mustIAM(t, err)
|
||||
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("encrypted payload"), key))
|
||||
mustIAM(t, err)
|
||||
before, err := f.obj.PutObject(f.ctx, f.bucket, name, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, UserDefined: meta})
|
||||
mustIAM(t, err)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "kms-write-only", `
|
||||
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"]},
|
||||
{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "kms-write-user", "kms-write-only")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
r := BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "required-key"}}
|
||||
err = r.KeyRotate(f.ctx, f.obj, before, sub)
|
||||
after, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, gerr)
|
||||
if err == nil && !crypto.S3KMS.IsEncrypted(after.UserDefined) {
|
||||
t.Error("rotation allowed by an aws:kms-only write policy must produce SSE-KMS or refuse the operation")
|
||||
}
|
||||
if err != nil && before.UserDefined[crypto.MetaSealedKeyS3] != after.UserDefined[crypto.MetaSealedKeyS3] {
|
||||
t.Error("a refused rotation must leave the version unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
// An explicit null marker collision must retain a held null data version.
|
||||
func TestBatchReviewPullMarkerNullLock(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
meta := batchLockMeta("LEGALHOLD")
|
||||
before, err := f.obj.PutObject(f.ctx, f.bucket, "pull/null", mustGetPutObjReader(t, strings.NewReader("held null"), 9, "", ""), ObjectOptions{VersionID: nullVersionID, Versioned: true, UserDefined: meta})
|
||||
mustIAM(t, err)
|
||||
job := BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
|
||||
}
|
||||
err = job.ReplicateFromSource(f.ctx, f.obj, nil, ObjectInfo{
|
||||
Name: before.Name, VersionID: nullVersionID, ModTime: UTCNow(), DeleteMarker: true,
|
||||
}, false, batchRootSubmitter(t))
|
||||
if !isErrObjectLocked(err) {
|
||||
t.Errorf("marker collision with protected null data must return ObjectLocked, got %v", err)
|
||||
}
|
||||
if !batchVersionExists(t, f.obj, f.bucket, before.Name, nullVersionID) {
|
||||
t.Fatal("protected null data must remain present")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// A read policy must use the newest tag state of the logical version even
|
||||
// when an interrupted movement left an older copy in the preferred pool.
|
||||
func TestBatchReviewPushUsesAuthoritativePoolTags(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
f := newBatchAuthzFixture(t)
|
||||
const name = "push/tagged"
|
||||
first := map[string]string{
|
||||
xhttp.AmzObjectTagging: "classification=public",
|
||||
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
|
||||
}
|
||||
second := map[string]string{
|
||||
xhttp.AmzObjectTagging: "classification=private",
|
||||
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
|
||||
}
|
||||
before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
|
||||
putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
|
||||
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if logical.UserTags != "classification=private" {
|
||||
t.Fatal("control: logical version must have the newer private tag")
|
||||
}
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "pooled-push", fmt.Sprintf(`
|
||||
{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::%s/*"]},
|
||||
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "pooled-push-user", "pooled-push")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: bucket, object: name, versionID: before.VersionID, tags: logical.UserTags}); !isBatchJobAuthzError(err) {
|
||||
t.Fatal("control: authoritative tags must deny the read")
|
||||
}
|
||||
writes := 0
|
||||
target, err := minio.NewCore("target.invalid", &minio.Options{
|
||||
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
_ = r.Body.Close()
|
||||
}
|
||||
if r.Method == http.MethodPut {
|
||||
writes++
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
|
||||
}),
|
||||
})
|
||||
mustIAM(t, err)
|
||||
r := BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
}
|
||||
err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
|
||||
if !isBatchJobAuthzError(err) || writes != 0 {
|
||||
t.Errorf("newer authoritative tags must deny push: err=%v writes=%d", err, writes)
|
||||
}
|
||||
}
|
||||
|
||||
// Verified token claims must not replace the actual version being authorized.
|
||||
func TestBatchReviewVersionConditionResistsClaimCollision(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
deniedVersion := mustGetUUID()
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "version-claims", batchAuthzFullAccess+fmt.Sprintf(`,
|
||||
{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, deniedVersion))
|
||||
parent := iamAdminUser(f.ctx, t, f.sys, "version-claims-parent", "version-claims")
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
|
||||
parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(), "versionid": "an-unrelated-claim-value",
|
||||
}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent.AccessKey
|
||||
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
|
||||
mustIAM(t, err)
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
req := batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: deniedVersion}
|
||||
if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
|
||||
t.Errorf("a token claim must not replace the actual denied version: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Retry totals describe a pass, including markers already successful before
|
||||
// another item caused that pass to retry.
|
||||
func TestBatchReviewMarkerSuccessCountResetsOnRetry(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchPutVersion(t, f.obj, f.bucket, "marker", nil)
|
||||
_, err := f.obj.DeleteObject(f.ctx, f.bucket, "marker", ObjectOptions{Versioned: true})
|
||||
mustIAM(t, err)
|
||||
oldTransport := remoteInstanceTransport
|
||||
remoteInstanceTransport = atomic.Value{}
|
||||
setRemoteInstanceTransport(batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
code, body := http.StatusForbidden, `<Error><Code>AccessDenied</Code><Message>test refusal</Message></Error>`
|
||||
h := make(http.Header)
|
||||
switch {
|
||||
case r.URL.Query().Has("location"):
|
||||
code, body = http.StatusOK, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-east-1</LocationConstraint>`
|
||||
case r.Method == http.MethodDelete:
|
||||
code, body = http.StatusNoContent, ""
|
||||
case r.Method == http.MethodHead && r.URL.Query().Get("versionId") != "":
|
||||
code, body = http.StatusMethodNotAllowed, ""
|
||||
h.Set("X-Amz-Delete-Marker", "true")
|
||||
}
|
||||
return &http.Response{StatusCode: code, Header: h, Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
}))
|
||||
t.Cleanup(func() { remoteInstanceTransport = oldTransport })
|
||||
job := BatchJobRequest{
|
||||
ID: "marker-success-retry" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
|
||||
Replicate: &BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote", Endpoint: "https://target.invalid", Creds: BatchJobReplicateCredentials{AccessKey: "test-access", SecretKey: "test-secret"}},
|
||||
},
|
||||
}
|
||||
job.Replicate.Flags.Retry.Attempts = 2
|
||||
job.Replicate.Flags.Retry.Delay = time.Millisecond
|
||||
mustIAM(t, job.Replicate.Start(f.ctx, f.obj, job))
|
||||
ri := f.jobInfo(t, &job)
|
||||
if !ri.Failed || ri.RetryAttempts != 2 {
|
||||
t.Fatalf("control: data failure must cause two attempts: failed=%v attempts=%d", ri.Failed, ri.RetryAttempts)
|
||||
}
|
||||
if ri.DeleteMarkers != 1 {
|
||||
t.Errorf("one successful marker must be counted once in the final attempt, got %d", ri.DeleteMarkers)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// A tag removal in a newer copy must govern both the authorization and the
|
||||
// metadata sent by a push. The submitter can read data but cannot read tags.
|
||||
func TestBatchReviewPushSendsOnlyAuthorizedMetadata(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
f := newBatchAuthzFixture(t)
|
||||
const name = "push/removed-tags"
|
||||
first := map[string]string{
|
||||
xhttp.AmzObjectTagging: "classification=private",
|
||||
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Add(-time.Hour).Format(time.RFC3339Nano),
|
||||
}
|
||||
second := map[string]string{
|
||||
xhttp.AmzObjectTagging: "",
|
||||
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
|
||||
}
|
||||
before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true, UserDefined: first})
|
||||
putConsistencyObject(t, z, bucket, name, 1, "test payload", ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
|
||||
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if logical.UserTags != "" {
|
||||
t.Fatal("control: the authoritative version must have no tags")
|
||||
}
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "data-without-tags", fmt.Sprintf(`
|
||||
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"]},
|
||||
{"Effect":"Deny","Action":["s3:GetObjectTagging","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket, bucket))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "data-without-tags-user", "data-without-tags")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: bucket, object: name, versionID: before.VersionID}); !isBatchJobAuthzError(err) {
|
||||
t.Fatal("control: reading tags must be denied")
|
||||
}
|
||||
writes := 0
|
||||
var sentTags string
|
||||
target, err := minio.NewCore("target.invalid", &minio.Options{
|
||||
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
_ = r.Body.Close()
|
||||
}
|
||||
if r.Method == http.MethodPut {
|
||||
writes++
|
||||
sentTags = r.Header.Get(xhttp.AmzObjectTagging)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Etag": {"\"test-etag\""}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
|
||||
}),
|
||||
})
|
||||
mustIAM(t, err)
|
||||
r := BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
}
|
||||
err = r.ReplicateToTarget(f.ctx, z, target, before, false, sub)
|
||||
if err != nil && !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("unexpected replication failure: %v", err)
|
||||
}
|
||||
if writes > 0 && sentTags != logical.UserTags {
|
||||
t.Errorf("push must send only metadata covered by authorization: writes=%d sentTags=%q authorizedTags=%q err=%v", writes, sentTags, logical.UserTags, err)
|
||||
}
|
||||
}
|
||||
|
||||
// LDAP's short username and its parent DN are distinct identities. The
|
||||
// background merge must not turn a deny for the verified short name into
|
||||
// a non-matching statement by replacing it with the parent DN.
|
||||
func TestBatchReviewLDAPUsernameDeny(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "ldap-short-name-deny", batchAuthzFullAccess+`,
|
||||
{"Effect":"Deny","Action":["s3:DeleteObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"ldap:username":"alice"}}}`)
|
||||
const dn = "uid=alice,ou=people,dc=example,dc=org"
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{
|
||||
"exp": UTCNow().Add(time.Hour).Unix(), ldapUser: dn, ldapActualUser: dn, ldapUserN: "alice",
|
||||
}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = dn
|
||||
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "ldap-short-name-deny")
|
||||
mustIAM(t, err)
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
p, err := sub.principal(f.ctx)
|
||||
mustIAM(t, err)
|
||||
cred.Claims = p.claims
|
||||
req, err := http.NewRequest(http.MethodDelete, "https://localhost/batch-authz/data", nil)
|
||||
mustIAM(t, err)
|
||||
args := policy.Args{
|
||||
AccountName: cred.AccessKey, Claims: p.claims, Action: policy.DeleteObjectVersionAction,
|
||||
BucketName: f.bucket, ObjectName: "data", ConditionValues: getConditionValues(req, "", cred),
|
||||
}
|
||||
if args.ConditionValues["username"][0] != "alice" || f.sys.isAllowedLocally(args) {
|
||||
t.Fatal("control: the request evaluator must deny the verified LDAP short username")
|
||||
}
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.DeleteObjectVersionAction, bucket: f.bucket, object: "data", versionID: mustGetUUID()}); !isBatchJobAuthzError(err) {
|
||||
t.Errorf("batch authorization must preserve the LDAP username deny or fail closed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Failure to read authoritative metadata is incomplete work, even if one
|
||||
// pool's copy can still be opened. In particular, skipErrs=false must not
|
||||
// turn an unreadable entry into a successful archive.
|
||||
func TestBatchReviewSnowballAuthoritativeReadFailure(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const name = "archive/unreadable-copy"
|
||||
before := putConsistencyObject(t, z, bucket, name, 0, "test payload", ObjectOptions{Versioned: true})
|
||||
set := z.serverPools[1].sets[0]
|
||||
getDisks := set.getDisks
|
||||
disks := append([]StorageAPI(nil), getDisks()...)
|
||||
for i := range disks {
|
||||
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: name}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return disks }
|
||||
defer func() { set.getDisks = getDisks }()
|
||||
gr, err := z.GetObjectNInfo(t.Context(), bucket, name, nil, nil, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
_, logicalErr := batchAuthoritativeRead(t.Context(), z, bucket, name, gr.ObjInfo)
|
||||
gr.Close()
|
||||
if logicalErr == nil {
|
||||
t.Fatal("control: the authoritative read must fail")
|
||||
}
|
||||
remote, err := minio.New("archive.invalid", &minio.Options{
|
||||
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
|
||||
}),
|
||||
})
|
||||
mustIAM(t, err)
|
||||
r := BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
}
|
||||
r.Source.Snowball.InMemory = ptr(true)
|
||||
r.Source.Snowball.Compress = ptr(false)
|
||||
r.Source.Snowball.SkipErrs = ptr(false)
|
||||
if err := r.writeAsArchive(t.Context(), z, remote, []ObjectInfo{before}, "", batchRootSubmitter(t)); err == nil {
|
||||
t.Error("archive with an unreadable authoritative version must not report success")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// A suspended destination does not keep a source version ID. Authorization
|
||||
// must therefore describe the null destination version that the write really
|
||||
// replaces, and must not let a source ID escape that model.
|
||||
func TestBatchReviewPullSuspendedVersionBinding(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
_, err := globalBucketMetadataSys.Update(f.ctx, f.bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`))
|
||||
mustIAM(t, err)
|
||||
|
||||
sourceVersion := mustGetUUID()
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "pull-source-version-deny", batchAuthzFullAccess+fmt.Sprintf(`,
|
||||
{"Effect":"Deny","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:versionid":"%s"}}}`, sourceVersion))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "pull-source-version-user", "pull-source-version-deny")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
job := BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
|
||||
}
|
||||
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
|
||||
Name: "pull/suspended-version", VersionID: sourceVersion, ModTime: UTCNow(), ETag: "batchetag",
|
||||
UserDefined: map[string]string{},
|
||||
}, false, sub)
|
||||
if !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("a source version ID must not evade the destination deny: %v", err)
|
||||
}
|
||||
if _, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/suspended-version", ObjectOptions{VersionID: sourceVersion}); gerr == nil {
|
||||
t.Fatal("a denied suspended-destination write created the source version")
|
||||
}
|
||||
}
|
||||
|
||||
// A failed source open in a non-skipping snowball must be reported to the
|
||||
// batch rather than silently omitted from a successful archive.
|
||||
type batchReviewOpenFailure struct {
|
||||
ObjectLayer
|
||||
name string
|
||||
}
|
||||
|
||||
func (a batchReviewOpenFailure) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
if object == a.name {
|
||||
return nil, fmt.Errorf("test: source open failed")
|
||||
}
|
||||
return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
func TestBatchReviewSnowballSourceOpenFailure(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
name := "archive/open-failure"
|
||||
entry := batchPutVersion(t, f.obj, f.bucket, name, nil)
|
||||
remote, err := minio.New("archive.invalid", &minio.Options{
|
||||
Region: "us-east-1", Creds: credentials.NewStaticV4("test-access", "test-secret", ""),
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
|
||||
}),
|
||||
})
|
||||
mustIAM(t, err)
|
||||
r := BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: f.bucket},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
}
|
||||
r.Source.Snowball.InMemory = ptr(true)
|
||||
r.Source.Snowball.Compress = ptr(false)
|
||||
r.Source.Snowball.SkipErrs = ptr(false)
|
||||
api := batchReviewOpenFailure{ObjectLayer: f.obj, name: name}
|
||||
if err := r.writeAsArchive(t.Context(), api, remote, []ObjectInfo{entry}, "", batchRootSubmitter(t)); err == nil {
|
||||
t.Fatal("a source open failure must not be reported as a successful archive")
|
||||
}
|
||||
}
|
||||
|
||||
// Rotation must apply tag conditions to the logical version, including
|
||||
// a more recent tag update stored in a pool other than the selected copy.
|
||||
func TestBatchReviewRotationAuthoritativeTags(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
f := newBatchAuthzFixture(t)
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("batch-round5-rotation-key")
|
||||
t.Cleanup(func() { GlobalKMS = previousKMS })
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "round5-rotation-tags", fmt.Sprintf(`
|
||||
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
|
||||
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "round5-rotation-user", "round5-rotation-tags")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
for _, classification := range []string{"private", "public"} {
|
||||
t.Run(classification, func(t *testing.T) {
|
||||
name := "rotation/authoritative-" + classification
|
||||
first := make(map[string]string)
|
||||
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
|
||||
mustIAM(t, err)
|
||||
first[xhttp.AmzObjectTagging] = "classification=public"
|
||||
first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
|
||||
second := maps.Clone(first)
|
||||
second[xhttp.AmzObjectTagging] = "classification=" + classification
|
||||
second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("test encrypted payload"), key))
|
||||
mustIAM(t, err)
|
||||
before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
|
||||
putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
|
||||
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if logical.UserTags != "classification="+classification {
|
||||
t.Fatal("control: the second pool must govern the logical tags")
|
||||
}
|
||||
job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
|
||||
err = job.KeyRotate(f.ctx, z, before, sub)
|
||||
after, readErr := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, readErr)
|
||||
changed := after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3]
|
||||
if classification == "private" {
|
||||
if !isBatchJobAuthzError(err) || changed {
|
||||
t.Fatalf("authoritative tag deny must prevent rewrapping: err=%v changed=%v", err, changed)
|
||||
}
|
||||
} else if err != nil || !changed {
|
||||
t.Fatalf("authorized unlocked version must rotate: err=%v changed=%v", err, changed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,538 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// Rotation and listing cases found in review.
|
||||
|
||||
// Rotation rewrites the copy GetObjectInfo returns, which can carry another
|
||||
// tag revision than the merged version of all pools: both are authorized.
|
||||
func TestBatchReviewRotationAuthorizesWrittenCopy(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
f := newBatchAuthzFixture(t)
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("review6-rotation-copy-key")
|
||||
t.Cleanup(func() { GlobalKMS = previousKMS })
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "review6-rotation-copy-deny", fmt.Sprintf(`
|
||||
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
|
||||
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "review6-rotation-copy-user", "review6-rotation-copy-deny")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
const name = "rotation/written-copy"
|
||||
first := make(map[string]string)
|
||||
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, bucket, name, first, nil)
|
||||
mustIAM(t, err)
|
||||
first[xhttp.AmzObjectTagging] = "classification=private"
|
||||
first[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
|
||||
second := maps.Clone(first)
|
||||
second[xhttp.AmzObjectTagging] = "classification=public"
|
||||
second[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("review6 rotation payload"), key))
|
||||
mustIAM(t, err)
|
||||
before := putConsistencyObject(t, z, bucket, name, 0, string(ciphertext), ObjectOptions{Versioned: true, UserDefined: first})
|
||||
putConsistencyObject(t, z, bucket, name, 1, string(ciphertext), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: second})
|
||||
logical, err := z.replicaObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if logical.UserTags != "classification=public" {
|
||||
t.Fatalf("control: merged tags=%q, want public", logical.UserTags)
|
||||
}
|
||||
job := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
|
||||
if err = job.KeyRotate(f.ctx, z, before, sub); !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("rotation of a copy the submitter may not read returned %v", err)
|
||||
}
|
||||
after, err := z.serverPools[0].GetObjectInfo(f.ctx, bucket, name, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if after.UserDefined[crypto.MetaSealedKeyS3] != before.UserDefined[crypto.MetaSealedKeyS3] {
|
||||
t.Fatal("a refused rotation rewrote the copy")
|
||||
}
|
||||
}
|
||||
|
||||
// batchWalkError fails every listing before it returns a result.
|
||||
type batchWalkError struct {
|
||||
ObjectLayer
|
||||
err error
|
||||
}
|
||||
|
||||
func (w batchWalkError) Walk(context.Context, string, string, chan<- itemOrErr[ObjectInfo], WalkOptions) error {
|
||||
return w.err
|
||||
}
|
||||
|
||||
// A listing outage pauses the run, as a streamed listing error does.
|
||||
func TestBatchReviewExpireWalkErrorPausesRun(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
rec, job := f.submit(t, globalActiveCred, f.expireYAML("data/"))
|
||||
if job == nil {
|
||||
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
listErr := errors.New("review6: listing failed")
|
||||
if err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job); !errors.Is(err, listErr) {
|
||||
t.Fatalf("start returned %v, want %v", err, listErr)
|
||||
}
|
||||
if ri := f.jobInfo(t, job); ri.Failed || ri.Complete || ri.PausedReason == "" {
|
||||
t.Fatalf("failed listing: complete=%v failed=%v", ri.Complete, ri.Failed)
|
||||
}
|
||||
}
|
||||
|
||||
// Two pools hold different writes of the null version, and a newer version
|
||||
// of the key is in the second one. Rotation rewrites the copy it read and
|
||||
// authorized, in the first pool, not the copy the latest version of the key
|
||||
// would route a write to; the other write keeps its key and stays readable.
|
||||
func TestBatchReviewRotationRewritesTheCopyItRead(t *testing.T) {
|
||||
for _, otherLatest := range []bool{false, true} {
|
||||
name := "addressed-version-is-latest"
|
||||
if otherLatest {
|
||||
name = "another-version-is-latest"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
t.Cleanup(resetTestGlobals)
|
||||
z, _ := consistencyPools(t)
|
||||
globalIAMSys.initStore(z, nil)
|
||||
mustIAM(t, globalIAMSys.Load(t.Context(), true))
|
||||
const bucket = "review7-rotation-copies"
|
||||
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
_, versionErr := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||
mustIAM(t, versionErr)
|
||||
oldKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("review7-rotation-key")
|
||||
t.Cleanup(func() { GlobalKMS = oldKMS })
|
||||
const object = "rotation/null-copy"
|
||||
const payload = "rotation must preserve this data"
|
||||
readCopy := func(pool int) ([]byte, error) {
|
||||
gr, err := z.serverPools[pool].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: nullVersionID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer gr.Close()
|
||||
return io.ReadAll(gr)
|
||||
}
|
||||
var before [2]ObjectInfo
|
||||
var keys [2]crypto.ObjectKey
|
||||
kinds := []crypto.Type{crypto.S3, crypto.S3KMS}
|
||||
for pool, kind := range kinds {
|
||||
meta := make(map[string]string)
|
||||
key, err := newEncryptMetadata(t.Context(), kind, "review7-rotation-key", nil, bucket, object, meta, nil)
|
||||
mustIAM(t, err)
|
||||
keys[pool] = key
|
||||
meta[xhttp.AmzObjectTagging] = "classification=public"
|
||||
if pool == 1 {
|
||||
meta[xhttp.AmzObjectTagging] = "classification=private"
|
||||
}
|
||||
meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Duration(pool+2) * time.Hour).Format(time.RFC3339Nano)
|
||||
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
|
||||
mustIAM(t, err)
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{
|
||||
Versioned: true, VersionID: nullVersionID, UserDefined: meta,
|
||||
MTime: UTCNow().Add(-time.Duration(pool+2) * time.Hour),
|
||||
})
|
||||
// As a read returns it, without the write's internal fields.
|
||||
before[pool], err = z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
|
||||
mustIAM(t, err)
|
||||
data, err := readCopy(pool)
|
||||
if err != nil || string(data) != payload {
|
||||
t.Fatalf("control: pool %d was not readable before rotation: %v", pool, err)
|
||||
}
|
||||
}
|
||||
if otherLatest {
|
||||
putConsistencyObject(t, z, bucket, object, 1, "later version", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
}
|
||||
selected, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID, Versioned: true})
|
||||
mustIAM(t, err)
|
||||
if selected.VersionID != nullVersionID || !selected.ModTime.Equal(before[0].ModTime) || !crypto.S3.IsEncrypted(selected.UserDefined) {
|
||||
t.Fatal("control: the addressed null version must select the SSE-S3 write in pool zero")
|
||||
}
|
||||
logical, err := batchAuthoritativeVersion(t.Context(), z, bucket, object, selected)
|
||||
mustIAM(t, err)
|
||||
if logical.UserTags != "classification=public" || selected.UserTags != logical.UserTags {
|
||||
t.Fatal("control: both current authorization inputs must have public tags")
|
||||
}
|
||||
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review7-rotation-key"}}
|
||||
sub := batchRootSubmitter(t)
|
||||
if otherLatest {
|
||||
batchAuthzPolicy(t.Context(), t, globalIAMSys, "review7-rotate-public", fmt.Sprintf(`
|
||||
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectVersion","s3:PutObject"],"Resource":["arn:aws:s3:::%s/*"]},
|
||||
{"Effect":"Deny","Action":["s3:GetObject","s3:GetObjectVersion"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/classification":"private"}}}`, bucket, bucket))
|
||||
cred := iamAdminUser(t.Context(), t, globalIAMSys, "review7-rotate-user", "review7-rotate-public")
|
||||
sub, err = newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
if err := sub.authorize(t.Context(), r.rotateAuthzRequests(before[1])...); !isBatchJobAuthzError(err) {
|
||||
t.Fatal("control: the older write in pool one must be denied")
|
||||
}
|
||||
}
|
||||
err = r.KeyRotate(t.Context(), z, selected, sub)
|
||||
if err != nil {
|
||||
t.Logf("rotation refused: %v", err)
|
||||
}
|
||||
for pool, want := range keys {
|
||||
after, readErr := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID})
|
||||
mustIAM(t, readErr)
|
||||
if otherLatest && pool == 1 && (!maps.Equal(after.UserDefined, before[pool].UserDefined) || !after.ModTime.Equal(before[pool].ModTime) || after.UserTags != before[pool].UserTags) {
|
||||
t.Error("rotation rewrote a copy denied by the submitter's tag-conditioned policy")
|
||||
}
|
||||
kind, _ := crypto.IsEncrypted(after.UserDefined)
|
||||
var got crypto.ObjectKey
|
||||
switch kind {
|
||||
case crypto.S3:
|
||||
got, readErr = crypto.S3.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
|
||||
case crypto.S3KMS:
|
||||
got, readErr = crypto.S3KMS.UnsealObjectKey(GlobalKMS, after.UserDefined, bucket, object)
|
||||
default:
|
||||
t.Fatalf("pool %d: rotation lost encryption metadata", pool)
|
||||
}
|
||||
mustIAM(t, readErr)
|
||||
if got != want {
|
||||
t.Errorf("pool %d: rotation changed the data key without reencrypting its ciphertext (rotation error: %v)", pool, err)
|
||||
}
|
||||
data, readErr := readCopy(pool)
|
||||
if readErr != nil || string(data) != payload {
|
||||
t.Errorf("pool %d: rotation left unreadable ciphertext: %v", pool, readErr)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A version in a pool that a decommission or a rebalance is moving is not
|
||||
// rewritten there: only the source holds it, so the rotation is left to a
|
||||
// later run.
|
||||
func TestBatchReviewRotationLeavesMovingSource(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
t.Cleanup(resetTestGlobals)
|
||||
z, _ := consistencyPools(t)
|
||||
globalIAMSys.initStore(z, nil)
|
||||
mustIAM(t, globalIAMSys.Load(t.Context(), true))
|
||||
const bucket = "review8-rotation-draining"
|
||||
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||
mustIAM(t, err)
|
||||
oldKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("review8-rotation-key")
|
||||
t.Cleanup(func() { GlobalKMS = oldKMS })
|
||||
const object = "rotation/draining"
|
||||
meta := make(map[string]string)
|
||||
key, err := newEncryptMetadata(t.Context(), crypto.S3, "review8-rotation-key", nil, bucket, object, meta, nil)
|
||||
mustIAM(t, err)
|
||||
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("draining rotation"), key))
|
||||
mustIAM(t, err)
|
||||
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
|
||||
beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
|
||||
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
|
||||
err = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
|
||||
if !errors.Is(err, errBatchRotateMoving) {
|
||||
t.Errorf("rotation of a version only a draining pool holds returned %v", err)
|
||||
}
|
||||
after, readErr := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, readErr)
|
||||
if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
|
||||
t.Fatal("rotation rewrote a draining pool")
|
||||
}
|
||||
// The job pauses and resumes in-process once the selected pool stops moving.
|
||||
job := BatchJobRequest{ID: "rotation-moving-resume", Started: UTCNow(), Submitter: batchRootSubmitter(t), KeyRotate: &r}
|
||||
batchTestJobPool(t.Context(), t, z)
|
||||
if err := r.Start(t.Context(), z, job); !isBatchJobUnavailable(err) {
|
||||
t.Fatalf("moving job did not pause: %v", err)
|
||||
}
|
||||
ri := &batchJobInfo{}
|
||||
mustIAM(t, ri.load(t.Context(), z, job))
|
||||
if ri.PausedReason == "" || ri.Failed || ri.Complete || ri.Object != "" || ri.ObjectsFailed != 0 {
|
||||
t.Fatalf("moving job report: %+v", ri)
|
||||
}
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[0].Decommission = nil
|
||||
z.poolMetaMutex.Unlock()
|
||||
mustIAM(t, r.Start(t.Context(), z, job))
|
||||
mustIAM(t, ri.load(t.Context(), z, job))
|
||||
if !ri.Complete || ri.Failed || ri.PausedReason != "" || ri.Objects != 1 {
|
||||
t.Fatalf("resumed rotation: %+v", ri)
|
||||
}
|
||||
}
|
||||
|
||||
// Once a move copied a version, the rotation leaves it to a later run, so the
|
||||
// source stays as the move's cleanup expects it.
|
||||
func TestBatchReviewRotationKeepsMigrationCleanup(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
kind string
|
||||
rotate bool
|
||||
}{{"decommission", false}, {"decommission", true}, {"rebalance", false}, {"rebalance", true}} {
|
||||
name := tc.kind + "/without-rotation"
|
||||
if tc.rotate {
|
||||
name = tc.kind + "/with-rotation"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
t.Cleanup(resetTestGlobals)
|
||||
z, _ := migrationTestPools(t)
|
||||
globalIAMSys.initStore(z, nil)
|
||||
mustIAM(t, globalIAMSys.Load(t.Context(), true))
|
||||
const bucket = "review8-rotation-migration"
|
||||
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||
mustIAM(t, err)
|
||||
oldKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("review8-rotation-key")
|
||||
t.Cleanup(func() { GlobalKMS = oldKMS })
|
||||
const object = "rotation/migration"
|
||||
meta := make(map[string]string)
|
||||
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
|
||||
mustIAM(t, err)
|
||||
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation and migration"), key))
|
||||
mustIAM(t, err)
|
||||
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
|
||||
beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
move := migrationTestMover(t, z, 0, tc.kind)
|
||||
mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, beforeInfo)))
|
||||
if _, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); err != nil {
|
||||
t.Fatalf("control: migration did not copy the version: %v", err)
|
||||
}
|
||||
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: ssekms, Key: "review8-rotation-key"}}
|
||||
var rotationErr error
|
||||
if tc.rotate {
|
||||
moved, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
rotationErr = r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
|
||||
if !errors.Is(rotationErr, errBatchRotateMoving) {
|
||||
t.Errorf("rotation during a move returned %v", rotationErr)
|
||||
}
|
||||
after, err := z.serverPools[1].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if !maps.Equal(after.UserDefined, moved.UserDefined) || after.UserTags != moved.UserTags {
|
||||
t.Error("rotation during a move rewrote the moved copy")
|
||||
}
|
||||
}
|
||||
after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if !after.ModTime.Equal(beforeInfo.ModTime) || !maps.Equal(after.UserDefined, beforeInfo.UserDefined) {
|
||||
t.Errorf("rotation changed a draining source after its version was copied (rotation error: %v)", rotationErr)
|
||||
}
|
||||
snapshot := []FileInfo{{VersionID: before.VersionID, ModTime: beforeInfo.ModTime}}
|
||||
if err := z.removeMovedVersions(t.Context(), bucket, object, 0, snapshot, nil); err != nil {
|
||||
t.Fatalf("rotation prevented migration cleanup: %v", err)
|
||||
}
|
||||
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Fatalf("migration left a source version behind: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The rewrite keeps the tags and the expiry of the version, which a read
|
||||
// returns apart from its metadata.
|
||||
func TestBatchReviewRotationKeepsTagsAndExpiry(t *testing.T) {
|
||||
for _, mode := range []string{"single-pool", "multiple-pools"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
t.Cleanup(resetTestGlobals)
|
||||
var z *erasureServerPools
|
||||
if mode == "single-pool" {
|
||||
z = purgeSinglePool(t, 4)
|
||||
} else {
|
||||
z, _ = consistencyPools(t)
|
||||
}
|
||||
globalIAMSys.initStore(z, nil)
|
||||
mustIAM(t, globalIAMSys.Load(t.Context(), true))
|
||||
const bucket = "review8-rotation-metadata"
|
||||
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||
mustIAM(t, err)
|
||||
oldKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("review8-rotation-metadata-key")
|
||||
t.Cleanup(func() { GlobalKMS = oldKMS })
|
||||
const object = "rotation/tagged"
|
||||
meta := make(map[string]string)
|
||||
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
|
||||
mustIAM(t, err)
|
||||
meta[xhttp.AmzObjectTagging] = "classification=private"
|
||||
meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
|
||||
meta["expires"] = UTCNow().Add(time.Hour).Format(http.TimeFormat)
|
||||
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("metadata preservation"), key))
|
||||
mustIAM(t, err)
|
||||
written := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta})
|
||||
before, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: written.VersionID})
|
||||
mustIAM(t, err)
|
||||
if before.UserTags == "" || before.Expires.IsZero() {
|
||||
t.Fatal("control: stored version is missing the metadata to preserve")
|
||||
}
|
||||
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
|
||||
mustIAM(t, r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t)))
|
||||
after, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if after.UserDefined[crypto.MetaSealedKeyS3] == before.UserDefined[crypto.MetaSealedKeyS3] {
|
||||
t.Error("control: rotation did not rewrap the key")
|
||||
}
|
||||
if after.UserTags != before.UserTags {
|
||||
t.Errorf("rotation changed stored tags: got %q, want %q", after.UserTags, before.UserTags)
|
||||
}
|
||||
if !after.Expires.Equal(before.Expires) {
|
||||
t.Errorf("rotation changed Expires: got %v, want %v", after.Expires, before.Expires)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Two pools hold the same write with different tags, with or without
|
||||
// tagging timestamps. A rotation, also during a move, leaves the tags that
|
||||
// the merged version has.
|
||||
func TestBatchReviewRotationKeepsMergedTags(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
ordered, moving bool
|
||||
}{{false, false}, {false, true}, {true, false}, {true, true}} {
|
||||
t.Run(fmt.Sprintf("timestamp=%t/moving=%t", tc.ordered, tc.moving), func(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
t.Cleanup(resetTestGlobals)
|
||||
z, _ := consistencyPools(t)
|
||||
globalIAMSys.initStore(z, nil)
|
||||
mustIAM(t, globalIAMSys.Load(t.Context(), true))
|
||||
const bucket = "review9-rotation-tag-tie"
|
||||
const object = "rotation/tag-tie"
|
||||
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||
mustIAM(t, err)
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("review9-tag-tie-key")
|
||||
t.Cleanup(func() { GlobalKMS = previousKMS })
|
||||
meta := make(map[string]string)
|
||||
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
|
||||
mustIAM(t, err)
|
||||
meta[xhttp.AmzObjectTagging] = "classification=private"
|
||||
if tc.ordered {
|
||||
meta[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Add(-time.Minute).Format(time.RFC3339Nano)
|
||||
}
|
||||
other := maps.Clone(meta)
|
||||
other[xhttp.AmzObjectTagging] = "classification=public"
|
||||
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("same write"), key))
|
||||
mustIAM(t, err)
|
||||
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{Versioned: true, UserDefined: meta, MTime: UTCNow().Add(-time.Hour)})
|
||||
putConsistencyObject(t, z, bucket, object, 1, string(body), ObjectOptions{Versioned: true, VersionID: before.VersionID, MTime: before.ModTime, UserDefined: other})
|
||||
prior, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if prior.UserTags != "classification=private" {
|
||||
t.Fatalf("control: logical tags=%q", prior.UserTags)
|
||||
}
|
||||
if tc.moving {
|
||||
migrationTestMover(t, z, 0, "decommission")
|
||||
}
|
||||
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
|
||||
rotationErr := r.KeyRotate(t.Context(), z, before, batchRootSubmitter(t))
|
||||
after, err := z.replicaObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if after.UserTags != prior.UserTags {
|
||||
t.Fatalf("rotation changed logical tags from %q to %q (rotation error: %v)", prior.UserTags, after.UserTags, rotationErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// batchRotateFlipDisk starts a decommission of pool 0 at the flipAt-th
|
||||
// version read, after the rotation's first read of the version.
|
||||
type batchRotateFlipDisk struct {
|
||||
StorageAPI
|
||||
z *erasureServerPools
|
||||
reads *atomic.Int32
|
||||
flipAt int32
|
||||
flipped *atomic.Bool
|
||||
}
|
||||
|
||||
func (d batchRotateFlipDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
|
||||
n := d.reads.Add(1)
|
||||
if n == d.flipAt && d.flipped.CompareAndSwap(false, true) {
|
||||
d.z.poolMetaMutex.Lock()
|
||||
d.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
|
||||
d.z.poolMetaMutex.Unlock()
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
|
||||
}
|
||||
|
||||
// A move that starts while the rotation authorizes and rewraps the version
|
||||
// defers it: the state is checked again right before the write, where the
|
||||
// pools layer checks it.
|
||||
func TestBatchReviewRotationRechecksPoolBeforeWrite(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
t.Cleanup(resetTestGlobals)
|
||||
z, _ := consistencyPools(t)
|
||||
globalIAMSys.initStore(z, nil)
|
||||
mustIAM(t, globalIAMSys.Load(t.Context(), true))
|
||||
const bucket = "review11-rotation-window"
|
||||
const object = "rotation/window"
|
||||
mustIAM(t, z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
|
||||
mustIAM(t, err)
|
||||
oldKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("review11-rotation-key")
|
||||
t.Cleanup(func() { GlobalKMS = oldKMS })
|
||||
|
||||
meta := make(map[string]string)
|
||||
key, err := newEncryptMetadata(t.Context(), crypto.S3, "", nil, bucket, object, meta, nil)
|
||||
mustIAM(t, err)
|
||||
body, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader("rotation window"), key))
|
||||
mustIAM(t, err)
|
||||
before := putConsistencyObject(t, z, bucket, object, 0, string(body), ObjectOptions{
|
||||
Versioned: true,
|
||||
UserDefined: meta,
|
||||
MTime: UTCNow().Add(-time.Hour),
|
||||
})
|
||||
beforeInfo, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
|
||||
var reads atomic.Int32
|
||||
var flipped atomic.Bool
|
||||
firstPass := int32(0)
|
||||
for _, pool := range z.serverPools {
|
||||
firstPass += int32(pool.SetDriveCount())
|
||||
}
|
||||
var restore []func()
|
||||
for _, pool := range z.serverPools {
|
||||
restore = append(restore, conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI {
|
||||
return batchRotateFlipDisk{
|
||||
StorageAPI: d,
|
||||
z: z,
|
||||
reads: &reads,
|
||||
flipAt: firstPass + 1,
|
||||
flipped: &flipped,
|
||||
}
|
||||
}))
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, fn := range restore {
|
||||
fn()
|
||||
}
|
||||
})
|
||||
|
||||
r := BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
|
||||
gotErr := r.KeyRotate(t.Context(), z, beforeInfo, batchRootSubmitter(t))
|
||||
if !flipped.Load() {
|
||||
t.Fatal("fault injection did not occur after the initial rotation read")
|
||||
}
|
||||
if !errors.Is(gotErr, errBatchRotateMoving) {
|
||||
t.Errorf("rotation did not defer after the pool became moving during pre-work: %v", gotErr)
|
||||
}
|
||||
after, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: before.VersionID})
|
||||
mustIAM(t, err)
|
||||
if after.UserDefined[crypto.MetaSealedKeyS3] != beforeInfo.UserDefined[crypto.MetaSealedKeyS3] {
|
||||
t.Error("rotation rewrote the pool after it became moving during pre-work")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func batchPullJob(bucket string) BatchJobReplicateV1 {
|
||||
return BatchJobReplicateV1{
|
||||
Source: BatchJobReplicateSource{Type: BatchJobReplicateResourceMinIO, Bucket: "remote"},
|
||||
Target: BatchJobReplicateTarget{Type: BatchJobReplicateResourceMinIO, Bucket: bucket},
|
||||
}
|
||||
}
|
||||
|
||||
// A pull does not read the destination version it may replace, so a policy
|
||||
// that conditions the write on that version's tags refuses it; a version
|
||||
// condition sees the version the pull writes.
|
||||
func TestBatchPullDestinationConditions(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
action policy.Action
|
||||
marker bool
|
||||
condition func(versionID string) string
|
||||
}{
|
||||
{"tags-put", policy.ReplicateObjectAction, false, func(string) string {
|
||||
return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
|
||||
}},
|
||||
{"tags-marker", policy.ReplicateDeleteAction, true, func(string) string {
|
||||
return `{"StringEquals":{"s3:ExistingObjectTag/class":"protected"}}`
|
||||
}},
|
||||
{"version-put", policy.PutObjectAction, false, func(v string) string {
|
||||
return fmt.Sprintf(`{"StringEquals":{"s3:versionid":%q}}`, v)
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
before := batchPutVersion(t, f.obj, f.bucket, "pull/guarded", map[string]string{xhttp.AmzObjectTagging: "class=protected"})
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "pull-guard", batchAuthzFullAccess+fmt.Sprintf(`,
|
||||
{"Effect":"Deny","Action":[%q],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":%s}`, tc.action, tc.condition(before.VersionID)))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "pull-guard-user", "pull-guard")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
job := batchPullJob(f.bucket)
|
||||
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "replacement"), ObjectInfo{
|
||||
Name: before.Name, VersionID: before.VersionID, ModTime: UTCNow(), ETag: "batchetag",
|
||||
DeleteMarker: tc.marker, UserDefined: map[string]string{},
|
||||
}, false, sub)
|
||||
if !isBatchJobAuthzError(err) {
|
||||
t.Errorf("pull returned %v, want a refusal", err)
|
||||
}
|
||||
after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, before.Name, ObjectOptions{VersionID: before.VersionID})
|
||||
if err != nil || after.ETag != before.ETag {
|
||||
t.Errorf("refused pull changed the destination version: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A pull does not encrypt what it writes; encryption headers reported by the
|
||||
// source do not satisfy a destination policy that requires encryption.
|
||||
func TestBatchPullEncryptionCondition(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "pull-encryption", `
|
||||
{"Effect":"Allow","Action":["s3:ReplicateObject"],"Resource":["arn:aws:s3:::batch-authz/*"]},
|
||||
{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:x-amz-server-side-encryption":"aws:kms","s3:x-amz-server-side-encryption-aws-kms-key-id":"required-key"}}}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "pull-encryption-user", "pull-encryption")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
job := batchPullJob(f.bucket)
|
||||
version := mustGetUUID()
|
||||
err = job.ReplicateFromSource(f.ctx, f.obj, batchLockRemote(t, "content"), ObjectInfo{
|
||||
Name: "pull/encrypted", VersionID: version, ModTime: UTCNow(), ETag: "batchetag",
|
||||
UserDefined: map[string]string{xhttp.AmzServerSideEncryption: "aws:kms", xhttp.AmzServerSideEncryptionKmsID: "required-key"},
|
||||
}, false, sub)
|
||||
if err == nil {
|
||||
oi, gerr := f.obj.GetObjectInfo(f.ctx, f.bucket, "pull/encrypted", ObjectOptions{VersionID: version})
|
||||
mustIAM(t, gerr)
|
||||
if !crypto.S3KMS.IsEncrypted(oi.UserDefined) {
|
||||
t.Fatal("a pull was allowed by an encryption condition it did not meet")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A conditional Deny on an S3 Tables action covers the S3 data actions it
|
||||
// implies, under the tables resource form; the job refuses rather than skip it.
|
||||
func TestBatchJobTableConditionalDeny(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "table-context", batchAuthzFullAccess+`,
|
||||
{"Effect":"Deny","Action":["s3tables:GetTableData"],"Resource":["arn:aws:s3tables:::bucket/batch-authz/table/table-id"],"Condition":{"Bool":{"aws:SecureTransport":"false"}}}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "table-context-user", "table-context")
|
||||
if f.sys.isAllowedLocally(policy.Args{AccountName: cred.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "table-id/data", ConditionValues: map[string][]string{"SecureTransport": {"false"}}}) {
|
||||
t.Fatal("control: with the request context the Deny applies")
|
||||
}
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "table-id/data"}); !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("authorize returned %v, want a refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
type batchArchiveRevocation struct {
|
||||
ObjectLayer
|
||||
revoke func()
|
||||
}
|
||||
|
||||
func (a *batchArchiveRevocation) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
if object == "archive/second" {
|
||||
a.revoke()
|
||||
}
|
||||
return a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
// Each snowball entry is authorized for the submitter as it is then.
|
||||
func TestBatchPushSnowballEntryRevalidation(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "archive-user", "")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
pluginURL, err := xnet.ParseHTTPURL("https://authz.invalid/")
|
||||
mustIAM(t, err)
|
||||
previous := newGlobalAuthZPluginFn()
|
||||
setGlobalAuthZPlugin(polplugin.New(polplugin.Args{
|
||||
URL: pluginURL, CloseRespFn: func(r io.ReadCloser) { _ = r.Close() },
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"result":true}`)), Request: r}, nil
|
||||
}),
|
||||
}))
|
||||
t.Cleanup(func() { setGlobalAuthZPlugin(previous) })
|
||||
entries := []ObjectInfo{
|
||||
batchPutVersion(t, f.obj, f.bucket, "archive/first", nil),
|
||||
batchPutVersion(t, f.obj, f.bucket, "archive/second", nil),
|
||||
}
|
||||
remote, err := minio.New("archive.invalid", &minio.Options{
|
||||
Region: "us-east-1", Creds: credentials.NewStaticV4("batch-access", "batch-secret", ""),
|
||||
Transport: batchLockTransport(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Body != nil {
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
|
||||
}),
|
||||
})
|
||||
mustIAM(t, err)
|
||||
job := BatchJobReplicateV1{Source: BatchJobReplicateSource{Bucket: f.bucket}, Target: BatchJobReplicateTarget{Bucket: "remote"}}
|
||||
job.Source.Snowball.InMemory = ptr(true)
|
||||
job.Source.Snowball.Compress = ptr(false)
|
||||
job.Source.Snowball.SkipErrs = ptr(false)
|
||||
api := &batchArchiveRevocation{ObjectLayer: f.obj, revoke: func() { mustIAM(t, f.sys.DeleteUser(f.ctx, cred.AccessKey, false)) }}
|
||||
if err = job.writeAsArchive(f.ctx, api, remote, entries, "", sub); !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("archive with a revoked submitter returned %v, want a refusal", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A definition holds one job type; one with several is refused, so that what
|
||||
// is authorized is what runs.
|
||||
func TestBatchJobSeveralSections(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
older := f.putVersions(t, "data/a")
|
||||
job := f.expireYAML("") + `keyrotate:
|
||||
apiVersion: v1
|
||||
bucket: other
|
||||
encryption:
|
||||
type: sse-s3
|
||||
`
|
||||
rec, queued := f.submit(t, globalActiveCred, job)
|
||||
if queued != nil {
|
||||
f.run(t, queued)
|
||||
}
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("submission status %d, want 400: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !f.versionExists(t, "data/a", older) {
|
||||
t.Fatal("a definition with two job types ran")
|
||||
}
|
||||
}
|
||||
|
||||
// Prefixes are walked once each in one order: a cursor in a later prefix is
|
||||
// after every key of the earlier ones, and an overlapping prefix is not
|
||||
// walked twice.
|
||||
func TestBatchExpirePrefixes(t *testing.T) {
|
||||
if got := batchCanonicalPrefixes([]string{"z/", "a/", "a/b/", "z/", ""}); len(got) != 1 || got[0] != "" {
|
||||
t.Fatalf("canonical prefixes %q", got)
|
||||
}
|
||||
if got := batchCanonicalPrefixes([]string{"z/", "a/b/", "a/"}); strings.Join(got, ",") != "a/,z/" {
|
||||
t.Fatalf("canonical prefixes %q", got)
|
||||
}
|
||||
f := newBatchAuthzFixture(t)
|
||||
done := f.putVersions(t, "a/done")
|
||||
todo := f.putVersions(t, "z/todo")
|
||||
batchPutVersion(t, f.obj, f.bucket, "x/data", batchLockMeta("LEGALHOLD"))
|
||||
batchPutVersion(t, f.obj, f.bucket, "x/data", nil)
|
||||
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf(`expire:
|
||||
apiVersion: v1
|
||||
bucket: %s
|
||||
prefix: ["z/", "x/", "a/", "x/"]
|
||||
rules:
|
||||
- type: object
|
||||
purge:
|
||||
retainVersions: 1
|
||||
`, f.bucket))
|
||||
if job == nil {
|
||||
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
// A run that completed a/ and stopped.
|
||||
ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Bucket: f.bucket, Object: "a/done"}
|
||||
mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
|
||||
f.run(t, job)
|
||||
if !f.versionExists(t, "a/done", done) {
|
||||
t.Error("resume did again a key before the cursor")
|
||||
}
|
||||
if f.versionExists(t, "z/todo", todo) {
|
||||
t.Error("resume skipped a later prefix")
|
||||
}
|
||||
if got := f.jobInfo(t, job).VersionsSkippedLocked; got != 1 {
|
||||
t.Errorf("a locked version under two equal prefixes was counted %d times", got)
|
||||
}
|
||||
}
|
||||
|
||||
type batchPartialWalk struct {
|
||||
ObjectLayer
|
||||
version ObjectInfo
|
||||
cancel context.CancelFunc
|
||||
read chan struct{}
|
||||
}
|
||||
|
||||
func (a *batchPartialWalk) Walk(_ context.Context, _ string, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
|
||||
go func() {
|
||||
defer close(results)
|
||||
results <- itemOrErr[ObjectInfo]{Item: a.version}
|
||||
if a.cancel != nil {
|
||||
<-a.read
|
||||
a.cancel()
|
||||
return
|
||||
}
|
||||
results <- itemOrErr[ObjectInfo]{Err: errors.New("test: listing failed within a key")}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *batchPartialWalk) GetObjectInfo(context.Context, string, string, ObjectOptions) (ObjectInfo, error) {
|
||||
if a.read != nil {
|
||||
close(a.read)
|
||||
}
|
||||
return a.version, nil
|
||||
}
|
||||
|
||||
// A rotation whose walk fails or is canceled within a key keeps that key
|
||||
// after its cursor and does not report completion.
|
||||
func TestBatchKeyRotationIncompleteWalk(t *testing.T) {
|
||||
for _, canceled := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("canceled=%v", canceled), func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
defer cancel()
|
||||
meta := batchLockMeta("LEGALHOLD")
|
||||
meta[crypto.MetaSealedKeyS3] = "test-sealed-key"
|
||||
api := &batchPartialWalk{ObjectLayer: f.obj, version: ObjectInfo{Bucket: f.bucket, Name: "rotation/key", VersionID: mustGetUUID(), UserDefined: meta}}
|
||||
if canceled {
|
||||
api.cancel, api.read = cancel, make(chan struct{})
|
||||
}
|
||||
job := BatchJobRequest{
|
||||
ID: "keyrotate-partial" + getKeySeparator() + "0", Started: UTCNow(), Submitter: batchRootSubmitter(t),
|
||||
KeyRotate: &BatchJobKeyRotateV1{Bucket: f.bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}},
|
||||
}
|
||||
_ = job.KeyRotate.Start(ctx, api, job)
|
||||
ri := f.jobInfo(t, &job)
|
||||
if ri.Complete || ri.Object == api.version.Name || ri.VersionsSkippedLocked != 0 {
|
||||
t.Fatalf("incomplete key committed: complete=%v cursor=%q skips=%d", ri.Complete, ri.Object, ri.VersionsSkippedLocked)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
// batchBlockedIAMStore holds the lazy load of one user's policy mapping, or
|
||||
// of one policy document, until released or until the load's context ends.
|
||||
type batchBlockedIAMStore struct {
|
||||
IAMStorageAPI
|
||||
mapping string // access key whose mapping load blocks
|
||||
doc string // policy whose document load blocks
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (s *batchBlockedIAMStore) block(ctx context.Context) error {
|
||||
s.once.Do(func() { close(s.started) })
|
||||
select {
|
||||
case <-s.release:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *batchBlockedIAMStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
|
||||
if name == s.mapping && !isGroup {
|
||||
if err := s.block(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.IAMStorageAPI.loadMappedPolicy(ctx, name, userType, isGroup, m)
|
||||
}
|
||||
|
||||
func (s *batchBlockedIAMStore) loadPolicyDoc(ctx context.Context, name string, out map[string]PolicyDoc) error {
|
||||
if name == s.doc {
|
||||
if err := s.block(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.IAMStorageAPI.loadPolicyDoc(ctx, name, out)
|
||||
}
|
||||
|
||||
// The batch path and the request path share one IAM cache. A lazy load that
|
||||
// batch authorization needs, of a mapping that is never cached (a user whose
|
||||
// policies come from a group) or of an evicted policy document, must not
|
||||
// hold request authentication and cached authorization while storage is
|
||||
// slow, and must end at the deadline of the job's context.
|
||||
func TestBatchAuthorizeSlowLoadKeepsCachedAuthorizationAvailable(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
// submitter returns the submitter whose authorization lazily loads,
|
||||
// and arranges for blocked to hold that load.
|
||||
submitter func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter
|
||||
}{
|
||||
{"mapping", func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter {
|
||||
// No mapping of the user itself is stored, so every lookup of
|
||||
// its policies reads storage before it falls back to the group.
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "snapshot-group-user", "")
|
||||
_, err := f.sys.AddUsersToGroup(f.ctx, "snapshot-group", []string{cred.AccessKey})
|
||||
mustIAM(t, err)
|
||||
_, err = f.sys.PolicyDBSet(f.ctx, "snapshot-group", "snapshot-full", regUser, true)
|
||||
mustIAM(t, err)
|
||||
blocked.mapping = cred.AccessKey
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
return sub
|
||||
}},
|
||||
{"document", func(t *testing.T, f *batchAuthzFixture, blocked *batchBlockedIAMStore) *batchJobSubmitter {
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "snapshot-evicted", batchAuthzFullAccess)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "snapshot-doc-user", "snapshot-evicted")
|
||||
cache := f.sys.store.lock()
|
||||
delete(cache.iamPolicyDocsMap, "snapshot-evicted")
|
||||
f.sys.store.unlock()
|
||||
blocked.doc = "snapshot-evicted"
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
return sub
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "snapshot-full", batchAuthzFullAccess)
|
||||
cached := iamAdminUser(f.ctx, t, f.sys, "snapshot-cached-user", "snapshot-full")
|
||||
store := f.sys.store
|
||||
original := store.IAMStorageAPI
|
||||
blocked := &batchBlockedIAMStore{IAMStorageAPI: original, started: make(chan struct{}), release: make(chan struct{})}
|
||||
sub := tc.submitter(t, f, blocked)
|
||||
req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
|
||||
if err := sub.authorize(f.ctx, req); err != nil {
|
||||
t.Fatalf("control: %v", err)
|
||||
}
|
||||
store.IAMStorageAPI = blocked
|
||||
var wg sync.WaitGroup
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(blocked.release) }) }
|
||||
t.Cleanup(func() { unblock(); wg.Wait(); store.IAMStorageAPI = original })
|
||||
|
||||
slowDone := make(chan error, 1)
|
||||
wg.Go(func() { slowDone <- sub.authorize(f.ctx, req) })
|
||||
select {
|
||||
case <-blocked.started:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the lazy load did not start")
|
||||
}
|
||||
cachedDone := make(chan bool, 2)
|
||||
wg.Go(func() {
|
||||
_, ok := f.sys.GetUser(f.ctx, cached.AccessKey)
|
||||
cachedDone <- ok
|
||||
})
|
||||
wg.Go(func() {
|
||||
cachedDone <- f.sys.IsAllowed(policy.Args{AccountName: cached.AccessKey, Action: policy.GetObjectAction, BucketName: f.bucket, ObjectName: "key"})
|
||||
})
|
||||
for range 2 {
|
||||
select {
|
||||
case ok := <-cachedDone:
|
||||
if !ok {
|
||||
t.Fatal("cached authentication or authorization was rejected")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("a slow IAM load during batch authorization blocked cached authentication")
|
||||
}
|
||||
}
|
||||
unblock()
|
||||
if err := <-slowDone; err != nil {
|
||||
t.Fatalf("authorization after the load completed: %v", err)
|
||||
}
|
||||
|
||||
// A load that does not return ends with the job's context, as an
|
||||
// unavailable dependency, not with the storage layer's patience.
|
||||
blocked.release = make(chan struct{})
|
||||
once = sync.Once{}
|
||||
unblock = func() { once.Do(func() { close(blocked.release) }) }
|
||||
ctx, cancel := context.WithTimeout(f.ctx, 200*time.Millisecond)
|
||||
defer cancel()
|
||||
deadlineDone := make(chan error, 1)
|
||||
wg.Go(func() { deadlineDone <- sub.authorize(ctx, req) })
|
||||
select {
|
||||
case err := <-deadlineDone:
|
||||
if !isBatchJobUnavailable(err) {
|
||||
t.Fatalf("a load that ended with the job's context: %v", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("a lazy load outlived the job's context")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A policy that a mapping still names after it was deleted takes no part in
|
||||
// the decision, for a batch job as for a request: the remaining mapped
|
||||
// policies decide. An STS parent user's mapping can name a deleted policy,
|
||||
// because deleting a policy checks only user and group mappings for use.
|
||||
func TestBatchAuthorizeSkipsDeletedMappedPolicy(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "mapped-full", batchAuthzFullAccess)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "mapped-deleted", `{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::batch-authz/*"]}`)
|
||||
parent := iamAdminUser(f.ctx, t, f.sys, "mapped-parent", "")
|
||||
_, err := f.sys.PolicyDBSet(f.ctx, parent.AccessKey, "mapped-full,mapped-deleted", stsUser, false)
|
||||
mustIAM(t, err)
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent.AccessKey
|
||||
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
|
||||
mustIAM(t, err)
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
req := batchAuthzRequest{action: policy.GetObjectAction, bucket: f.bucket, object: "key", versionID: mustGetUUID()}
|
||||
if err := sub.authorize(f.ctx, req); err != nil {
|
||||
t.Fatalf("control: %v", err)
|
||||
}
|
||||
p, err := sub.principal(f.ctx)
|
||||
mustIAM(t, err)
|
||||
args := policy.Args{AccountName: cred.AccessKey, Action: req.action, BucketName: req.bucket, ObjectName: req.object, Claims: p.claims, ConditionValues: p.conditionValues(req)}
|
||||
|
||||
mustIAM(t, f.sys.DeletePolicy(f.ctx, "mapped-deleted", true))
|
||||
if !f.sys.IsAllowed(args) {
|
||||
t.Fatal("the request path denied the remaining mapped policy")
|
||||
}
|
||||
if err := sub.authorize(f.ctx, req); err != nil {
|
||||
t.Fatalf("batch authorization diverged from the request after a mapped policy was deleted: %v", err)
|
||||
}
|
||||
|
||||
// With every mapped policy deleted, both deny; the job is refused, not
|
||||
// paused, as nothing is unavailable.
|
||||
mustIAM(t, f.sys.DeletePolicy(f.ctx, "mapped-full", true))
|
||||
if f.sys.IsAllowed(args) {
|
||||
t.Fatal("the request path allowed without any mapped policy")
|
||||
}
|
||||
if err := sub.authorize(f.ctx, req); !isBatchJobAuthzError(err) {
|
||||
t.Fatalf("batch authorization without any mapped policy: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,919 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/arn"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
// A batch job acts on objects long after the request that submitted it has
|
||||
// gone. It therefore records who submitted it and acts only as that
|
||||
// principal:
|
||||
//
|
||||
// - StartBatchJob binds the authenticated credential to the job
|
||||
// (batchJobSubmitter). Nothing in the submitted definition names it.
|
||||
// - Every start of the job, and every object it touches, resolves the
|
||||
// binding against IAM again. A credential that is gone, disabled,
|
||||
// expired, revoked or edited since submission fails the job; a reused
|
||||
// access key does not match the binding.
|
||||
// - Each action is evaluated with the submitter's policies and session
|
||||
// policy but without a request. A policy statement that would need
|
||||
// request context the job does not have (source address, transport,
|
||||
// headers and so on) refuses the action instead of being skipped.
|
||||
//
|
||||
// A job without a binding was queued by an earlier release. It is never
|
||||
// authorized, whoever submitted it, and has to be submitted again.
|
||||
|
||||
//go:generate msgp -file $GOFILE -unexported
|
||||
|
||||
// batchJobSubmitter is the server-assigned identity of the credential that
|
||||
// submitted a batch job.
|
||||
type batchJobSubmitter struct {
|
||||
Version int `msg:"v"`
|
||||
Type string `msg:"t"`
|
||||
AccessKey string `msg:"ak"`
|
||||
// ParentUser is the parent of a derived credential.
|
||||
ParentUser string `msg:"pu"`
|
||||
// UpdatedAt and RevokedBefore are the revision of the identity record;
|
||||
// any edit of the identity changes them. They are unset for root.
|
||||
UpdatedAt time.Time `msg:"ua"`
|
||||
RevokedBefore time.Time `msg:"rb"`
|
||||
// Binding is a keyed fingerprint of the credential itself, see
|
||||
// batchSubmitterBinding.
|
||||
Binding []byte `msg:"bd"`
|
||||
}
|
||||
|
||||
const batchJobSubmitterVersion = 1
|
||||
|
||||
// Submitter types. A credential is classified in the order IsAllowed
|
||||
// evaluates it.
|
||||
const (
|
||||
batchSubmitterRoot = "root"
|
||||
batchSubmitterUser = "user"
|
||||
batchSubmitterTemporary = "temporary"
|
||||
batchSubmitterService = "service-account"
|
||||
)
|
||||
|
||||
//msgp:ignore batchJobAuthzError
|
||||
|
||||
// batchJobAuthzError refuses a batch job, or one of its actions, for its
|
||||
// submitter. It ends the job: it is persisted as the job's failure and is
|
||||
// never retried.
|
||||
type batchJobAuthzError struct {
|
||||
reason string
|
||||
}
|
||||
|
||||
func (e batchJobAuthzError) Error() string {
|
||||
return e.reason
|
||||
}
|
||||
|
||||
func isBatchJobAuthzError(err error) bool {
|
||||
var e batchJobAuthzError
|
||||
return errors.As(err, &e)
|
||||
}
|
||||
|
||||
var (
|
||||
errBatchJobResubmit = batchJobAuthzError{reason: "the batch job does not record who submitted it; submit it again"}
|
||||
errBatchJobChanged = batchJobAuthzError{reason: "the credential that submitted the batch job has changed since submission; submit it again"}
|
||||
errBatchJobInvalid = batchJobAuthzError{reason: "the credential that submitted the batch job is no longer valid"}
|
||||
)
|
||||
|
||||
func batchSubmitterType(cred auth.Credentials) string {
|
||||
switch {
|
||||
case cred.IsTemp():
|
||||
return batchSubmitterTemporary
|
||||
case cred.IsServiceAccount():
|
||||
return batchSubmitterService
|
||||
default:
|
||||
return batchSubmitterUser
|
||||
}
|
||||
}
|
||||
|
||||
// batchSubmitterBinding is a fingerprint of one credential keyed with its own
|
||||
// secret. It changes when the secret, the session token or the parent
|
||||
// changes, and a new credential that reuses the access key does not have it.
|
||||
func batchSubmitterBinding(typ string, cred auth.Credentials) []byte {
|
||||
mac := hmac.New(sha256.New, []byte(cred.SecretKey))
|
||||
for _, s := range []string{"silo batch-job submitter v1", typ, cred.AccessKey, cred.ParentUser, cred.SessionToken} {
|
||||
var n [8]byte
|
||||
binary.BigEndian.PutUint64(n[:], uint64(len(s)))
|
||||
mac.Write(n[:])
|
||||
mac.Write([]byte(s))
|
||||
}
|
||||
return mac.Sum(nil)
|
||||
}
|
||||
|
||||
// identityRecord returns the stored identity of a non-root access key.
|
||||
func (sys *IAMSys) identityRecord(accessKey string) (UserIdentity, bool, error) {
|
||||
if !sys.Initialized() {
|
||||
return UserIdentity{}, false, errServerNotInitialized
|
||||
}
|
||||
u, ok := sys.store.GetUser(accessKey)
|
||||
return u, ok, nil
|
||||
}
|
||||
|
||||
// newBatchJobSubmitter binds the credential that authenticated a
|
||||
// StartBatchJob request.
|
||||
func newBatchJobSubmitter(cred auth.Credentials) (*batchJobSubmitter, error) {
|
||||
root := globalActiveCred
|
||||
if cred.AccessKey == "" {
|
||||
return nil, errBatchJobInvalid
|
||||
}
|
||||
if cred.AccessKey == root.AccessKey {
|
||||
if subtle.ConstantTimeCompare([]byte(cred.SecretKey), []byte(root.SecretKey)) != 1 {
|
||||
return nil, errBatchJobChanged
|
||||
}
|
||||
return &batchJobSubmitter{
|
||||
Version: batchJobSubmitterVersion,
|
||||
Type: batchSubmitterRoot,
|
||||
AccessKey: root.AccessKey,
|
||||
Binding: batchSubmitterBinding(batchSubmitterRoot, root),
|
||||
}, nil
|
||||
}
|
||||
u, ok, err := globalIAMSys.identityRecord(cred.AccessKey)
|
||||
if err != nil {
|
||||
return nil, batchUnavailable("IAM identity lookup", err)
|
||||
}
|
||||
// The record must be the one the request was authenticated with.
|
||||
if !ok || u.Credentials.AccessKey != cred.AccessKey || u.Credentials.ParentUser != cred.ParentUser ||
|
||||
subtle.ConstantTimeCompare([]byte(u.Credentials.SecretKey), []byte(cred.SecretKey)) != 1 ||
|
||||
subtle.ConstantTimeCompare([]byte(u.Credentials.SessionToken), []byte(cred.SessionToken)) != 1 {
|
||||
return nil, errBatchJobChanged
|
||||
}
|
||||
typ := batchSubmitterType(u.Credentials)
|
||||
return &batchJobSubmitter{
|
||||
Version: batchJobSubmitterVersion,
|
||||
Type: typ,
|
||||
AccessKey: u.Credentials.AccessKey,
|
||||
ParentUser: u.Credentials.ParentUser,
|
||||
UpdatedAt: u.UpdatedAt,
|
||||
RevokedBefore: u.RevokedBefore,
|
||||
Binding: batchSubmitterBinding(typ, u.Credentials),
|
||||
}, nil
|
||||
}
|
||||
|
||||
//msgp:ignore batchJobPrincipal batchAuthzRequest
|
||||
|
||||
// batchJobPrincipal is a submitter resolved against current IAM state.
|
||||
type batchJobPrincipal struct {
|
||||
ctx context.Context
|
||||
typ string
|
||||
accessKey string
|
||||
parentUser string
|
||||
groups []string
|
||||
claims map[string]any
|
||||
owner bool
|
||||
temp bool // evaluated as a temporary credential
|
||||
service bool // evaluated as a service account
|
||||
}
|
||||
|
||||
// principal resolves the binding against current IAM state. It fails if the
|
||||
// credential no longer exists, is no longer valid, or is no longer the one
|
||||
// that was bound.
|
||||
func (s *batchJobSubmitter) principal(ctx context.Context) (batchJobPrincipal, error) {
|
||||
if s == nil || s.AccessKey == "" {
|
||||
return batchJobPrincipal{}, errBatchJobResubmit
|
||||
}
|
||||
if s.Version != batchJobSubmitterVersion {
|
||||
return batchJobPrincipal{}, batchJobAuthzError{reason: fmt.Sprintf("unsupported batch job submitter version %d; submit the job again", s.Version)}
|
||||
}
|
||||
if !batchIAMReady() {
|
||||
return batchJobPrincipal{}, batchUnavailable("IAM is not initialized", errServerNotInitialized)
|
||||
}
|
||||
root := globalActiveCred
|
||||
if s.Type == batchSubmitterRoot {
|
||||
if s.AccessKey != root.AccessKey || !hmac.Equal(s.Binding, batchSubmitterBinding(batchSubmitterRoot, root)) {
|
||||
return batchJobPrincipal{}, errBatchJobChanged
|
||||
}
|
||||
if !globalAPIConfig.permitRootAccess() {
|
||||
return batchJobPrincipal{}, errBatchJobInvalid
|
||||
}
|
||||
return batchJobPrincipal{ctx: ctx, typ: batchSubmitterRoot, accessKey: root.AccessKey, owner: true}, nil
|
||||
}
|
||||
switch s.Type {
|
||||
case batchSubmitterUser, batchSubmitterTemporary, batchSubmitterService:
|
||||
default:
|
||||
return batchJobPrincipal{}, errBatchJobResubmit
|
||||
}
|
||||
if s.AccessKey == root.AccessKey {
|
||||
return batchJobPrincipal{}, errBatchJobChanged
|
||||
}
|
||||
u, ok, err := globalIAMSys.CheckKey(ctx, s.AccessKey)
|
||||
if err != nil {
|
||||
return batchJobPrincipal{}, batchUnavailable("IAM credential lookup", err)
|
||||
}
|
||||
if !ok {
|
||||
// Deleted, disabled, expired or revoked.
|
||||
return batchJobPrincipal{}, errBatchJobInvalid
|
||||
}
|
||||
cred := u.Credentials
|
||||
typ := batchSubmitterType(cred)
|
||||
if typ != s.Type || cred.AccessKey != s.AccessKey || cred.ParentUser != s.ParentUser ||
|
||||
!u.UpdatedAt.Equal(s.UpdatedAt) || !u.RevokedBefore.Equal(s.RevokedBefore) ||
|
||||
!hmac.Equal(s.Binding, batchSubmitterBinding(typ, cred)) {
|
||||
return batchJobPrincipal{}, errBatchJobChanged
|
||||
}
|
||||
if cred.IsExpired() || cred.Status == auth.AccountOff {
|
||||
return batchJobPrincipal{}, errBatchJobInvalid
|
||||
}
|
||||
p := batchJobPrincipal{
|
||||
ctx: ctx,
|
||||
typ: typ,
|
||||
accessKey: cred.AccessKey,
|
||||
parentUser: cred.ParentUser,
|
||||
groups: cred.Groups,
|
||||
claims: map[string]any{},
|
||||
temp: cred.IsTemp(),
|
||||
service: !cred.IsTemp() && cred.IsServiceAccount(),
|
||||
}
|
||||
if typ != batchSubmitterUser {
|
||||
// Verify the token as a request would, with the current signing keys.
|
||||
jwtClaims, err := extractJWTClaims(u)
|
||||
if err != nil {
|
||||
if errors.Is(err, errTokenSigningKeyUnavailable) {
|
||||
return batchJobPrincipal{}, batchUnavailable("token signing key", err)
|
||||
}
|
||||
return batchJobPrincipal{}, errBatchJobInvalid
|
||||
}
|
||||
p.claims = jwtClaims.Map()
|
||||
if err := decodeBatchSessionPolicy(p.claims); err != nil {
|
||||
return batchJobPrincipal{}, errBatchJobInvalid
|
||||
}
|
||||
}
|
||||
p.owner = cred.ParentUser == root.AccessKey && cred.AccessKey != siteReplicatorSvcAcc
|
||||
if p.owner && !globalAPIConfig.permitRootAccess() {
|
||||
return batchJobPrincipal{}, errBatchJobInvalid
|
||||
}
|
||||
if _, ok := p.claims[policy.SessionPolicyName]; ok {
|
||||
p.owner = false
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// decodeBatchSessionPolicy decodes a session policy claim for evaluation.
|
||||
// A request decodes it while verifying its token, except when an
|
||||
// authorization plugin is configured; a batch job decodes it always, so the
|
||||
// session policy also limits what the plugin allows.
|
||||
func decodeBatchSessionPolicy(claims map[string]any) error {
|
||||
sp, ok := claims[policy.SessionPolicyName]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
s, ok := sp.(string)
|
||||
if !ok {
|
||||
return errAuthentication
|
||||
}
|
||||
b, err := base64.StdEncoding.DecodeString(s)
|
||||
if err != nil {
|
||||
return errAuthentication
|
||||
}
|
||||
claims[sessionPolicyNameExtracted] = string(b)
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchAuthzRequest is one action a batch job is about to take.
|
||||
type batchAuthzRequest struct {
|
||||
action policy.Action
|
||||
bucket string
|
||||
object string // empty for a bucket action
|
||||
// versionID is the version an object version action addresses.
|
||||
versionID string
|
||||
// tags are the current tags of the object version acted on.
|
||||
tags string
|
||||
// tagsUnknown is set when the job does not read the version it acts
|
||||
// on, so s3:ExistingObjectTag is context it does not have.
|
||||
tagsUnknown bool
|
||||
// values are request values the job supplies itself, such as the
|
||||
// listing prefix or the encryption and retention it writes, keyed by
|
||||
// condition key name.
|
||||
values map[string][]string
|
||||
}
|
||||
|
||||
// batchAuthzMarker marks the condition values of a batch job for an
|
||||
// authorization plugin. Requests cannot set s3:authType.
|
||||
const batchAuthzMarker = "batch-job"
|
||||
|
||||
// Condition keys a batch job always provides, as a request would: identity,
|
||||
// time, the addressed version and the stored tags. Keys a request takes from
|
||||
// itself are provided only when the job sets them (see batchAuthzRequest
|
||||
// values); every other key is request context the job does not have.
|
||||
var batchServerConditionKeys = map[string]bool{
|
||||
condition.AWSCurrentTime.ToKey().String(): true,
|
||||
condition.AWSEpochTime.ToKey().String(): true,
|
||||
condition.AWSPrincipalType.ToKey().String(): true,
|
||||
condition.AWSUserID.ToKey().String(): true,
|
||||
condition.AWSUsername.ToKey().String(): true,
|
||||
condition.AWSGroups.ToKey().String(): true,
|
||||
condition.S3VersionID.ToKey().String(): true,
|
||||
condition.ExistingObjectTag.ToKey().String(): true,
|
||||
}
|
||||
|
||||
// Request keys a job may supply, by the name they have in condition values.
|
||||
var batchRequestConditionKeys = map[string]condition.KeyName{
|
||||
condition.S3Prefix.Name(): condition.S3Prefix,
|
||||
condition.S3XAmzServerSideEncryption.Name(): condition.S3XAmzServerSideEncryption,
|
||||
condition.S3XAmzServerSideEncryptionAwsKmsKeyID.Name(): condition.S3XAmzServerSideEncryptionAwsKmsKeyID,
|
||||
condition.S3ObjectLockMode.Name(): condition.S3ObjectLockMode,
|
||||
condition.S3ObjectLockRetainUntilDate.Name(): condition.S3ObjectLockRetainUntilDate,
|
||||
condition.S3ObjectLockLegalHold.Name(): condition.S3ObjectLockLegalHold,
|
||||
condition.RequestObjectTagKeys.Name(): condition.RequestObjectTagKeys,
|
||||
}
|
||||
|
||||
// suppliedConditionKeys returns the condition keys whose values req provides.
|
||||
func (req batchAuthzRequest) suppliedConditionKeys() map[string]bool {
|
||||
supplied := make(map[string]bool, len(batchServerConditionKeys)+len(req.values))
|
||||
for k := range batchServerConditionKeys {
|
||||
supplied[k] = true
|
||||
}
|
||||
if req.tagsUnknown {
|
||||
delete(supplied, condition.ExistingObjectTag.ToKey().String())
|
||||
}
|
||||
for name := range req.values {
|
||||
if key, ok := batchRequestConditionKeys[name]; ok {
|
||||
supplied[key.ToKey().String()] = true
|
||||
}
|
||||
if strings.HasPrefix(name, condition.RequestObjectTag.Name()+"/") {
|
||||
supplied[condition.RequestObjectTag.ToKey().String()] = true
|
||||
}
|
||||
}
|
||||
return supplied
|
||||
}
|
||||
|
||||
func batchKeySupplied(supplied map[string]bool, name string) bool {
|
||||
if i := strings.IndexByte(name, '/'); i >= 0 {
|
||||
name = name[:i]
|
||||
}
|
||||
if strings.HasPrefix(name, "jwt:") || strings.HasPrefix(name, "ldap:") {
|
||||
// Claims of the verified token, as a request has them.
|
||||
return true
|
||||
}
|
||||
return supplied[name]
|
||||
}
|
||||
|
||||
// conditionValues are the values a request with the same identity would
|
||||
// carry, without any value taken from the request itself.
|
||||
func (p batchJobPrincipal) conditionValues(req batchAuthzRequest) map[string][]string {
|
||||
now := UTCNow()
|
||||
username := p.accessKey
|
||||
if p.temp || p.service {
|
||||
username = p.parentUser
|
||||
}
|
||||
principalType := "User"
|
||||
if len(p.claims) > 0 {
|
||||
principalType = "AssumedRole"
|
||||
}
|
||||
if username == globalActiveCred.AccessKey {
|
||||
principalType = "Account"
|
||||
}
|
||||
// Identity and time, then the claims over them, as a request has them
|
||||
// (an LDAP session's short username replaces its parent DN).
|
||||
values := map[string][]string{
|
||||
"CurrentTime": {now.Format(time.RFC3339)},
|
||||
"EpochTime": {strconv.FormatInt(now.Unix(), 10)},
|
||||
"principaltype": {principalType},
|
||||
"userid": {username},
|
||||
"username": {username},
|
||||
}
|
||||
for k, v := range p.claims {
|
||||
if s, ok := v.(string); ok {
|
||||
values[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{s}
|
||||
}
|
||||
}
|
||||
// No claim stands for a value of the object or of the job, which
|
||||
// follow and always win.
|
||||
for k := range values {
|
||||
if k == "versionid" || k == "prefix" || k == "authtype" ||
|
||||
strings.HasPrefix(k, "existingobjecttag/") || strings.HasPrefix(k, "requestobjecttag") ||
|
||||
strings.HasPrefix(k, "x-amz-") || strings.HasPrefix(k, "object-lock-") {
|
||||
delete(values, k)
|
||||
}
|
||||
}
|
||||
// For an authorization plugin: this is a batch job, not a request.
|
||||
values["authType"] = []string{batchAuthzMarker}
|
||||
if req.versionID != "" {
|
||||
values["versionid"] = []string{req.versionID}
|
||||
}
|
||||
if req.tags != "" {
|
||||
if t, err := tags.ParseObjectTags(req.tags); err == nil {
|
||||
for k, v := range t.ToMap() {
|
||||
values[pathJoin("ExistingObjectTag", k)] = []string{v}
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, v := range req.values {
|
||||
values[k] = v
|
||||
}
|
||||
if grpsVal, ok := p.claims["groups"]; ok {
|
||||
if grpsIs, ok := grpsVal.([]any); ok {
|
||||
grps := []string{}
|
||||
for _, gI := range grpsIs {
|
||||
if g, ok := gI.(string); ok {
|
||||
grps = append(grps, g)
|
||||
}
|
||||
}
|
||||
if len(grps) > 0 {
|
||||
values["groups"] = grps
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, ok := values["groups"]; !ok && len(p.groups) > 0 {
|
||||
values["groups"] = p.groups
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// authorize evaluates one action of the job as its submitter.
|
||||
func (p batchJobPrincipal) authorize(req batchAuthzRequest) error {
|
||||
args := policy.Args{
|
||||
AccountName: p.accessKey,
|
||||
Groups: p.groups,
|
||||
Action: req.action,
|
||||
BucketName: req.bucket,
|
||||
ObjectName: req.object,
|
||||
ConditionValues: p.conditionValues(req),
|
||||
IsOwner: p.owner,
|
||||
Claims: p.claims,
|
||||
}
|
||||
return p.authorizeArgs(args, &req)
|
||||
}
|
||||
|
||||
// authorizeArgs uses the same policy snapshot for request-context checks and
|
||||
// the decision. No boolean IAM evaluator re-reads and silently drops policies.
|
||||
func (p batchJobPrincipal) authorizeArgs(args policy.Args, req *batchAuthzRequest) error {
|
||||
denied := batchJobAuthzError{reason: "the batch job submitter is not allowed " + string(args.Action)}
|
||||
if req != nil {
|
||||
denied.reason = batchDeniedReason(*req)
|
||||
}
|
||||
ctx := p.ctx
|
||||
if ctx == nil {
|
||||
ctx = GlobalContext
|
||||
}
|
||||
plugin := newGlobalAuthZPluginFn()
|
||||
var snap *batchPolicySnapshot
|
||||
if plugin == nil && !args.IsOwner {
|
||||
snap = globalIAMSys.store.batchSnapshot(ctx)
|
||||
defer snap.release()
|
||||
}
|
||||
policies, err := globalIAMSys.batchLocalPolicies(p, args, plugin == nil, snap)
|
||||
if err != nil {
|
||||
if errors.Is(err, errNoSuchPolicy) || errors.Is(err, errAuthentication) || errors.Is(err, errNoSuchUser) || errors.Is(err, errNoSuchGroup) {
|
||||
return denied
|
||||
}
|
||||
return batchUnavailable("IAM policy lookup", err)
|
||||
}
|
||||
if req != nil {
|
||||
supplied := req.suppliedConditionKeys()
|
||||
for _, pol := range policies {
|
||||
for _, st := range pol.Statements {
|
||||
if batchStatementNeedsRequest(st, args, supplied) {
|
||||
return batchJobAuthzError{reason: denied.reason + ": a policy statement depends on request context that a batch job does not have"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if plugin != nil {
|
||||
ok, err := plugin.IsAllowedContext(ctx, args)
|
||||
if err != nil {
|
||||
return batchUnavailable("authorization plugin", err)
|
||||
}
|
||||
if !ok {
|
||||
return denied
|
||||
}
|
||||
// The plugin does not see a session policy; it only limits further.
|
||||
if has, allowed := p.sessionPolicyDecision(args); has && !allowed {
|
||||
return denied
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if args.IsOwner {
|
||||
return nil
|
||||
}
|
||||
if p.service {
|
||||
if p.claims[parentClaim] != p.parentUser {
|
||||
return denied
|
||||
}
|
||||
if _, ok := p.claims[iamPolicyClaimNameSA()].(string); !ok {
|
||||
return denied
|
||||
}
|
||||
}
|
||||
if has, allowed := p.sessionPolicyDecision(args); has && !allowed {
|
||||
return denied
|
||||
}
|
||||
ownerDerived := (p.temp || p.service) && p.parentUser == globalActiveCred.AccessKey
|
||||
if !ownerDerived {
|
||||
if p.service {
|
||||
args.AccountName = p.parentUser
|
||||
}
|
||||
if len(policies) == 0 || !policies[0].IsAllowed(args) {
|
||||
return denied
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func batchDeniedReason(req batchAuthzRequest) string {
|
||||
resource := req.bucket
|
||||
if req.object != "" {
|
||||
resource = pathJoin(req.bucket, req.object)
|
||||
}
|
||||
if req.versionID != "" {
|
||||
resource += " (versionId " + req.versionID + ")"
|
||||
}
|
||||
return fmt.Sprintf("the batch job submitter is not allowed %s on %s", req.action, resource)
|
||||
}
|
||||
|
||||
// sessionPolicyDecision evaluates the session policy the way IsAllowed would
|
||||
// for this credential.
|
||||
func (p batchJobPrincipal) sessionPolicyDecision(args policy.Args) (bool, bool) {
|
||||
switch {
|
||||
case p.temp:
|
||||
return isAllowedBySessionPolicy(args)
|
||||
case p.service:
|
||||
if p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
|
||||
return false, false
|
||||
}
|
||||
return isAllowedBySessionPolicyForServiceAccount(args)
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
// batchIAMLoadTimeout bounds one storage read of a lazy IAM load during
|
||||
// batch authorization, as MergePolicies bounds a policy document load.
|
||||
const batchIAMLoadTimeout = 5 * time.Second
|
||||
|
||||
// batchPolicySnapshot reads the mappings and policy documents of a submitter
|
||||
// at one cache revision while cached authentication and authorization
|
||||
// proceed.
|
||||
//
|
||||
// It holds the store's writer mutex, so no writer changes the cache between
|
||||
// its reads, and the cache read lock, which it releases around each storage
|
||||
// read: a lazy mapping load or a policy document load. Request
|
||||
// authentication then neither waits for storage nor queues behind a writer
|
||||
// waiting for the cache lock, which taking the write lock back after each
|
||||
// read would cause. It writes nothing to the cache but the lazily loaded
|
||||
// mappings, which the request path stores under the read lock as well.
|
||||
//
|
||||
//msgp:ignore batchPolicySnapshot
|
||||
type batchPolicySnapshot struct {
|
||||
*IAMStoreSys
|
||||
ctx context.Context
|
||||
cache *iamCache
|
||||
mu sync.Mutex
|
||||
reads int // storage reads in progress; the read lock is released while any
|
||||
}
|
||||
|
||||
// batchSnapshot takes the snapshot; release it with release. rlock and
|
||||
// runlock are the cache's read lock (IAMStorageAPI), not IAMStoreSys.lock,
|
||||
// which also takes writeMu.
|
||||
func (store *IAMStoreSys) batchSnapshot(ctx context.Context) *batchPolicySnapshot {
|
||||
store.writeMu.Lock()
|
||||
return &batchPolicySnapshot{IAMStoreSys: store, ctx: ctx, cache: store.rlock()}
|
||||
}
|
||||
|
||||
func (s *batchPolicySnapshot) release() {
|
||||
s.runlock()
|
||||
s.writeMu.Unlock()
|
||||
}
|
||||
|
||||
// read runs one storage read without the cache lock, bounded by the job's
|
||||
// context and batchIAMLoadTimeout. Concurrent reads (the group fallback of
|
||||
// policyDBGetGroups) release the lock before the first and take it back
|
||||
// after the last.
|
||||
func (s *batchPolicySnapshot) read(fn func(ctx context.Context) error) error {
|
||||
s.mu.Lock()
|
||||
if s.reads == 0 {
|
||||
s.runlock()
|
||||
}
|
||||
s.reads++
|
||||
s.mu.Unlock()
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.reads--
|
||||
if s.reads == 0 {
|
||||
s.rlock()
|
||||
}
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(s.ctx, batchIAMLoadTimeout)
|
||||
defer cancel()
|
||||
return fn(ctx)
|
||||
}
|
||||
|
||||
func (s *batchPolicySnapshot) loadMappedPolicy(_ context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
|
||||
return s.read(func(ctx context.Context) error {
|
||||
return s.IAMStoreSys.loadMappedPolicy(ctx, name, userType, isGroup, m)
|
||||
})
|
||||
}
|
||||
|
||||
// loadGroup would write the cache under the read lock. The snapshot resolves
|
||||
// the policies of users, which never loads a group.
|
||||
func (s *batchPolicySnapshot) loadGroup(context.Context, string, map[string]GroupInfo) error {
|
||||
return errors.New("batch authorization does not load groups")
|
||||
}
|
||||
|
||||
func (s *batchPolicySnapshot) loadPolicyDoc(name string, out map[string]PolicyDoc) error {
|
||||
return s.read(func(ctx context.Context) error { return s.IAMStoreSys.loadPolicyDoc(ctx, name, out) })
|
||||
}
|
||||
|
||||
// batchLocalPolicies returns every local policy that can take part in the
|
||||
// decision on args: the mapped policies that IsAllowed would merge when
|
||||
// withMapped is set (no authorization plugin), and the session policy.
|
||||
func (sys *IAMSys) batchLocalPolicies(p batchJobPrincipal, args policy.Args, withMapped bool, snap *batchPolicySnapshot) ([]policy.Policy, error) {
|
||||
var out []policy.Policy
|
||||
if withMapped && args.IsOwner {
|
||||
// IsAllowed does not evaluate any policy for the owner.
|
||||
return nil, nil
|
||||
}
|
||||
if withMapped {
|
||||
// The caller holds snap through evaluation. Lazy mapping loads and
|
||||
// selected documents therefore belong to one cache revision.
|
||||
cache := snap.cache
|
||||
mappedPolicies := func(name string) ([]string, error) {
|
||||
names, _, err := cache.policyDBGet(snap, name, false, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groups, err := cache.policyDBGetGroups(snap, len(names) > 0, args.Groups...)
|
||||
return append(names, groups...), err
|
||||
}
|
||||
var names []string
|
||||
ownerDerived := (p.temp || p.service) && p.parentUser == globalActiveCred.AccessKey
|
||||
switch {
|
||||
case p.typ == batchSubmitterRoot || ownerDerived:
|
||||
case p.temp || p.service:
|
||||
if roleArn := args.GetRoleArn(); roleArn != "" {
|
||||
a, err := arn.Parse(roleArn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, newMappedPolicy(sys.rolesMap[a]).toSlice()...)
|
||||
} else {
|
||||
mapped, err := mappedPolicies(p.parentUser)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, mapped...)
|
||||
// The claimed policies apply only when nothing is mapped.
|
||||
if len(mapped) == 0 {
|
||||
if claimed, ok := policy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID()); ok {
|
||||
names = append(names, claimed.ToSlice()...)
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
mapped, err := mappedPolicies(p.accessKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, mapped...)
|
||||
}
|
||||
var selected []policy.Policy
|
||||
for _, name := range names {
|
||||
doc, ok := cache.iamPolicyDocsMap[name]
|
||||
if !ok {
|
||||
loaded := make(map[string]PolicyDoc)
|
||||
err := snap.loadPolicyDoc(name, loaded)
|
||||
if errors.Is(err, errNoSuchPolicy) {
|
||||
// Confirmed absent, as when a mapping still names a
|
||||
// deleted policy: it takes no part in the decision, as
|
||||
// in MergePolicies. Any other failure could hide a Deny.
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, ok = loaded[name]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("load IAM policy %q returned no document", name)
|
||||
}
|
||||
}
|
||||
selected = append(selected, doc.Policy)
|
||||
}
|
||||
out = append(out, policy.MergePolicies(selected...))
|
||||
}
|
||||
if p.temp || p.service {
|
||||
if p.service && p.claims[iamPolicyClaimNameSA()] == inheritedPolicyType {
|
||||
return out, nil
|
||||
}
|
||||
if sp, ok := p.claims[sessionPolicyNameExtracted]; ok {
|
||||
s, ok := sp.(string)
|
||||
if !ok {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
parsed, err := policy.ParseConfig(strings.NewReader(s))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, *parsed)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// batchStatementNeedsRequest reports whether a statement that applies to
|
||||
// args depends on a condition key that the job does not supply. Applying
|
||||
// such a statement without its value could grant (Allow) or drop (Deny)
|
||||
// what a request with that context would not, so the job refuses the action.
|
||||
// Resources are matched without conditions; a statement is taken to apply
|
||||
// whenever it might.
|
||||
func batchStatementNeedsRequest(st policy.Statement, args policy.Args, supplied map[string]bool) bool {
|
||||
if (!st.Actions.Match(args.Action) && !st.Actions.IsEmpty()) || st.NotActions.Match(args.Action) {
|
||||
return false
|
||||
}
|
||||
// Policy variables in resources or condition values.
|
||||
if data, err := json.Marshal(st); err == nil {
|
||||
s := string(data)
|
||||
for {
|
||||
i := strings.Index(s, "${")
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
s = s[i+2:]
|
||||
j := strings.IndexByte(s, '}')
|
||||
if j < 0 {
|
||||
break
|
||||
}
|
||||
if !batchKeySupplied(supplied, s[:j]) {
|
||||
return true
|
||||
}
|
||||
s = s[j+1:]
|
||||
}
|
||||
}
|
||||
if !batchStatementMayApply(st, args) {
|
||||
return false
|
||||
}
|
||||
for key := range st.Conditions.Keys() {
|
||||
if !batchKeySupplied(supplied, key.String()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// batchStatementMayApply matches the resources of a statement, in both forms
|
||||
// a bucket request can take. A statement whose resources the evaluator
|
||||
// translates (S3 Tables and vectors actions cover S3 data actions) is taken
|
||||
// to apply.
|
||||
func batchStatementMayApply(st policy.Statement, args policy.Args) bool {
|
||||
for _, actions := range []policy.ActionSet{st.Actions, st.NotActions} {
|
||||
for action := range actions {
|
||||
if policy.TableAction(action).IsValid() || policy.VectorsAction(action).IsValid() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, resources := range []policy.ResourceSet{st.Resources, st.NotResources} {
|
||||
for r := range resources {
|
||||
if r.Type != policy.ResourceARNS3 && r.Type != policy.ResourceARNAll {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
resources := []string{args.BucketName + "/" + strings.TrimPrefix(args.ObjectName, "/")}
|
||||
if args.ObjectName == "" {
|
||||
resources = []string{args.BucketName, args.BucketName + "/"}
|
||||
}
|
||||
for _, resource := range resources {
|
||||
if len(st.Resources) > 0 && !st.Resources.Match(resource, args.ConditionValues) {
|
||||
continue
|
||||
}
|
||||
if len(st.NotResources) > 0 && st.NotResources.Match(resource, args.ConditionValues) {
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// authorize resolves the submitter and evaluates each request.
|
||||
func (s *batchJobSubmitter) authorize(ctx context.Context, reqs ...batchAuthzRequest) error {
|
||||
p, err := s.principal(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, req := range reqs {
|
||||
if err := p.authorize(req); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// startAuthzRequests are the bucket actions a job needs before it acts on any
|
||||
// object: the listings it walks. Object actions are authorized per object.
|
||||
func (j BatchJobRequest) startAuthzRequests() []batchAuthzRequest {
|
||||
list := func(bucket string, prefixes []string) []batchAuthzRequest {
|
||||
if len(prefixes) == 0 {
|
||||
prefixes = []string{""}
|
||||
}
|
||||
var reqs []batchAuthzRequest
|
||||
for _, prefix := range prefixes {
|
||||
values := map[string][]string{condition.S3Prefix.Name(): {prefix}}
|
||||
reqs = append(reqs,
|
||||
batchAuthzRequest{action: policy.ListBucketAction, bucket: bucket, values: values},
|
||||
batchAuthzRequest{action: policy.ListBucketVersionsAction, bucket: bucket, values: values},
|
||||
)
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
// The same order as Type() and the job pool.
|
||||
switch {
|
||||
case j.Replicate != nil:
|
||||
if !j.Replicate.RemoteToLocal() {
|
||||
return list(j.Replicate.Source.Bucket, j.Replicate.Source.Prefix.F())
|
||||
}
|
||||
case j.KeyRotate != nil:
|
||||
return list(j.KeyRotate.Bucket, []string{j.KeyRotate.Prefix})
|
||||
case j.Expire != nil:
|
||||
return list(j.Expire.Bucket, batchCanonicalPrefixes(j.Expire.Prefix.F()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// jobSections counts the job types a definition holds. A job is one of them.
|
||||
func (j BatchJobRequest) jobSections() int {
|
||||
n := 0
|
||||
for _, set := range []bool{j.Replicate != nil, j.KeyRotate != nil, j.Expire != nil} {
|
||||
if set {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
var errBatchJobSections = batchJobAuthzError{reason: "a batch job must define exactly one job type"}
|
||||
|
||||
// refuse ends the job for its submitter.
|
||||
func (ri *batchJobInfo) refuse(err error) {
|
||||
ri.mu.Lock()
|
||||
defer ri.mu.Unlock()
|
||||
if ri.FailureReason == "" {
|
||||
ri.FailureReason = err.Error()
|
||||
}
|
||||
ri.Complete = false
|
||||
ri.Failed = true
|
||||
ri.PausedReason = ""
|
||||
}
|
||||
|
||||
func (ri *batchJobInfo) refused() bool {
|
||||
ri.mu.RLock()
|
||||
defer ri.mu.RUnlock()
|
||||
return ri.FailureReason != ""
|
||||
}
|
||||
|
||||
// startAuthorized checks the submitter of a job that is about to run. It
|
||||
// returns done for a job that must not run now: one refused before, which
|
||||
// stays refused, or one refused now, which is persisted as failed first.
|
||||
// An error that is not a decision leaves the job to a later start.
|
||||
func (ri *batchJobInfo) startAuthorized(ctx context.Context, api ObjectLayer, job BatchJobRequest) (done bool, err error) {
|
||||
if !ri.refused() {
|
||||
err := job.Submitter.authorize(ctx, job.startAuthzRequests()...)
|
||||
if err == nil && job.jobSections() != 1 {
|
||||
err = errBatchJobSections
|
||||
}
|
||||
if err == nil {
|
||||
ri.mu.Lock()
|
||||
ri.PausedReason = ""
|
||||
ri.mu.Unlock()
|
||||
// Persist the report version before acting. Older binaries must not
|
||||
// resume an authorization-bearing job with server authority.
|
||||
if err := ri.updateAfter(ctx, api, 0, job); err != nil {
|
||||
return true, batchUnavailable("saving job progress", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
if isBatchJobUnavailable(err) {
|
||||
return true, ri.savePaused(ctx, api, job, err)
|
||||
}
|
||||
if !isBatchJobAuthzError(err) {
|
||||
return true, err
|
||||
}
|
||||
ri.refuse(err)
|
||||
batchLogIf(ctx, fmt.Errorf("batch job %s refused: %w", job.ID, err))
|
||||
}
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
if err := ri.updateAfter(ctx, api, 0, job); err != nil {
|
||||
return true, batchUnavailable("saving job refusal", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *batchJobSubmitter) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "v":
|
||||
z.Version, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Version")
|
||||
return
|
||||
}
|
||||
case "t":
|
||||
z.Type, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Type")
|
||||
return
|
||||
}
|
||||
case "ak":
|
||||
z.AccessKey, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AccessKey")
|
||||
return
|
||||
}
|
||||
case "pu":
|
||||
z.ParentUser, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ParentUser")
|
||||
return
|
||||
}
|
||||
case "ua":
|
||||
z.UpdatedAt, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "UpdatedAt")
|
||||
return
|
||||
}
|
||||
case "rb":
|
||||
z.RevokedBefore, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "RevokedBefore")
|
||||
return
|
||||
}
|
||||
case "bd":
|
||||
z.Binding, err = dc.ReadBytes(z.Binding)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Binding")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *batchJobSubmitter) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 7
|
||||
// write "v"
|
||||
err = en.Append(0x87, 0xa1, 0x76)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.Version)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Version")
|
||||
return
|
||||
}
|
||||
// write "t"
|
||||
err = en.Append(0xa1, 0x74)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Type)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Type")
|
||||
return
|
||||
}
|
||||
// write "ak"
|
||||
err = en.Append(0xa2, 0x61, 0x6b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.AccessKey)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AccessKey")
|
||||
return
|
||||
}
|
||||
// write "pu"
|
||||
err = en.Append(0xa2, 0x70, 0x75)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.ParentUser)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ParentUser")
|
||||
return
|
||||
}
|
||||
// write "ua"
|
||||
err = en.Append(0xa2, 0x75, 0x61)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.UpdatedAt)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "UpdatedAt")
|
||||
return
|
||||
}
|
||||
// write "rb"
|
||||
err = en.Append(0xa2, 0x72, 0x62)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.RevokedBefore)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "RevokedBefore")
|
||||
return
|
||||
}
|
||||
// write "bd"
|
||||
err = en.Append(0xa2, 0x62, 0x64)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.Binding)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Binding")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *batchJobSubmitter) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 7
|
||||
// string "v"
|
||||
o = append(o, 0x87, 0xa1, 0x76)
|
||||
o = msgp.AppendInt(o, z.Version)
|
||||
// string "t"
|
||||
o = append(o, 0xa1, 0x74)
|
||||
o = msgp.AppendString(o, z.Type)
|
||||
// string "ak"
|
||||
o = append(o, 0xa2, 0x61, 0x6b)
|
||||
o = msgp.AppendString(o, z.AccessKey)
|
||||
// string "pu"
|
||||
o = append(o, 0xa2, 0x70, 0x75)
|
||||
o = msgp.AppendString(o, z.ParentUser)
|
||||
// string "ua"
|
||||
o = append(o, 0xa2, 0x75, 0x61)
|
||||
o = msgp.AppendTime(o, z.UpdatedAt)
|
||||
// string "rb"
|
||||
o = append(o, 0xa2, 0x72, 0x62)
|
||||
o = msgp.AppendTime(o, z.RevokedBefore)
|
||||
// string "bd"
|
||||
o = append(o, 0xa2, 0x62, 0x64)
|
||||
o = msgp.AppendBytes(o, z.Binding)
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *batchJobSubmitter) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "v":
|
||||
z.Version, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Version")
|
||||
return
|
||||
}
|
||||
case "t":
|
||||
z.Type, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Type")
|
||||
return
|
||||
}
|
||||
case "ak":
|
||||
z.AccessKey, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "AccessKey")
|
||||
return
|
||||
}
|
||||
case "pu":
|
||||
z.ParentUser, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ParentUser")
|
||||
return
|
||||
}
|
||||
case "ua":
|
||||
z.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "UpdatedAt")
|
||||
return
|
||||
}
|
||||
case "rb":
|
||||
z.RevokedBefore, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "RevokedBefore")
|
||||
return
|
||||
}
|
||||
case "bd":
|
||||
z.Binding, bts, err = msgp.ReadBytesBytes(bts, z.Binding)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Binding")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *batchJobSubmitter) Msgsize() (s int) {
|
||||
s = 1 + 2 + msgp.IntSize + 2 + msgp.StringPrefixSize + len(z.Type) + 3 + msgp.StringPrefixSize + len(z.AccessKey) + 3 + msgp.StringPrefixSize + len(z.ParentUser) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.BytesPrefixSize + len(z.Binding)
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
func TestMarshalUnmarshalbatchJobSubmitter(t *testing.T) {
|
||||
v := batchJobSubmitter{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgbatchJobSubmitter(b *testing.B) {
|
||||
v := batchJobSubmitter{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgbatchJobSubmitter(b *testing.B) {
|
||||
v := batchJobSubmitter{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalbatchJobSubmitter(b *testing.B) {
|
||||
v := batchJobSubmitter{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodebatchJobSubmitter(t *testing.T) {
|
||||
v := batchJobSubmitter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodebatchJobSubmitter Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := batchJobSubmitter{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodebatchJobSubmitter(b *testing.B) {
|
||||
v := batchJobSubmitter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodebatchJobSubmitter(b *testing.B) {
|
||||
v := batchJobSubmitter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
type batchDiscoveryLayer struct {
|
||||
ObjectLayer
|
||||
walk func(context.Context, chan<- itemOrErr[ObjectInfo]) error
|
||||
read func(context.Context, string) ([]byte, error)
|
||||
}
|
||||
|
||||
func (a batchDiscoveryLayer) Walk(ctx context.Context, _, _ string, results chan<- itemOrErr[ObjectInfo], _ WalkOptions) error {
|
||||
return a.walk(ctx, results)
|
||||
}
|
||||
|
||||
func (a batchDiscoveryLayer) GetObjectNInfo(ctx context.Context, _, object string, _ *HTTPRangeSpec, _ http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
data, err := a.read(ctx, object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return NewGetObjectReaderFromReader(bytes.NewReader(data), ObjectInfo{Name: object}, opts)
|
||||
}
|
||||
|
||||
// A partially successful startup scan must retry the missing work without
|
||||
// re-admitting work that finished while the scan was waiting to retry.
|
||||
func TestBatchDiscoveryRecoversInProcess(t *testing.T) {
|
||||
for _, failure := range []string{"walk-start", "walk-stream", "manifest-read"} {
|
||||
t.Run(failure, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
pool := &BatchJobPool{ctx: ctx, jobCh: make(chan *BatchJobRequest, 10)}
|
||||
attempts := 0
|
||||
pool.objLayer = batchDiscoveryLayer{
|
||||
walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
|
||||
attempts++
|
||||
if attempts == 1 && failure == "walk-start" {
|
||||
return errErasureReadQuorum
|
||||
}
|
||||
defer close(results)
|
||||
results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "finished"}}
|
||||
if attempts == 1 && failure == "walk-stream" {
|
||||
results <- itemOrErr[ObjectInfo]{Err: errErasureReadQuorum}
|
||||
} else {
|
||||
results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "pending"}}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
read: func(_ context.Context, name string) ([]byte, error) {
|
||||
if attempts == 1 && failure == "manifest-read" && name == "pending" {
|
||||
return nil, errErasureReadQuorum
|
||||
}
|
||||
job := BatchJobRequest{ID: name}
|
||||
return job.MarshalMsg(nil)
|
||||
},
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
pool.resume(func() time.Duration { return 0 })
|
||||
}()
|
||||
synctest.Wait()
|
||||
if failure != "walk-start" {
|
||||
select {
|
||||
case job := <-pool.jobCh:
|
||||
if job.ID != "finished" || !pool.beginJob(job) {
|
||||
t.Fatal("first scan did not admit the healthy manifest")
|
||||
}
|
||||
pool.finishJob(job, nil)
|
||||
default:
|
||||
t.Fatal("first scan lost the healthy manifest")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(20 * time.Minute):
|
||||
t.Fatal("discovery failed to finish after storage recovered")
|
||||
}
|
||||
if attempts != 2 {
|
||||
t.Fatalf("discovery attempts = %d, want 2", attempts)
|
||||
}
|
||||
want := 1
|
||||
if failure == "walk-start" {
|
||||
want = 2
|
||||
}
|
||||
if len(pool.jobCh) != want {
|
||||
t.Fatalf("queued %d jobs, want %d; completed work was re-admitted or pending work was lost", len(pool.jobCh), want)
|
||||
}
|
||||
for len(pool.jobCh) > 0 {
|
||||
job := <-pool.jobCh
|
||||
if failure != "walk-start" && job.ID != "pending" {
|
||||
t.Fatalf("re-admitted %q", job.ID)
|
||||
}
|
||||
if !pool.beginJob(job) {
|
||||
t.Fatal("recovered job has no owner")
|
||||
}
|
||||
pool.finishJob(job, nil)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchDiscoverySkipsUnusableManifests(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
|
||||
attempts := 0
|
||||
pool.objLayer = batchDiscoveryLayer{
|
||||
walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
|
||||
attempts++
|
||||
defer close(results)
|
||||
for _, name := range []string{"canceled", "deleted", "malformed", "future", "empty-id"} {
|
||||
results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: name}}
|
||||
}
|
||||
return nil
|
||||
},
|
||||
read: func(_ context.Context, name string) ([]byte, error) {
|
||||
switch name {
|
||||
case "deleted":
|
||||
return nil, errConfigNotFound
|
||||
case "malformed":
|
||||
return []byte("bad manifest"), nil
|
||||
case "future":
|
||||
return []byte("SILO-BATCH-V999\n"), nil
|
||||
case "empty-id":
|
||||
return (&BatchJobRequest{}).MarshalMsg(nil)
|
||||
}
|
||||
// The administrator cancels after Walk, during manifest load.
|
||||
mustIAM(t, pool.canceler(name, true))
|
||||
job := BatchJobRequest{ID: name}
|
||||
return job.MarshalMsg(nil)
|
||||
},
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
pool.resume(func() time.Duration { return 0 })
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Hour):
|
||||
t.Fatal("unusable manifests kept discovery retrying")
|
||||
}
|
||||
if len(pool.jobCh) != 0 || attempts != 1 {
|
||||
t.Fatalf("canceled work was queued or retried: queue=%d scans=%d", len(pool.jobCh), attempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestBatchDiscoveryShutdownDuringBackoff(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
attempts := 0
|
||||
pool := &BatchJobPool{ctx: ctx, objLayer: batchDiscoveryLayer{
|
||||
walk: func(context.Context, chan<- itemOrErr[ObjectInfo]) error {
|
||||
attempts++
|
||||
return errErasureReadQuorum
|
||||
},
|
||||
}}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
pool.resume(func() time.Duration { return 0 })
|
||||
}()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("discovery abandoned a transient startup error")
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("discovery did not stop on shutdown")
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Fatalf("discovery scanned after shutdown: %d", attempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestBatchDiscoveryKeepsNewlyFinishedJobsTerminal(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
pool := &BatchJobPool{ctx: t.Context(), jobCh: make(chan *BatchJobRequest, 1)}
|
||||
attempts := 0
|
||||
pool.objLayer = batchDiscoveryLayer{
|
||||
walk: func(_ context.Context, results chan<- itemOrErr[ObjectInfo]) error {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
// This job was submitted and failed while startup discovery
|
||||
// had not yet read any manifests. The retry must not rerun it.
|
||||
job := &BatchJobRequest{ID: "new-job"}
|
||||
mustIAM(t, pool.queueJob(job))
|
||||
if !pool.beginJob(<-pool.jobCh) {
|
||||
t.Fatal("submitted job has no owner")
|
||||
}
|
||||
pool.finishJob(job, errErasureWriteQuorum)
|
||||
return errErasureReadQuorum
|
||||
}
|
||||
defer close(results)
|
||||
if owner := pool.jobOwners["new-job"]; owner == nil || owner.state != "terminal" || owner.req != nil || owner.cancel != nil {
|
||||
t.Fatalf("terminal marker lost or retained the job payload: %+v", owner)
|
||||
}
|
||||
results <- itemOrErr[ObjectInfo]{Item: ObjectInfo{Name: "new-job"}}
|
||||
return nil
|
||||
},
|
||||
read: func(_ context.Context, name string) ([]byte, error) {
|
||||
return (&BatchJobRequest{ID: name}).MarshalMsg(nil)
|
||||
},
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
pool.resume(func() time.Duration { return 0 })
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Hour):
|
||||
t.Fatal("discovery did not recover")
|
||||
}
|
||||
if attempts != 2 || len(pool.jobCh) != 0 {
|
||||
t.Fatalf("discovery re-admitted a failed job: scans=%d queue=%d", attempts, len(pool.jobCh))
|
||||
}
|
||||
if !pool.discoveryDone || len(pool.jobOwners) != 0 {
|
||||
t.Fatal("completed discovery retained terminal job markers")
|
||||
}
|
||||
job := &BatchJobRequest{ID: "after-discovery"}
|
||||
mustIAM(t, pool.queueJob(job))
|
||||
if !pool.beginJob(<-pool.jobCh) {
|
||||
t.Fatal("new submission after discovery has no owner")
|
||||
}
|
||||
pool.finishJob(job, nil)
|
||||
if len(pool.jobOwners) != 0 {
|
||||
t.Fatal("normal completion after discovery leaked an owner")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
// A completed expire job is not run again when the server restarts, as
|
||||
// replication and key rotation jobs are not: in particular it is not checked
|
||||
// against its submitter again, which could turn it into a failed job.
|
||||
|
||||
// An expire job that completed before the upgrade records no submitter.
|
||||
func TestBatchExpireCompletedLegacyStaysComplete(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
older := f.putVersions(t, "data/a")
|
||||
job := &BatchJobRequest{}
|
||||
mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), job))
|
||||
job.ID = "expire-legacydone" + getKeySeparator() + "0"
|
||||
job.User = globalActiveCred.AccessKey
|
||||
job.Started = time.Now()
|
||||
mustIAM(t, job.save(f.ctx, f.obj))
|
||||
ri := &batchJobInfo{JobID: job.ID, JobType: string(job.Type()), StartTime: job.Started, Complete: true, Objects: 3}
|
||||
mustIAM(t, ri.updateAfter(f.ctx, f.obj, 0, *job))
|
||||
|
||||
loaded := &BatchJobRequest{}
|
||||
mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
|
||||
f.run(t, loaded) // resumed after a restart
|
||||
got := f.jobInfo(t, job)
|
||||
if !got.Complete || got.Failed || got.FailureReason != "" {
|
||||
t.Fatalf("complete=%v failed=%v reason=%q, want the completed state kept", got.Complete, got.Failed, got.FailureReason)
|
||||
}
|
||||
if !f.versionExists(t, "data/a", older) {
|
||||
t.Error("a completed job ran again")
|
||||
}
|
||||
}
|
||||
|
||||
// A new expire job completed with a temporary credential that has expired
|
||||
// since.
|
||||
func TestBatchExpireCompletedAfterCredentialExpiry(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
|
||||
parent := iamAdminUser(f.ctx, t, f.sys, "batch-parent", "batch-full")
|
||||
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(3 * time.Second).Unix()}, globalActiveCred.SecretKey)
|
||||
mustIAM(t, err)
|
||||
cred.ParentUser = parent.AccessKey
|
||||
_, err = f.sys.SetTempUser(f.ctx, cred.AccessKey, cred, "")
|
||||
mustIAM(t, err)
|
||||
older := f.putVersions(t, "data/a")
|
||||
rec, job := f.submit(t, cred, f.expireYAML(""))
|
||||
if job == nil {
|
||||
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
f.run(t, job)
|
||||
if f.versionExists(t, "data/a", older) {
|
||||
t.Fatal("the job did not expire the older version")
|
||||
}
|
||||
if ri := f.jobInfo(t, job); !ri.Complete {
|
||||
t.Fatalf("complete=%v failed=%v", ri.Complete, ri.Failed)
|
||||
}
|
||||
time.Sleep(time.Until(cred.Expiration) + time.Second)
|
||||
|
||||
loaded := &BatchJobRequest{}
|
||||
mustIAM(t, loaded.load(f.ctx, f.obj, getJobPath(*job)))
|
||||
f.run(t, loaded) // resumed after a restart
|
||||
if ri := f.jobInfo(t, job); !ri.Complete || ri.Failed || ri.FailureReason != "" {
|
||||
t.Errorf("complete=%v failed=%v reason=%q after a restart, want the completed state kept", ri.Complete, ri.Failed, ri.FailureReason)
|
||||
}
|
||||
|
||||
// Control: a job that did not complete is still refused once its
|
||||
// credential has expired.
|
||||
f.putVersions(t, "data/b")
|
||||
pending := &BatchJobRequest{}
|
||||
mustIAM(t, yaml.Unmarshal([]byte(f.expireYAML("")), pending))
|
||||
pending.ID = "expire-pending" + getKeySeparator() + "0"
|
||||
pending.User = cred.AccessKey
|
||||
pending.Started = time.Now()
|
||||
pending.Submitter = loaded.Submitter
|
||||
mustIAM(t, pending.save(f.ctx, f.obj))
|
||||
f.run(t, pending)
|
||||
if ri := f.jobInfo(t, pending); ri.Complete || !ri.Failed || ri.FailureReason == "" {
|
||||
t.Errorf("pending job with an expired submitter: complete=%v failed=%v reason=%q, want refused", ri.Complete, ri.Failed, ri.FailureReason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,560 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// A batch expiration that keeps no versions purges each key it matched under
|
||||
// the purge's lock. What it authorizes for the submitter, and checks for
|
||||
// Object Lock, are the versions the purge deletes then, not the versions the
|
||||
// walk saw before.
|
||||
|
||||
// batchPurgeHook runs change once, after the walk of a job and before its
|
||||
// first purge of all versions.
|
||||
type batchPurgeHook struct {
|
||||
ObjectLayer
|
||||
once sync.Once
|
||||
change func()
|
||||
}
|
||||
|
||||
func (h *batchPurgeHook) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if opts.DeleteAllVersions {
|
||||
h.once.Do(h.change)
|
||||
}
|
||||
return h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
func (f *batchAuthzFixture) purgeYAML(prefix string) string {
|
||||
return fmt.Sprintf(`expire:
|
||||
apiVersion: v1
|
||||
bucket: %s
|
||||
prefix: %q
|
||||
rules:
|
||||
- type: object
|
||||
purge:
|
||||
retainVersions: 0
|
||||
`, f.bucket, prefix)
|
||||
}
|
||||
|
||||
// runChanged submits a purge of prefix as cred and runs it, with change made
|
||||
// between its walk and its purge, and returns the job's final state.
|
||||
func (f *batchAuthzFixture) runChanged(t *testing.T, cred auth.Credentials, prefix string, change func()) *batchJobInfo {
|
||||
t.Helper()
|
||||
rec, job := f.submit(t, cred, f.purgeYAML(prefix))
|
||||
if job == nil {
|
||||
t.Fatalf("submission status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
hook := &batchPurgeHook{ObjectLayer: f.obj, change: change}
|
||||
if err := job.Expire.Start(f.ctx, hook, *job); err != nil {
|
||||
t.Logf("start: %v", err)
|
||||
}
|
||||
return f.jobInfo(t, job)
|
||||
}
|
||||
|
||||
// putOlder writes a version of key older than every version it has, as a
|
||||
// replicated or pulled version can be, with metadata meta.
|
||||
func (f *batchAuthzFixture) putOlder(t *testing.T, key, versionID string, meta map[string]string) {
|
||||
t.Helper()
|
||||
data := []byte("older " + key)
|
||||
_, err := f.obj.PutObject(f.ctx, f.bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""),
|
||||
ObjectOptions{Versioned: true, VersionID: versionID, MTime: UTCNow().Add(-time.Hour), UserDefined: meta})
|
||||
mustIAM(t, err)
|
||||
}
|
||||
|
||||
func batchPurgeVersionIDs(t *testing.T, f *batchAuthzFixture, key string) []string {
|
||||
t.Helper()
|
||||
var ids []string
|
||||
marker, vmarker := "", ""
|
||||
for {
|
||||
res, err := f.obj.ListObjectVersions(f.ctx, f.bucket, key, marker, vmarker, "", 1000)
|
||||
mustIAM(t, err)
|
||||
for _, o := range res.Objects {
|
||||
if o.Name == key {
|
||||
ids = append(ids, o.VersionID)
|
||||
}
|
||||
}
|
||||
if !res.IsTruncated {
|
||||
return ids
|
||||
}
|
||||
marker, vmarker = res.NextMarker, res.NextVersionIDMarker
|
||||
}
|
||||
}
|
||||
|
||||
// A version that the submitter may not expire, because of its tags or its
|
||||
// version ID, is refused although the walk did not see it so: the job fails
|
||||
// and the key keeps all its versions.
|
||||
func TestBatchExpirePurgeAuthorizesCurrentVersions(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
action string // the action a conditional Deny denies
|
||||
deny string // its condition
|
||||
// change returns the version IDs that change made.
|
||||
change func(t *testing.T, f *batchAuthzFixture, key, older string) []string
|
||||
}{
|
||||
{
|
||||
name: "tagged-after-walk",
|
||||
action: "s3:GetObjectVersion",
|
||||
deny: `{"StringEquals":{"s3:ExistingObjectTag/keep":["yes"]}}`,
|
||||
change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
|
||||
_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older})
|
||||
mustIAM(t, err)
|
||||
return nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "version-after-walk",
|
||||
action: "s3:DeleteObjectVersion",
|
||||
deny: `{"StringEquals":{"s3:versionid":["%s"]}}`,
|
||||
change: func(t *testing.T, f *batchAuthzFixture, key, older string) []string {
|
||||
f.putOlder(t, key, batchPurgeDeniedVersion, nil)
|
||||
return []string{batchPurgeDeniedVersion}
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
deny := tc.deny
|
||||
if strings.Contains(deny, "%s") {
|
||||
deny = fmt.Sprintf(deny, batchPurgeDeniedVersion)
|
||||
}
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "batch-purge", batchAuthzFullAccess+`,
|
||||
{"Effect":"Deny","Action":["`+tc.action+`"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":`+deny+`}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "batch-purge-user", "batch-purge")
|
||||
const key = "purge/key"
|
||||
older := f.putVersions(t, key)
|
||||
want := batchPurgeVersionIDs(t, f, key)
|
||||
ri := f.runChanged(t, cred, "purge/", func() {
|
||||
want = append(want, tc.change(t, f, key, older)...)
|
||||
})
|
||||
for _, id := range want {
|
||||
if !f.versionExists(t, key, id) {
|
||||
t.Errorf("version %s was purged although the submitter may not delete it", id)
|
||||
}
|
||||
}
|
||||
if !ri.Failed || ri.Complete || ri.FailureReason == "" {
|
||||
t.Errorf("job complete=%v failed=%v reason=%q, want a refused job", ri.Complete, ri.Failed, ri.FailureReason)
|
||||
}
|
||||
if ri.Objects != 0 || ri.KeysRefused != 0 {
|
||||
t.Errorf("job counts objects=%d refused=%d", ri.Objects, ri.KeysRefused)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const batchPurgeDeniedVersion = "6c4b2f8e-1d3a-4e5b-9c7d-0a1b2c3d4e5f"
|
||||
|
||||
// A version that Object Lock protects refuses the whole key, also when it
|
||||
// was added after the walk. The job completes with the key counted as
|
||||
// refused, not failed.
|
||||
func TestBatchExpirePurgeRefusesLockedAfterWalk(t *testing.T) {
|
||||
for _, kind := range batchLockKinds {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
enableBucketObjectLock(t, f.bucket)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
|
||||
const key = "purge/locked"
|
||||
f.putVersions(t, key)
|
||||
want := batchPurgeVersionIDs(t, f, key)
|
||||
locked := mustGetUUID()
|
||||
ri := f.runChanged(t, cred, "purge/", func() {
|
||||
f.putOlder(t, key, locked, batchLockMeta(kind))
|
||||
want = append(want, locked)
|
||||
})
|
||||
for _, id := range want {
|
||||
if !f.versionExists(t, key, id) {
|
||||
t.Errorf("version %s of a key with a locked version was purged", id)
|
||||
}
|
||||
}
|
||||
if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
|
||||
t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
|
||||
ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// transition marks version versionID of key as transitioned to a remote
|
||||
// tier, as a lifecycle transition leaves it.
|
||||
func (f *batchAuthzFixture) transition(t *testing.T, key, versionID string) {
|
||||
t.Helper()
|
||||
set := f.obj.(*erasureServerPools).serverPools[0]
|
||||
infos, errs := readAllFileInfo(f.ctx, set.getHashedSet(key).getDisks(), "", f.bucket, key, versionID, false, false)
|
||||
mustIAM(t, errs[0])
|
||||
fi := infos[0]
|
||||
fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote/"+key+"/"+versionID
|
||||
mustIAM(t, set.DecomTieredObject(f.ctx, f.bucket, key, fi, ObjectOptions{}))
|
||||
}
|
||||
|
||||
// A key with a transitioned version is refused as a whole, as a key with a
|
||||
// locked version is: the job completes with the key counted as refused, not
|
||||
// failed, and the key keeps every version.
|
||||
func TestBatchExpirePurgeRefusesTieredKey(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
|
||||
const key = "purge/tiered"
|
||||
older := f.putVersions(t, key)
|
||||
f.transition(t, key, older)
|
||||
want := batchPurgeVersionIDs(t, f, key)
|
||||
ri := f.runChanged(t, cred, "purge/", func() {})
|
||||
for _, id := range want {
|
||||
if !f.versionExists(t, key, id) {
|
||||
t.Errorf("version %s of a key with a transitioned version was purged", id)
|
||||
}
|
||||
}
|
||||
if !ri.Complete || ri.Failed || ri.KeysRefused != 1 || ri.ObjectsFailed != 0 || ri.Objects != 0 {
|
||||
t.Errorf("job complete=%v failed=%v refused=%d failed objects=%d objects=%d, want complete with 1 refused key",
|
||||
ri.Complete, ri.Failed, ri.KeysRefused, ri.ObjectsFailed, ri.Objects)
|
||||
}
|
||||
}
|
||||
|
||||
// Controls: a purge by root and by an authorized user removes every version,
|
||||
// including one added after the walk.
|
||||
func TestBatchExpirePurgeAuthorizedControls(t *testing.T) {
|
||||
for _, root := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("root=%v", root), func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
cred := globalActiveCred
|
||||
if !root {
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "batch-full", batchAuthzFullAccess)
|
||||
cred = iamAdminUser(f.ctx, t, f.sys, "batch-user", "batch-full")
|
||||
}
|
||||
const key = "purge/key"
|
||||
f.putVersions(t, key)
|
||||
added := mustGetUUID()
|
||||
ri := f.runChanged(t, cred, "purge/", func() {
|
||||
f.putOlder(t, key, added, map[string]string{xhttp.AmzObjectTagging: "keep=yes"})
|
||||
})
|
||||
if ids := batchPurgeVersionIDs(t, f, key); len(ids) != 0 {
|
||||
t.Errorf("an authorized purge kept %v", ids)
|
||||
}
|
||||
if !ri.Complete || ri.Failed || ri.Objects != 3 || ri.KeysRefused != 0 {
|
||||
t.Errorf("job complete=%v failed=%v objects=%d refused=%d, want complete with 3 objects",
|
||||
ri.Complete, ri.Failed, ri.Objects, ri.KeysRefused)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Cases found in the sixth review round.
|
||||
|
||||
// batchPurgeBudgetChange runs change once the first purge stopped at its
|
||||
// budget after removing versions.
|
||||
type batchPurgeBudgetChange struct {
|
||||
ObjectLayer
|
||||
change func()
|
||||
calls int
|
||||
}
|
||||
|
||||
func (h *batchPurgeBudgetChange) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
oi, err := h.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
|
||||
if opts.DeleteAllVersions {
|
||||
h.calls++
|
||||
if h.calls == 1 && errors.Is(err, errDeleteAllBudget) && opts.DeleteAllResult.Removed > 0 {
|
||||
h.change()
|
||||
}
|
||||
}
|
||||
return oi, err
|
||||
}
|
||||
|
||||
// A purge that continues after its budget authorizes the versions again, as
|
||||
// they are then; the refusal and the versions already removed persist
|
||||
// across another start.
|
||||
func TestBatchExpirePurgeBudgetReauthorizes(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
const key = "purge/budget"
|
||||
oldest := batchPutVersion(t, f.obj, f.bucket, key, nil)
|
||||
middle := batchPutVersion(t, f.obj, f.bucket, key, nil)
|
||||
latest := batchPutVersion(t, f.obj, f.bucket, key, nil)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "review6-budget-deny", batchAuthzFullAccess+`,
|
||||
{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`)
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "review6-budget-user", "review6-budget-deny")
|
||||
rec, job := f.submit(t, cred, f.purgeYAML("purge/"))
|
||||
if job == nil {
|
||||
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
oldBudget := deleteAllBudget
|
||||
deleteAllBudget.versions = 1
|
||||
t.Cleanup(func() { deleteAllBudget = oldBudget })
|
||||
hook := &batchPurgeBudgetChange{ObjectLayer: f.obj, change: func() {
|
||||
_, err := f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: middle.VersionID})
|
||||
mustIAM(t, err)
|
||||
}}
|
||||
mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
|
||||
ri := f.jobInfo(t, job)
|
||||
if hook.calls != 2 || !ri.Failed || ri.Complete || ri.FailureReason == "" || ri.Objects != 1 || ri.KeysRefused != 0 {
|
||||
t.Fatalf("continuation: calls=%d complete=%v failed=%v reason=%q objects=%d refused=%d", hook.calls, ri.Complete, ri.Failed, ri.FailureReason, ri.Objects, ri.KeysRefused)
|
||||
}
|
||||
if f.versionExists(t, key, oldest.VersionID) || !f.versionExists(t, key, middle.VersionID) || !f.versionExists(t, key, latest.VersionID) {
|
||||
t.Fatal("only the first authorized version may be removed")
|
||||
}
|
||||
_, err := f.obj.DeleteObjectTags(f.ctx, f.bucket, key, ObjectOptions{VersionID: middle.VersionID})
|
||||
mustIAM(t, err)
|
||||
mustIAM(t, job.Expire.Start(f.ctx, hook, *job))
|
||||
after := f.jobInfo(t, job)
|
||||
if hook.calls != 2 || after.FailureReason != ri.FailureReason || after.Objects != 1 || after.Complete {
|
||||
t.Fatal("persisted refusal must prevent a fresh purge after the tags change")
|
||||
}
|
||||
}
|
||||
|
||||
// The purge's lock excludes tag writers from its authorization through its
|
||||
// last delete, for a directory key in a lock domain of its own on one pool
|
||||
// and for a key in a second pool.
|
||||
func TestBatchExpirePurgeExcludesTagWriter(t *testing.T) {
|
||||
for _, mode := range []string{ldDsync, "two-pools"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
var z *erasureServerPools
|
||||
var bucket string
|
||||
pool, key := 0, "purge/tags/"
|
||||
if mode == "two-pools" {
|
||||
z, bucket = consistencyPools(t)
|
||||
pool = 1
|
||||
} else {
|
||||
z, bucket = ldPool(t, mode)
|
||||
for i := 0; ; i++ {
|
||||
key = fmt.Sprintf("purge/tags-%d/", i)
|
||||
if z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) == 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
before := putConsistencyObject(t, z, bucket, encodeDirObject(key), pool, "", ObjectOptions{Versioned: true})
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
var released atomic.Bool
|
||||
unblock := func() {
|
||||
if released.CompareAndSwap(false, true) {
|
||||
close(release)
|
||||
}
|
||||
}
|
||||
defer unblock()
|
||||
purged := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{
|
||||
DeleteAllVersions: true,
|
||||
DeleteAllLatest: &deleteAllMatch{VersionID: before.VersionID, ModTime: before.ModTime},
|
||||
deleteAllAuthorize: func(versions []ObjectInfo) error {
|
||||
close(entered)
|
||||
select {
|
||||
case <-release:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
},
|
||||
})
|
||||
purged <- err
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("purge never reached authorization")
|
||||
}
|
||||
written := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.PutObjectTags(ctx, bucket, key, "keep=yes", ObjectOptions{VersionID: before.VersionID})
|
||||
written <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-written:
|
||||
t.Errorf("tag writer crossed the purge's authorization lock: %v", err)
|
||||
case <-time.After(ldWait):
|
||||
unblock()
|
||||
err := <-written
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Errorf("writer must observe the completed purge: %v", err)
|
||||
}
|
||||
}
|
||||
unblock()
|
||||
mustIAM(t, <-purged)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// batchTagWriteFault fails metadata updates of one key, so that a tag write
|
||||
// reaches only the drives it does not wrap.
|
||||
type batchTagWriteFault struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
}
|
||||
|
||||
func (d batchTagWriteFault) UpdateMetadata(ctx context.Context, bucket, object string, fi FileInfo, opts UpdateMetadataOpts) error {
|
||||
if bucket == d.bucket && object == d.object {
|
||||
return errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.UpdateMetadata(ctx, bucket, object, fi, opts)
|
||||
}
|
||||
|
||||
// A failed tag write can leave a revision that the submitter may not expire
|
||||
// on a minority of drives. The purge removes it with the version, so it is
|
||||
// authorized too.
|
||||
func TestBatchExpirePurgeAuthorizesDriveRevisions(t *testing.T) {
|
||||
resetTestGlobals()
|
||||
t.Cleanup(resetTestGlobals)
|
||||
// One drive of sixteen holds the failed tag write, so the read quorum of
|
||||
// the metadata must stay below sixteen. Earlier tests can leave a
|
||||
// storage class of parity 0 behind (newTestConfig on a single drive).
|
||||
previous := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previous) })
|
||||
z := purgeSinglePool(t, 16)
|
||||
globalIAMSys.initStore(z, nil)
|
||||
mustIAM(t, globalIAMSys.Load(t.Context(), true))
|
||||
f := &batchAuthzFixture{ctx: t.Context(), sys: globalIAMSys, obj: z, bucket: "review6-batch-authz"}
|
||||
mustIAM(t, z.MakeBucket(f.ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
batchTestJobPool(f.ctx, t, z)
|
||||
const key = "purge/minority-tags"
|
||||
older := f.putVersions(t, key)
|
||||
batchAuthzPolicy(f.ctx, t, f.sys, "review6-minority-deny", strings.ReplaceAll(batchAuthzFullAccess+`,
|
||||
{"Effect":"Deny","Action":["s3:GetObjectVersion"],"Resource":["arn:aws:s3:::batch-authz/*"],"Condition":{"StringEquals":{"s3:ExistingObjectTag/keep":"yes"}}}`, "batch-authz", f.bucket))
|
||||
cred := iamAdminUser(f.ctx, t, f.sys, "review6-minority-user", "review6-minority-deny")
|
||||
sub, err := newBatchJobSubmitter(cred)
|
||||
mustIAM(t, err)
|
||||
if err := sub.authorize(f.ctx, batchAuthzRequest{action: policy.GetObjectVersionAction, bucket: f.bucket, object: key, versionID: older, tags: "keep=yes"}); !isBatchJobAuthzError(err) {
|
||||
t.Fatal("control: this tag revision must deny the read needed for expiration")
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(key)
|
||||
original := set.getDisks
|
||||
disks := append([]StorageAPI(nil), original()...)
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i := 1; i < len(wrapped); i++ {
|
||||
wrapped[i] = batchTagWriteFault{StorageAPI: wrapped[i], bucket: f.bucket, object: key}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return wrapped }
|
||||
_, err = f.obj.PutObjectTags(f.ctx, f.bucket, key, "keep=yes", ObjectOptions{VersionID: older, UserDefined: map[string]string{
|
||||
ReservedMetadataPrefixLower + TaggingTimestamp: UTCNow().Format(time.RFC3339Nano),
|
||||
}})
|
||||
set.getDisks = original
|
||||
if err == nil {
|
||||
t.Fatal("control: the tag write must fail its quorum")
|
||||
}
|
||||
fi, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{})
|
||||
mustIAM(t, err)
|
||||
if fi.Metadata[xhttp.AmzObjectTagging] != "keep=yes" {
|
||||
t.Fatal("control: the successful drive must hold the denied tag revision")
|
||||
}
|
||||
history, err := z.deleteAllHistory(f.ctx, f.bucket, key)
|
||||
mustIAM(t, err)
|
||||
for _, v := range history {
|
||||
for _, c := range v.copies {
|
||||
if c.fi.Metadata[xhttp.AmzObjectTagging] != "" {
|
||||
t.Fatal("control: the quorum history must still have the original tags")
|
||||
}
|
||||
}
|
||||
}
|
||||
ri := f.runChanged(t, cred, "purge/", func() {})
|
||||
if !ri.Failed || ri.Complete || !strings.Contains(ri.FailureReason, "is not allowed") {
|
||||
t.Errorf("denied physical revision must refuse the job: complete=%v failed=%v reason=%q", ri.Complete, ri.Failed, ri.FailureReason)
|
||||
}
|
||||
if _, err := disks[0].ReadVersion(f.ctx, "", f.bucket, key, older, ReadOptions{}); err != nil {
|
||||
t.Errorf("denied physical revision was removed: %v", err)
|
||||
}
|
||||
if got := batchPurgeVersionIDs(t, f, key); len(got) != 2 {
|
||||
t.Errorf("denial must keep the whole key; remaining versions=%v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Cases found in the seventh review round.
|
||||
|
||||
// The time the authorization takes counts against the purge's budget: a
|
||||
// late allow and a deny both delete nothing.
|
||||
func TestBatchExpirePurgeBudgetIncludesAuthorization(t *testing.T) {
|
||||
z := purgeSinglePool(t, 16)
|
||||
p := newPurgeTest(t, z, "review7-purge-auth-time", MakeBucketOptions{VersioningEnabled: true}, "")
|
||||
const key = "purge/key"
|
||||
p.version(0, key, "older", 2)
|
||||
p.version(0, key, "latest", 1)
|
||||
exp := p.matched(key)
|
||||
oldBudget := deleteAllBudget
|
||||
deleteAllBudget.duration = 10 * time.Millisecond
|
||||
t.Cleanup(func() { deleteAllBudget = oldBudget })
|
||||
denied := errors.New("review7: revision denied")
|
||||
for _, decision := range []error{nil, denied} {
|
||||
called := false
|
||||
var res deleteAllResult
|
||||
_, err := z.DeleteObject(t.Context(), p.bucket, key, ObjectOptions{
|
||||
DeleteAllVersions: true,
|
||||
DeleteAllLatest: &deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime},
|
||||
DeleteAllResult: &res,
|
||||
deleteAllAuthorize: func(versions []ObjectInfo) error {
|
||||
called = true
|
||||
if len(versions) < 2 {
|
||||
t.Error("authorization did not receive both approved versions")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
return decision
|
||||
},
|
||||
})
|
||||
want := decision
|
||||
if want == nil {
|
||||
want = errDeleteAllBudget
|
||||
}
|
||||
if !called || !errors.Is(err, want) || res.Started || res.Complete || res.Removed != 0 {
|
||||
t.Errorf("late authorization: called=%v err=%v result=%+v", called, err, res)
|
||||
}
|
||||
if got := p.held(0, key); len(got) != 2 {
|
||||
t.Fatalf("purge removed versions after late authorization: %v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A listing outage pauses without moving the cursor or refusing the job;
|
||||
// a later run completes it in both modes.
|
||||
func TestBatchExpireWalkFailureResumes(t *testing.T) {
|
||||
for _, purgeAll := range []bool{false, true} {
|
||||
name := "retain-one"
|
||||
if purgeAll {
|
||||
name = "retain-zero"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
f := newBatchAuthzFixture(t)
|
||||
const key = "data/key"
|
||||
older := f.putVersions(t, key)
|
||||
definition := f.expireYAML("data/")
|
||||
if purgeAll {
|
||||
definition = f.purgeYAML("data/")
|
||||
}
|
||||
rec, job := f.submit(t, globalActiveCred, definition)
|
||||
if job == nil {
|
||||
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
listErr := errors.New("review7: transient listing failure")
|
||||
err := job.Expire.Start(f.ctx, batchWalkError{ObjectLayer: f.obj, err: listErr}, *job)
|
||||
if !errors.Is(err, listErr) {
|
||||
t.Fatalf("listing error: %v", err)
|
||||
}
|
||||
ri := f.jobInfo(t, job)
|
||||
if ri.Failed || ri.Complete || ri.PausedReason == "" || ri.FailureReason != "" || ri.Object != "" || !f.versionExists(t, key, older) {
|
||||
t.Fatalf("listing failure changed progress or became terminal: failed=%v complete=%v reason=%q cursor=%q", ri.Failed, ri.Complete, ri.FailureReason, ri.Object)
|
||||
}
|
||||
mustIAM(t, job.Expire.Start(f.ctx, f.obj, *job))
|
||||
ri = f.jobInfo(t, job)
|
||||
wantRemoved := int64(1)
|
||||
if purgeAll {
|
||||
wantRemoved = 2
|
||||
}
|
||||
if ri.Failed || !ri.Complete || ri.FailureReason != "" || ri.Objects != wantRemoved || f.versionExists(t, key, older) {
|
||||
t.Fatalf("resume did not complete: failed=%v complete=%v reason=%q removed=%d", ri.Failed, ri.Complete, ri.FailureReason, ri.Objects)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+342
-178
@@ -27,16 +27,18 @@ import (
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/minio/pkg/v3/xtime"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
@@ -289,16 +291,6 @@ type BatchJobExpire struct {
|
||||
|
||||
var _ yaml.Unmarshaler = &BatchJobExpire{}
|
||||
|
||||
// RedactSensitive will redact any sensitive information in b.
|
||||
func (r *BatchJobExpire) RedactSensitive() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
if r.NotificationCfg.Token != "" {
|
||||
r.NotificationCfg.Token = redactedText
|
||||
}
|
||||
}
|
||||
|
||||
// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
|
||||
func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
|
||||
type expireJob BatchJobExpire
|
||||
@@ -345,32 +337,6 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Expire expires object versions which have already matched supplied filter conditions
|
||||
func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
|
||||
opts := ObjectOptions{
|
||||
PrefixEnabledFn: vc.PrefixEnabled,
|
||||
VersionSuspended: vc.Suspended(),
|
||||
}
|
||||
|
||||
allErrs := make([]error, 0, len(objsToDel))
|
||||
|
||||
for {
|
||||
count := len(objsToDel)
|
||||
if count == 0 {
|
||||
break
|
||||
}
|
||||
if count > maxDeleteList {
|
||||
count = maxDeleteList
|
||||
}
|
||||
_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
|
||||
allErrs = append(allErrs, errs...)
|
||||
// Next batch of deletion
|
||||
objsToDel = objsToDel[count:]
|
||||
}
|
||||
|
||||
return allErrs
|
||||
}
|
||||
|
||||
const (
|
||||
batchExpireName = "batch-expire.bin"
|
||||
batchExpireFormat = 1
|
||||
@@ -381,22 +347,230 @@ const (
|
||||
batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
type objInfoCache map[string]*ObjectInfo
|
||||
|
||||
func newObjInfoCache() objInfoCache {
|
||||
return objInfoCache(make(map[string]*ObjectInfo))
|
||||
type expireObjInfo struct {
|
||||
ObjectInfo
|
||||
ExpireAll bool
|
||||
DeleteMarkerCount int64
|
||||
}
|
||||
|
||||
func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
|
||||
oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
|
||||
// isErrObjectLocked reports a version that Object Lock protects.
|
||||
func isErrObjectLocked(err error) bool {
|
||||
var locked ObjectLocked
|
||||
return errors.As(err, &locked)
|
||||
}
|
||||
|
||||
func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
|
||||
oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
|
||||
return oi, ok
|
||||
func (r *BatchJobExpire) tagsFiltered() bool {
|
||||
for _, rule := range r.Rules {
|
||||
if len(rule.Tags) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
|
||||
// expireAuthzRequests are the actions the job takes on one version: it reads
|
||||
// the version, and its tags when a rule filters by them, and deletes it.
|
||||
func (r *BatchJobExpire) expireAuthzRequests(name, versionID, userTags string) []batchAuthzRequest {
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
}
|
||||
reqs := []batchAuthzRequest{
|
||||
{action: policy.GetObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
|
||||
{action: policy.GetObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
|
||||
{action: policy.DeleteObjectAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
|
||||
{action: policy.DeleteObjectVersionAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
|
||||
}
|
||||
if r.tagsFiltered() {
|
||||
reqs = append(reqs,
|
||||
batchAuthzRequest{action: policy.GetObjectTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
|
||||
batchAuthzRequest{action: policy.GetObjectVersionTaggingAction, bucket: r.Bucket, object: name, versionID: versionID, tags: userTags},
|
||||
)
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
|
||||
// expireVersion deletes one version that is older than the versions the rule
|
||||
// retains. Under the object's write lock, with the version as it is then, it
|
||||
// leaves a version that Object Lock protects (ObjectLocked) and refuses one
|
||||
// that the submitter may not delete (batchJobAuthzError). An unreadable
|
||||
// version is not deleted.
|
||||
func (r *BatchJobExpire) expireVersion(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, exp expireObjInfo) error {
|
||||
p, err := job.Submitter.principal(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
versionID := exp.VersionID
|
||||
if versionID == "" {
|
||||
// An empty version ID would add a delete marker instead.
|
||||
versionID = nullVersionID
|
||||
}
|
||||
opts := ObjectOptions{
|
||||
VersionID: versionID,
|
||||
Versioned: vc.PrefixEnabled(exp.Name),
|
||||
VersionSuspended: vc.Suspended(),
|
||||
}
|
||||
opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) error {
|
||||
// A delete marker reads as MethodNotAllowed; any other error,
|
||||
// including a missing version, stops the delete.
|
||||
marker := isErrMethodNotAllowed(gerr) && (goi.DeleteMarker || !goi.VersionPurgeStatus.Empty())
|
||||
if gerr != nil && !marker {
|
||||
return gerr
|
||||
}
|
||||
if enforceRetentionForDeletion(ctx, goi) {
|
||||
return ObjectLocked{Bucket: r.Bucket, Object: exp.Name, VersionID: versionID}
|
||||
}
|
||||
for _, req := range r.expireAuthzRequests(exp.Name, versionID, goi.UserTags) {
|
||||
if err := p.authorize(req); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
_, err = api.DeleteObject(ctx, r.Bucket, exp.Name, opts)
|
||||
return err
|
||||
}
|
||||
|
||||
// expireVersions deletes the versions of one batch entry and records the
|
||||
// outcome in res. It returns an error that ends the job.
|
||||
func (r *BatchJobExpire) expireVersions(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, delay time.Duration, res *batchCounts) error {
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
err := r.expireVersion(ctx, api, vc, job, exp)
|
||||
stopFn(exp, err)
|
||||
switch {
|
||||
case err == nil:
|
||||
if exp.DeleteMarker {
|
||||
ri.countRemoved(r.Bucket, 0, 1)
|
||||
} else {
|
||||
ri.countRemoved(r.Bucket, 1, 0)
|
||||
}
|
||||
return nil
|
||||
case isBatchJobStopped(err):
|
||||
return err
|
||||
case isErrObjectLocked(err):
|
||||
// Left in place, not retried and not a failure.
|
||||
res.versionsSkipped++
|
||||
return nil
|
||||
case isErrVersionNotFound(err) || isErrObjectNotFound(err):
|
||||
// Removed concurrently.
|
||||
return nil
|
||||
}
|
||||
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", r.Bucket, exp.Name, exp.VersionID, err, attempts))
|
||||
if attempts < retryAttempts {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
if exp.DeleteMarker {
|
||||
res.deleteMarkersFailed++
|
||||
} else {
|
||||
res.objectsFailed++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// expireAllVersions purges every version of a key whose rule retains none,
|
||||
// with the purge of exactly that key (erasureServerPools.deleteAllVersions),
|
||||
// while the version the walk matched is still its latest. Under its lock,
|
||||
// before it deletes anything, the purge refuses the key when a version is
|
||||
// transitioned or Object Lock protects one, and has every write it would
|
||||
// delete, each copy and each revision a drive holds, authorized for the
|
||||
// submitter, with the version ID and tags it has then.
|
||||
//
|
||||
// What the purge removed is counted at once; a refused key is counted in res
|
||||
// as refused, and the versions a failed purge left in res as failures. A
|
||||
// failed purge is retried up to retryAttempts times, each time with the
|
||||
// version the walk matched; a purge that stopped at its budget after
|
||||
// removing versions continues without spending an attempt. A key whose
|
||||
// latest version changed since the walk is not expired. It returns an error
|
||||
// that ends the job: an authorization failure or the end of ctx.
|
||||
func (r *BatchJobExpire) expireAllVersions(ctx context.Context, api ObjectLayer, job BatchJobRequest, ri *batchJobInfo, exp expireObjInfo, retryAttempts int, res *batchCounts) error {
|
||||
var objects, markers int64
|
||||
matched := deleteAllMatch{VersionID: exp.VersionID, ModTime: exp.ModTime}
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
latest := matched
|
||||
var purge deleteAllResult
|
||||
p, err := job.Submitter.principal(ctx)
|
||||
if err == nil {
|
||||
_, err = api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
|
||||
DeleteAllVersions: true,
|
||||
DeleteAllLatest: &latest,
|
||||
DeleteAllResult: &purge,
|
||||
deleteAllAuthorize: r.expireAllAuthorize(p, exp.Name),
|
||||
})
|
||||
}
|
||||
// Removed versions are gone for any later run: count them now.
|
||||
if purge.Removed > 0 {
|
||||
ri.countRemoved(r.Bucket, int64(purge.Removed-purge.Markers), int64(purge.Markers))
|
||||
objects += int64(purge.Removed - purge.Markers)
|
||||
markers += int64(purge.Markers)
|
||||
}
|
||||
stopFn(exp, err)
|
||||
switch {
|
||||
case err == nil:
|
||||
return nil
|
||||
case isBatchJobStopped(err):
|
||||
return err
|
||||
case errors.Is(err, errDeleteAllChanged):
|
||||
// Changed since the walk matched it: not expired.
|
||||
return nil
|
||||
case isErrObjectLocked(err), errors.Is(err, errDeleteAllTiered):
|
||||
// Refused as a whole: nothing removed, not retried and not a
|
||||
// failure. A later run of the job checks the key again.
|
||||
res.keysRefused++
|
||||
return nil
|
||||
case errors.Is(err, errDeleteAllBudget) && purge.Removed > 0:
|
||||
attempts-- // progress
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
|
||||
}
|
||||
res.expireAllFailed(exp, objects, markers)
|
||||
return nil
|
||||
}
|
||||
|
||||
// expireAllAuthorize returns the authorization of a purge of all versions of
|
||||
// name for p: each version the purge would delete, as it holds it, needs the
|
||||
// actions of expiring it (expireAuthzRequests).
|
||||
func (r *BatchJobExpire) expireAllAuthorize(p batchJobPrincipal, name string) func([]ObjectInfo) error {
|
||||
return func(versions []ObjectInfo) error {
|
||||
for _, v := range versions {
|
||||
for _, req := range r.expireAuthzRequests(name, v.VersionID, v.UserTags) {
|
||||
if err := p.authorize(req); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// expireAllFailed counts the versions of info, as the walk saw them, that a
|
||||
// failed purge of all its versions did not remove: those beyond the objects
|
||||
// and markers it removed, at least one.
|
||||
func (c *batchCounts) expireAllFailed(info expireObjInfo, objects, markers int64) {
|
||||
failedObjects := max(int64(info.NumVersions)-info.DeleteMarkerCount-objects, 0)
|
||||
failedMarkers := max(info.DeleteMarkerCount-markers, 0)
|
||||
if failedObjects+failedMarkers == 0 {
|
||||
failedObjects = 1
|
||||
}
|
||||
c.objectsFailed += failedObjects
|
||||
c.deleteMarkersFailed += failedMarkers
|
||||
}
|
||||
|
||||
// batchObjsForDelete expires the batches of expireCh. walkDone reports, once
|
||||
// expireCh is closed, whether the walk reached its end: only then is its last
|
||||
// key known to be complete.
|
||||
func batchObjsForDelete(ctx context.Context, refuse func(error), walkDone func() bool, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
|
||||
vc, _ := globalBucketVersioningSys.Get(r.Bucket)
|
||||
retryAttempts := job.Expire.Retry.Attempts
|
||||
if retryAttempts <= 0 {
|
||||
@@ -407,130 +581,53 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
|
||||
delay = batchExpireJobDefaultRetryDelay
|
||||
}
|
||||
|
||||
var i int
|
||||
progress := newBatchProgress(ri, r.Bucket)
|
||||
defer func() {
|
||||
wk.Wait()
|
||||
if walkDone() {
|
||||
progress.finish()
|
||||
}
|
||||
}()
|
||||
for toExpire := range expireCh {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
if i > 0 {
|
||||
seq := progress.begin()
|
||||
if seq > 0 {
|
||||
if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
i++
|
||||
wk.Take()
|
||||
go func(toExpire []expireObjInfo) {
|
||||
go func(seq uint64, toExpire []expireObjInfo) {
|
||||
defer wk.Give()
|
||||
|
||||
toExpireAll := make([]expireObjInfo, 0, len(toExpire))
|
||||
toDel := make([]ObjectToDelete, 0, len(toExpire))
|
||||
oiCache := newObjInfoCache()
|
||||
var res batchCounts
|
||||
for _, exp := range toExpire {
|
||||
var err error
|
||||
if exp.ExpireAll {
|
||||
toExpireAll = append(toExpireAll, exp)
|
||||
continue
|
||||
err = r.expireAllVersions(ctx, api, job, ri, exp, retryAttempts, &res)
|
||||
} else {
|
||||
err = r.expireVersions(ctx, api, vc, job, ri, exp, retryAttempts, delay, &res)
|
||||
}
|
||||
// Cache ObjectInfo value via pointers for
|
||||
// subsequent use to track objects which
|
||||
// couldn't be deleted.
|
||||
od := ObjectToDelete{
|
||||
ObjectV: ObjectV{
|
||||
ObjectName: exp.Name,
|
||||
VersionID: exp.VersionID,
|
||||
},
|
||||
}
|
||||
toDel = append(toDel, od)
|
||||
oiCache.Add(od, &exp.ObjectInfo)
|
||||
}
|
||||
|
||||
// DeleteObject(deletePrefix: true) to expire all versions of an object
|
||||
for _, exp := range toExpireAll {
|
||||
var success bool
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
ri.trackMultipleObjectVersions(exp, success)
|
||||
return
|
||||
default:
|
||||
if err != nil {
|
||||
// The batch is not done; a later run does it again.
|
||||
if isBatchJobStopped(err) {
|
||||
refuse(err)
|
||||
}
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
|
||||
DeletePrefix: true,
|
||||
DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
|
||||
})
|
||||
if err != nil {
|
||||
stopFn(exp, err)
|
||||
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts))
|
||||
} else {
|
||||
stopFn(exp, err)
|
||||
success = true
|
||||
break
|
||||
}
|
||||
}
|
||||
ri.trackMultipleObjectVersions(exp, success)
|
||||
}
|
||||
|
||||
// DeleteMultiple objects
|
||||
toDelCopy := make([]ObjectToDelete, len(toDel))
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
globalBatchJobsMetrics.save(ri.JobID, ri)
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
// Copying toDel to select from objects whose
|
||||
// deletion failed
|
||||
copy(toDelCopy, toDel)
|
||||
var failed int
|
||||
errs := r.Expire(ctx, api, vc, toDel)
|
||||
// reslice toDel in preparation for next retry attempt
|
||||
toDel = toDel[:0]
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
stopFn(toDelCopy[i], err)
|
||||
batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID,
|
||||
err, attempts))
|
||||
failed++
|
||||
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
|
||||
ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts)
|
||||
}
|
||||
if attempts != retryAttempts {
|
||||
// retry
|
||||
toDel = append(toDel, toDelCopy[i])
|
||||
}
|
||||
} else {
|
||||
stopFn(toDelCopy[i], nil)
|
||||
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
|
||||
ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
globalBatchJobsMetrics.save(ri.JobID, ri)
|
||||
|
||||
if failed == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Add a delay between retry attempts
|
||||
if attempts < retryAttempts {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
}(toExpire)
|
||||
if n := len(toExpire); n > 0 {
|
||||
progress.done(seq, toExpire[n-1].Name, res)
|
||||
}
|
||||
globalBatchJobsMetrics.save(ri.JobID, ri)
|
||||
}(seq, toExpire)
|
||||
}
|
||||
}
|
||||
|
||||
type expireObjInfo struct {
|
||||
ObjectInfo
|
||||
ExpireAll bool
|
||||
DeleteMarkerCount int64
|
||||
}
|
||||
|
||||
// Start the batch expiration job, resumes if there was a pending job via "job.ID"
|
||||
func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
|
||||
ri := &batchJobInfo{
|
||||
@@ -539,6 +636,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
StartTime: job.Started,
|
||||
}
|
||||
if err := ri.loadOrInit(ctx, api, job); err != nil {
|
||||
return batchUnavailable("loading job progress", err)
|
||||
}
|
||||
// A completed job is not run again, as replication and key rotation
|
||||
// jobs are not: its submitter is not checked again either, so a
|
||||
// credential that expired since cannot turn it into a failed job.
|
||||
if ri.Complete {
|
||||
return nil
|
||||
}
|
||||
if done, err := ri.startAuthorized(ctx, api, job); done || err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -560,27 +666,47 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
|
||||
ctx, cancelCause := context.WithCancelCause(ctx)
|
||||
defer cancelCause(nil)
|
||||
// refuse ends the job for its submitter: record why before stopping.
|
||||
refuse := func(err error) {
|
||||
if isBatchJobUnavailable(err) {
|
||||
ri.pause(err)
|
||||
} else {
|
||||
ri.refuse(err)
|
||||
}
|
||||
cancelCause(err)
|
||||
}
|
||||
|
||||
// The walk stops at a listing failure: work after it is left to a later
|
||||
// run, so the cursor cannot move past what was not listed.
|
||||
walkCtx, stopWalk := context.WithCancel(ctx)
|
||||
defer stopWalk()
|
||||
results := make(chan itemOrErr[ObjectInfo], workerSize)
|
||||
go func() {
|
||||
prefixes := r.Prefix.F()
|
||||
if len(prefixes) == 0 {
|
||||
prefixes = []string{""}
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
// One ascending walk over disjoint prefixes, which the cursor
|
||||
// describes.
|
||||
for _, prefix := range batchCanonicalPrefixes(r.Prefix.F()) {
|
||||
if walkCtx.Err() != nil {
|
||||
break
|
||||
}
|
||||
prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
|
||||
err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{
|
||||
err := api.Walk(walkCtx, r.Bucket, prefix, prefixResultCh, WalkOptions{
|
||||
Marker: lastObject,
|
||||
LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions
|
||||
VersionsSort: WalkVersionsSortDesc,
|
||||
})
|
||||
if err != nil {
|
||||
cancelCause(err)
|
||||
if walkCtx.Err() == nil {
|
||||
cancelCause(batchListingError(err))
|
||||
}
|
||||
xioutil.SafeClose(results)
|
||||
return
|
||||
}
|
||||
for result := range prefixResultCh {
|
||||
results <- result
|
||||
select {
|
||||
case results <- result:
|
||||
case <-walkCtx.Done():
|
||||
// Stopped: let the walk end without a reader.
|
||||
}
|
||||
}
|
||||
}
|
||||
xioutil.SafeClose(results)
|
||||
@@ -588,7 +714,9 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
|
||||
// Goroutine to periodically save batch-expire job's in-memory state
|
||||
saverQuitCh := make(chan struct{})
|
||||
saverDoneCh := make(chan struct{})
|
||||
go func() {
|
||||
defer close(saverDoneCh)
|
||||
saveTicker := time.NewTicker(10 * time.Second)
|
||||
defer saveTicker.Stop()
|
||||
quit := false
|
||||
@@ -615,9 +743,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
|
||||
expireCh := make(chan []expireObjInfo, workerSize)
|
||||
expireDoneCh := make(chan struct{})
|
||||
var walkDone atomic.Bool
|
||||
go func() {
|
||||
defer close(expireDoneCh)
|
||||
batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
|
||||
batchObjsForDelete(ctx, refuse, walkDone.Load, r, ri, job, api, wk, expireCh)
|
||||
}()
|
||||
|
||||
var (
|
||||
@@ -629,6 +758,8 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
done bool
|
||||
)
|
||||
deleteMarkerCountMap := map[string]int64{}
|
||||
// pushToExpire is called between keys only, so that a batch holds all
|
||||
// the versions of its keys and its last key is done with it.
|
||||
pushToExpire := func() {
|
||||
// set preObject deleteMarkerCount
|
||||
if len(toDel) > 0 {
|
||||
@@ -653,6 +784,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
}
|
||||
}
|
||||
}
|
||||
var listingErr error
|
||||
for {
|
||||
select {
|
||||
case result, ok := <-results:
|
||||
@@ -662,7 +794,14 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
}
|
||||
if result.Err != nil {
|
||||
failed = true
|
||||
listingErr = batchListingError(result.Err)
|
||||
batchLogIf(ctx, result.Err)
|
||||
stopWalk()
|
||||
done = true
|
||||
break
|
||||
}
|
||||
if !batchResumeAfter(lastObject, result.Item.Name) {
|
||||
// Completed by the run that set the cursor.
|
||||
continue
|
||||
}
|
||||
if result.Item.DeleteMarker {
|
||||
@@ -720,7 +859,6 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
toDel = append(toDel, expireObjInfo{
|
||||
ObjectInfo: result.Item,
|
||||
})
|
||||
pushToExpire()
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
}
|
||||
@@ -729,16 +867,15 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
}
|
||||
}
|
||||
|
||||
if context.Cause(ctx) != nil {
|
||||
xioutil.SafeClose(expireCh)
|
||||
return context.Cause(ctx)
|
||||
}
|
||||
pushToExpire()
|
||||
// Send any remaining objects downstream
|
||||
if len(toDel) > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case expireCh <- toDel:
|
||||
if context.Cause(ctx) == nil {
|
||||
walkDone.Store(!failed)
|
||||
pushToExpire()
|
||||
// Send any remaining objects downstream
|
||||
if len(toDel) > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case expireCh <- toDel:
|
||||
}
|
||||
}
|
||||
}
|
||||
xioutil.SafeClose(expireCh)
|
||||
@@ -746,13 +883,40 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
|
||||
<-expireDoneCh // waits for the expire goroutine to complete
|
||||
wk.Wait() // waits for all expire workers to retire
|
||||
|
||||
ri.Complete = !failed && ri.ObjectsFailed == 0
|
||||
ri.Failed = failed || ri.ObjectsFailed > 0
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
|
||||
// Close the saverQuitCh - this also triggers saving in-memory state
|
||||
// immediately one last time before we exit this method.
|
||||
// Persist the final state once the saver has stopped, so that its last
|
||||
// save cannot overwrite it.
|
||||
xioutil.SafeClose(saverQuitCh)
|
||||
<-saverDoneCh
|
||||
refused := ri.refused()
|
||||
cause := context.Cause(ctx)
|
||||
if cause == nil {
|
||||
cause = listingErr
|
||||
}
|
||||
if cause != nil && !refused {
|
||||
if isBatchJobUnavailable(cause) {
|
||||
return ri.savePaused(ctx, api, job, cause)
|
||||
}
|
||||
if isErrBucketNotFound(cause) {
|
||||
ri.mu.Lock()
|
||||
ri.Failed = true
|
||||
ri.mu.Unlock()
|
||||
}
|
||||
saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
|
||||
batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
|
||||
cancel()
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
return cause
|
||||
}
|
||||
|
||||
ri.mu.Lock()
|
||||
itemsFailed := ri.expireFailures()
|
||||
ri.Complete = !refused && !failed && !itemsFailed
|
||||
ri.Failed = refused || failed || itemsFailed
|
||||
ri.mu.Unlock()
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
saveCtx, cancel := context.WithTimeout(GlobalContext, 30*time.Second)
|
||||
batchLogIf(saveCtx, ri.updateAfter(saveCtx, api, 0, job))
|
||||
cancel()
|
||||
|
||||
// Notify expire jobs final status to the configured endpoint
|
||||
buf, _ := json.Marshal(ri)
|
||||
|
||||
+637
-186
File diff suppressed because it is too large
Load Diff
+178
-9
@@ -179,6 +179,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Submitter":
|
||||
if dc.IsNil() {
|
||||
err = dc.ReadNil()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Submitter")
|
||||
return
|
||||
}
|
||||
z.Submitter = nil
|
||||
} else {
|
||||
if z.Submitter == nil {
|
||||
z.Submitter = new(batchJobSubmitter)
|
||||
}
|
||||
err = z.Submitter.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Submitter")
|
||||
return
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -192,9 +210,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 6
|
||||
// map header, size 7
|
||||
// write "ID"
|
||||
err = en.Append(0x86, 0xa2, 0x49, 0x44)
|
||||
err = en.Append(0x87, 0xa2, 0x49, 0x44)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -274,15 +292,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "Submitter"
|
||||
err = en.Append(0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if z.Submitter == nil {
|
||||
err = en.WriteNil()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
err = z.Submitter.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Submitter")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 6
|
||||
// map header, size 7
|
||||
// string "ID"
|
||||
o = append(o, 0x86, 0xa2, 0x49, 0x44)
|
||||
o = append(o, 0x87, 0xa2, 0x49, 0x44)
|
||||
o = msgp.AppendString(o, z.ID)
|
||||
// string "User"
|
||||
o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
|
||||
@@ -323,6 +358,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
// string "Submitter"
|
||||
o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x72)
|
||||
if z.Submitter == nil {
|
||||
o = msgp.AppendNil(o)
|
||||
} else {
|
||||
o, err = z.Submitter.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Submitter")
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -413,6 +459,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Submitter":
|
||||
if msgp.IsNil(bts) {
|
||||
bts, err = msgp.ReadNilBytes(bts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
z.Submitter = nil
|
||||
} else {
|
||||
if z.Submitter == nil {
|
||||
z.Submitter = new(batchJobSubmitter)
|
||||
}
|
||||
bts, err = z.Submitter.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Submitter")
|
||||
return
|
||||
}
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -445,6 +508,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
|
||||
} else {
|
||||
s += z.Expire.Msgsize()
|
||||
}
|
||||
s += 10
|
||||
if z.Submitter == nil {
|
||||
s += msgp.NilSize
|
||||
} else {
|
||||
s += z.Submitter.Msgsize()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -568,6 +637,30 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BytesFailed")
|
||||
return
|
||||
}
|
||||
case "vsl":
|
||||
z.VersionsSkippedLocked, err = dc.ReadInt64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "VersionsSkippedLocked")
|
||||
return
|
||||
}
|
||||
case "kr":
|
||||
z.KeysRefused, err = dc.ReadInt64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "KeysRefused")
|
||||
return
|
||||
}
|
||||
case "fr":
|
||||
z.FailureReason, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "FailureReason")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.PausedReason, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PausedReason")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -581,9 +674,9 @@ func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 17
|
||||
// map header, size 21
|
||||
// write "v"
|
||||
err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76)
|
||||
err = en.Append(0xde, 0x0, 0x15, 0xa1, 0x76)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -752,15 +845,55 @@ func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BytesFailed")
|
||||
return
|
||||
}
|
||||
// write "vsl"
|
||||
err = en.Append(0xa3, 0x76, 0x73, 0x6c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt64(z.VersionsSkippedLocked)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "VersionsSkippedLocked")
|
||||
return
|
||||
}
|
||||
// write "kr"
|
||||
err = en.Append(0xa2, 0x6b, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt64(z.KeysRefused)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "KeysRefused")
|
||||
return
|
||||
}
|
||||
// write "fr"
|
||||
err = en.Append(0xa2, 0x66, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.FailureReason)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "FailureReason")
|
||||
return
|
||||
}
|
||||
// write "pr"
|
||||
err = en.Append(0xa2, 0x70, 0x72)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.PausedReason)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PausedReason")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 17
|
||||
// map header, size 21
|
||||
// string "v"
|
||||
o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76)
|
||||
o = append(o, 0xde, 0x0, 0x15, 0xa1, 0x76)
|
||||
o = msgp.AppendInt(o, z.Version)
|
||||
// string "jid"
|
||||
o = append(o, 0xa3, 0x6a, 0x69, 0x64)
|
||||
@@ -810,6 +943,18 @@ func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "bf"
|
||||
o = append(o, 0xa2, 0x62, 0x66)
|
||||
o = msgp.AppendInt64(o, z.BytesFailed)
|
||||
// string "vsl"
|
||||
o = append(o, 0xa3, 0x76, 0x73, 0x6c)
|
||||
o = msgp.AppendInt64(o, z.VersionsSkippedLocked)
|
||||
// string "kr"
|
||||
o = append(o, 0xa2, 0x6b, 0x72)
|
||||
o = msgp.AppendInt64(o, z.KeysRefused)
|
||||
// string "fr"
|
||||
o = append(o, 0xa2, 0x66, 0x72)
|
||||
o = msgp.AppendString(o, z.FailureReason)
|
||||
// string "pr"
|
||||
o = append(o, 0xa2, 0x70, 0x72)
|
||||
o = msgp.AppendString(o, z.PausedReason)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -933,6 +1078,30 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BytesFailed")
|
||||
return
|
||||
}
|
||||
case "vsl":
|
||||
z.VersionsSkippedLocked, bts, err = msgp.ReadInt64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "VersionsSkippedLocked")
|
||||
return
|
||||
}
|
||||
case "kr":
|
||||
z.KeysRefused, bts, err = msgp.ReadInt64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "KeysRefused")
|
||||
return
|
||||
}
|
||||
case "fr":
|
||||
z.FailureReason, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "FailureReason")
|
||||
return
|
||||
}
|
||||
case "pr":
|
||||
z.PausedReason, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "PausedReason")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -947,6 +1116,6 @@ func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *batchJobInfo) Msgsize() (s int) {
|
||||
s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
|
||||
s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.StringPrefixSize + len(z.FailureReason) + 3 + msgp.StringPrefixSize + len(z.PausedReason)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user