mirror of
https://github.com/pgsty/minio.git
synced 2026-11-12 10:35:33 +03:00
Compare commits
1168 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 24ecc44bac | |||
| 0ae4915a93 | |||
| 4cd777a5e0 | |||
| 65028d4a35 | |||
| caac9d216e | |||
| 057192913c | |||
| f25cbdf43c | |||
| 6da4a9c7bb | |||
| 80ca120088 | |||
| a669946357 | |||
| 7ffc162ea8 | |||
| bcfd7fbbcf | |||
| 486e2e48ea | |||
| 2ddf2ca934 | |||
| 403ec7cf21 | |||
| 29b1a29044 | |||
| b4ab8e095a | |||
| ff4f4d4649 | |||
| 9987ff570b | |||
| cff8235068 | |||
| 9ef132c33b | |||
| ff8269575a | |||
| 7743d952dc | |||
| 944f3c1477 | |||
| 1d3bd02089 | |||
| 38de8e6936 | |||
| 6347fb6636 | |||
| 32e668eb94 | |||
| c51f9ef940 | |||
| 1a91edecae | |||
| c88308cf0e | |||
| 88837fb753 | |||
| d0283ff354 | |||
| f449a7ae2c | |||
| a113b2c394 | |||
| 74851834c0 | |||
| e377bb949a | |||
| c905d3fe21 | |||
| b6e9d235fe | |||
| 6968f7237a | |||
| 4a6c97463f | |||
| 6c912ac960 | |||
| 708cebe7f0 | |||
| 152023e837 | |||
| 0f16e19239 | |||
| 2c38e44e48 | |||
| 82739574b5 | |||
| f78d677ab6 | |||
| e39e2306d6 | |||
| 52229a21cb | |||
| 961f7dea82 | |||
| feeeef71f1 | |||
| 65c4d550cb | |||
| f9b4a8d6e8 | |||
| e11d851aee | |||
| ac81f0248c | |||
| 83bf15a703 | |||
| cc960adbee | |||
| c66c5828ea | |||
| 19387cafab | |||
| 7c0673279b | |||
| 7ce0d71a96 | |||
| dd2542e96c | |||
| 21d60eab7c | |||
| 4d2320ba8b | |||
| a4a74e9844 | |||
| 9588978028 | |||
| 479940b7d0 | |||
| 8cd967803c | |||
| a0e1163fb6 | |||
| 8ccd1ee34a | |||
| ca258c04cb | |||
| 30bd5e2669 | |||
| 38637897ba | |||
| c727c8b684 | |||
| 993d96feef | |||
| b2b26d9c95 | |||
| cba3dd276b | |||
| a47fc75c26 | |||
| 42cfdf246f | |||
| e5c8794b8b | |||
| ac90a873eb | |||
| 5ce68ad7fd | |||
| 099e88516d | |||
| 82a6ad2c10 | |||
| c1a78224cf | |||
| 39f9350697 | |||
| e31081d79d | |||
| f02d282754 | |||
| a89e0bab7d | |||
| 3a90af0bcd | |||
| 53ceb0791f | |||
| 2cd98a0d21 | |||
| a0b10c05e5 | |||
| 04135fa6cd | |||
| 42dc6329e6 | |||
| 9b8ba97f9f | |||
| 7705605b5a | |||
| 414bcb0c73 | |||
| f4710948c4 | |||
| 3f4488c589 | |||
| 9434fff215 | |||
| 695962fae8 | |||
| 8f13c8c3bf | |||
| c1cae51fb5 | |||
| 31d16f6cc2 | |||
| a50ea92c64 | |||
| 5b2ced0119 | |||
| 8a0ba093dd | |||
| fbd8dfe60f | |||
| 60aff22931 | |||
| 5fc7da345d | |||
| fd2c38fbef | |||
| ba245c6c46 | |||
| 496027b589 | |||
| 8bd4f6568b | |||
| 9d7660b409 | |||
| da55499db0 | |||
| 22f8e39b58 | |||
| eba23bbac4 | |||
| 4550535cbb | |||
| 8432fd5ac2 | |||
| 7c948adf88 | |||
| 56b7045c20 | |||
| b1a109a611 | |||
| 7a311a3b66 | |||
| d55b6b9909 | |||
| f4389fb322 | |||
| 331208bec1 | |||
| 7680e5f81d | |||
| 6acf038a84 | |||
| bdf4e386cf | |||
| ad8a34858f | |||
| 162eced7d2 | |||
| bec1f7c26a | |||
| 8771617199 | |||
| 54bc995f0a | |||
| 6c89a81af4 | |||
| 8fa2898ff1 | |||
| 10ca0a6936 | |||
| b3314e97a6 | |||
| 3b9a948045 | |||
| 3781a0f9ad | |||
| e79b289325 | |||
| 6d4c1156d6 | |||
| 3f72c7fcc7 | |||
| d521c84d55 | |||
| 946b070744 | |||
| 4a21dce2b5 | |||
| 5fe7f9fa93 | |||
| 65f34cd823 | |||
| 196e7e072b | |||
| 6f97663174 | |||
| aed7a1818a | |||
| b50d90183e | |||
| 6b06da76cb | |||
| 6ca6788bb7 | |||
| 2e23e61a45 | |||
| 9cdf490bc5 | |||
| cfed671ea3 | |||
| 53ce92b9ca | |||
| 7350a29fec | |||
| 5cc2c62c66 | |||
| 4bc5ed6c76 | |||
| e99a597899 | |||
| 73dde66dbe | |||
| e30c0e7ca3 | |||
| 8fc200c0cc | |||
| 708296ae1b | |||
| fbb5e75e01 | |||
| f327b21557 | |||
| 45b7253f39 | |||
| 05bb655efc | |||
| 8fdfcfb562 | |||
| e7c144eeac | |||
| e98172d72d | |||
| f2d063e7b9 | |||
| a50f26b7f5 | |||
| 69294cf98a | |||
| c397fb6c7a | |||
| 961b0b524e | |||
| 860fc200b0 | |||
| 109a9e3f35 | |||
| 5f971fea6e | |||
| 0d7abe3b9f | |||
| 94fbcd8ebe | |||
| 879d5dd236 | |||
| 34187e047d | |||
| 0ee722f8c3 | |||
| b7d11141e1 | |||
| e9babf3dac | |||
| 0bb81f2e9c | |||
| bea0b050cd | |||
| ce62980d4e | |||
| dc88865908 | |||
| 9fbd931058 | |||
| b0264bdb90 | |||
| 95d6f43cc8 | |||
| 9cb94eb4a9 | |||
| bd0819330d | |||
| 8d9e83fd99 | |||
| be02333529 | |||
| 506f121576 | |||
| ca488cce87 | |||
| 11dc723324 | |||
| dd6ea18901 | |||
| 3369eeb920 | |||
| 9032f49f25 | |||
| fbc6f3f6e8 | |||
| fba883839d | |||
| a93214ea63 | |||
| e6b0fc465b | |||
| 70fbcfee4a | |||
| 0b074d0fae | |||
| d67e4d5b17 | |||
| 891c60d83d | |||
| fe3e49c4eb | |||
| 58306a9d34 | |||
| 41091d9472 | |||
| a4cfb5e1ed | |||
| 51aa59a737 | |||
| 8bedb419a9 | |||
| f56a182b71 | |||
| 317b40ef90 | |||
| e938ece492 | |||
| 02331a612c | |||
| 8317557f70 | |||
| 1bb7a2a295 | |||
| 215ca58d6a | |||
| 12f570a307 | |||
| e4b619ce1a | |||
| 0a286153bb | |||
| 22d59e757d | |||
| 0daa2dbf59 | |||
| 96c2304ae8 | |||
| 343dd2f491 | |||
| 38f35463b7 | |||
| 5573986e8e | |||
| f3367a1b20 | |||
| a3c2f7b0e8 | |||
| 8fbec30998 | |||
| a7466eeb0e | |||
| 8b1e819bf3 | |||
| fe63664164 | |||
| 4598827dcb | |||
| 9afdb05bf4 | |||
| 9569a85cee | |||
| 54721b7c7b | |||
| 91d8bddbd1 | |||
| 80adc87a14 | |||
| 117ad1b65b | |||
| 2229509362 | |||
| 6ef8e87492 | |||
| 0a25083fdb | |||
| 15137d0327 | |||
| 8c9974bc0f | |||
| 079b6c2b50 | |||
| 0924b34a17 | |||
| 754f7a8a39 | |||
| 64bafe1dfe | |||
| c3e456e7e6 | |||
| 57aaeafd2f | |||
| 3c2e1a87e2 | |||
| da95a2d13f | |||
| cc5e05fdeb | |||
| a79c390cca | |||
| 8a56af439c | |||
| f6e581ce54 | |||
| 8953f88780 | |||
| 4b4a98d5e5 | |||
| 7472818d94 | |||
| ad44fe8d3e | |||
| 55e713db0a | |||
| 33322e6638 | |||
| a1792ca0d1 | |||
| ac8c43fe9c | |||
| 4d40ee00e9 | |||
| 06f59ad631 | |||
| 877e0cac03 | |||
| ef67c39910 | |||
| 508710f4d1 | |||
| 3aa3d9cf14 | |||
| c2fedb4c3f | |||
| 03dc65e12d | |||
| b8d62a8068 | |||
| dbc2368a7b | |||
| 54aed421b8 | |||
| d5e8dac1cf | |||
| 96ec8fcba1 | |||
| 0663eb69ed | |||
| 0594d37230 | |||
| 3cc30bcc18 | |||
| 99c1a642a4 | |||
| c60f54e5be | |||
| 483389f2e2 | |||
| c0f2f84285 | |||
| 069d118329 | |||
| a7b1834772 | |||
| 6415dec37a | |||
| 74253e1ddc | |||
| 01b3fb91e5 | |||
| 2dc917e87f | |||
| 0a284a1a10 | |||
| 5c8339e1e8 | |||
| fd37418da2 | |||
| bbfea29c2b | |||
| aa703dc903 | |||
| 8cd80fec8c | |||
| 780882efcf | |||
| c5636143c6 | |||
| ba6218b354 | |||
| 8e32de3ba9 | |||
| e37508fb8f | |||
| b46a717425 | |||
| 7926df0b80 | |||
| 557df666fd | |||
| f91b257f50 | |||
| 28a2d1eb3d | |||
| a0ae1489e5 | |||
| edfb310a59 | |||
| a2312028b9 | |||
| 78f1f69d57 | |||
| e1e33077e8 | |||
| b3e7de010d | |||
| f5b04865f4 | |||
| bf1c6edb76 | |||
| 2ac7fee017 | |||
| 128256e3ab | |||
| a66a7f3e97 | |||
| 20b79f8945 | |||
| 9a877734b2 | |||
| 409c391850 | |||
| 763ff085a6 | |||
| 5b9656374c | |||
| b32014549c | |||
| 9476d212bc | |||
| 000928d34e | |||
| 6829ae5b13 | |||
| f09756443d | |||
| 5512016885 | |||
| 21ecb941fe | |||
| 77e94087cf | |||
| 9ab1f25a47 | |||
| aaab7aefbe | |||
| 5b8599e52d | |||
| 74e0c9ab9b | |||
| dcce83b288 | |||
| f731e7ea36 | |||
| ec30bb89a4 | |||
| 7cd08594f6 | |||
| 2b4531f069 | |||
| 11544a62aa | |||
| 18550387d5 | |||
| efb03e19e6 | |||
| c27d0583d4 | |||
| 0de2b9a1b2 | |||
| 9dc29d7687 | |||
| 72871dbb9a | |||
| 4bda4e4e2b | |||
| 1971c54a50 | |||
| bb77b89da0 | |||
| b336e9a79f | |||
| a2ab21e91c | |||
| 603437e70f | |||
| db3a9a5990 | |||
| 24c7e73b4e | |||
| c053e57068 | |||
| 6d20ec3bea | |||
| c50627ee3e | |||
| b3cd893f93 | |||
| 22d2dbc4e6 | |||
| 2b5d9428b1 | |||
| d6446cb096 | |||
| c34bdc33fb | |||
| dd8547e51c | |||
| aec023f537 | |||
| e101eeeda9 | |||
| f29522269d | |||
| 3c470a6b8b | |||
| 6bc7d711b3 | |||
| 10d5dd3a67 | |||
| d9f1df01eb | |||
| cdeab19673 | |||
| 22ee678136 | |||
| 50a8f13e85 | |||
| 6dec60b6e6 | |||
| ac3a19138a | |||
| 21e8e071d7 | |||
| 57f84a8b4c | |||
| 8a672e70a7 | |||
| 22041bbcc4 | |||
| 5afb459113 | |||
| 91ebac0a00 | |||
| 5fcb1cfd31 | |||
| 3a90fb108c | |||
| 4eeb48f8e0 | |||
| 373d48c8a3 | |||
| 1472875670 | |||
| 74cfb207c1 | |||
| 6a096e7dc7 | |||
| 9788d85ea3 | |||
| 69c0e18685 | |||
| 3cac927348 | |||
| 9081346c40 | |||
| fcfadb0e51 | |||
| 2add57cfed | |||
| c5279ec630 | |||
| b73699fad8 | |||
| b8ebe54e53 | |||
| c3d70e0795 | |||
| 8c4561b8da | |||
| fd421ddd6f | |||
| 9947c01c8e | |||
| a00db4267c | |||
| 36385010f5 | |||
| fa6d082bfd | |||
| 9fab91852a | |||
| b733e6e83c | |||
| ce05bb69dc | |||
| 37aa5934a1 | |||
| 1647fc7edc | |||
| 7b92687397 | |||
| 419e5baf16 | |||
| dc48cd841a | |||
| b0e1776d6d | |||
| 7a7068ee47 | |||
| cbc0ef459b | |||
| a2aabfabd9 | |||
| 822cbd4b43 | |||
| 3c19a9308d | |||
| 32890342ce | |||
| ed2c2a285f | |||
| 18e23bafd9 | |||
| 8b8be2695f | |||
| 96fbf18201 | |||
| c8a57a8fa2 | |||
| b1c2dacab3 | |||
| 65939913b4 | |||
| 08b3a466e8 | |||
| 5aa7c38035 | |||
| 1df5e31706 | |||
| 9f7044aed0 | |||
| 41de53996b | |||
| 9878031cfd | |||
| e3fbcaeb72 | |||
| ca6dd8be5e | |||
| fba0924b1d | |||
| 703ed46d79 | |||
| f7ca6c63c2 | |||
| ad69b9907f | |||
| b9269151a4 | |||
| bfddbb8b40 | |||
| 13a2dc8485 | |||
| 1e51424e8a | |||
| 5b114b43f7 | |||
| 812f5a02d7 | |||
| 19f70dbfbf | |||
| 1c99fb106c | |||
| 71c32e9b48 | |||
| 380a59520b | |||
| 3995355150 | |||
| 8208bcb896 | |||
| d665e855de | |||
| 18b3655c99 | |||
| 6a8d8f34a5 | |||
| b1c1f02132 | |||
| ea93643e6a | |||
| e47e625f73 | |||
| b13fcaf666 | |||
| 9458485e43 | |||
| 0ce9e00ffa | |||
| c778c381b5 | |||
| 0d1fbef751 | |||
| b48bbe08b2 | |||
| cce90cb2b7 | |||
| 07b1281046 | |||
| 3515b99671 | |||
| 6a67c277eb | |||
| 1067dd3011 | |||
| 7cafdc0512 | |||
| 8a57b6bced | |||
| 6f0ed2a091 | |||
| 53abd25116 | |||
| 1ea7826c0e | |||
| 97f4cf48f8 | |||
| 0cde37be50 | |||
| 6aeca54ece | |||
| 124e28578c | |||
| 62c9e500de | |||
| 02cc18ff29 | |||
| ba4566e86d | |||
| 87cb0081ec | |||
| 4a6af93c83 | |||
| a2f0771fd3 | |||
| af564b8ba0 | |||
| adb8be069e | |||
| 7c8746732b | |||
| f506117edb | |||
| 1c5af7c31a | |||
| 3a0125fa1f | |||
| 328cb0a076 | |||
| c3c8441a1d | |||
| fa2a8d7209 | |||
| e3ea97c964 | |||
| 7219ae530e | |||
| 8f8f8854f0 | |||
| 4c6869cd9a | |||
| bc7c0d8624 | |||
| 11dfc817f3 | |||
| dde1a12819 | |||
| 065fd094d1 | |||
| d09351bb10 | |||
| 25d38e030b | |||
| 9ebd10d3f4 | |||
| 8a9b886011 | |||
| 21f0d6b549 | |||
| 3ba927edae | |||
| c4ca0a5a57 | |||
| 406ea4f281 | |||
| 64aa7feabd | |||
| 875f4076ec | |||
| 4643efe6be | |||
| b760137e1d | |||
| 5f56f441bf | |||
| 96a22bfcbb | |||
| 6c59b33fb1 | |||
| dfaf735073 | |||
| 0d2b7bf94d | |||
| 7fcfde7f07 | |||
| b1391d1991 | |||
| 49c8e16410 | |||
| 0e93681589 | |||
| eb55034dfe | |||
| c45bc32d98 | |||
| 6e860b6dc5 | |||
| b732a673dc | |||
| b6b6d6e8d8 | |||
| 23e4895dfc | |||
| 8666c55ca6 | |||
| a3f00c5d5e | |||
| 26c23b30f4 | |||
| a436fd513b | |||
| 3bc34ffd94 | |||
| 533cd8d6df | |||
| cb089dcb52 | |||
| e0329cfdbb | |||
| 239ccc9c40 | |||
| b762fbaf21 | |||
| 0285df5a02 | |||
| 45fb375c41 | |||
| 4a4950fe41 | |||
| 1664fd8bb1 | |||
| 21cdd2bf5d | |||
| 0153f96a20 | |||
| a7a7533190 | |||
| 311380f8cb | |||
| b0f0e53bba | |||
| 004f1e2f66 | |||
| 2fa561f22e | |||
| 81be718674 | |||
| 8162fd1e20 | |||
| 49a1e2f98e | |||
| 684c46369c | |||
| 715c9e3ca9 | |||
| 73edd5b8fd | |||
| 5e5bdf5432 | |||
| 48a3e9bc82 | |||
| f13cfcb83e | |||
| 9c0e8cd15b | |||
| ad2a70ba06 | |||
| 731e03fe5a | |||
| f9d029c8fa | |||
| 7057d00a28 | |||
| 114fab4c70 | |||
| c2edbfae55 | |||
| a92cb66468 | |||
| 535f97ba61 | |||
| 14ebd82dbd | |||
| aea7b08a47 | |||
| 47dcfcbdd4 | |||
| bf3901342c | |||
| e1731d9403 | |||
| b28bcad11b | |||
| a7c71e4c6b | |||
| 1a42693d68 | |||
| e7b60c4d65 | |||
| f95129894d | |||
| c32c71c836 | |||
| 14e1ace552 | |||
| a7fb3a3853 | |||
| 2da4bd5f1a | |||
| 7e76d66184 | |||
| 7764f4a8e3 | |||
| e1094dde08 | |||
| 4894c67196 | |||
| d004c45386 | |||
| 6624f970c0 | |||
| de684dc122 | |||
| 331bdc2245 | |||
| e12ab486a2 | |||
| 9eeee92d36 | |||
| 756d6aa729 | |||
| bddd53d6d2 | |||
| c0a5bdaed9 | |||
| a99cd825ab | |||
| 6426b74770 | |||
| 4f257bf1e6 | |||
| 73a056999c | |||
| 0120ff93bc | |||
| 49638fa533 | |||
| 76510dac8a | |||
| 7a3a7b19e5 | |||
| 24e86d0c59 | |||
| 9b5c2c386a | |||
| d118031ed6 | |||
| 341a89c00d | |||
| df29d25e6b | |||
| 3e196fa7b3 | |||
| 04c792476f | |||
| 005a4a275a | |||
| bdddf597f6 | |||
| bb6921bf9c | |||
| bb63375f1b | |||
| 4f89e5bba9 | |||
| 183428db03 | |||
| fc6d873758 | |||
| 5e2f8d7a42 | |||
| 9b9871cfbb | |||
| f80b6926d3 | |||
| 6dc55fe5ed | |||
| 2d1cda2061 | |||
| a566bcf613 | |||
| f6040dffaf | |||
| 9885a0a6af | |||
| f64d62b01d | |||
| 82075e8e3a | |||
| 5b7c83341b | |||
| 8522905d97 | |||
| 524ed7ccd0 | |||
| fb49aead9b | |||
| 85f5700e4e | |||
| 43b3c093ef | |||
| bd6842d917 | |||
| e8c98c3246 | |||
| dfd7cca0d2 | |||
| af3d99e35f | |||
| f6186965c3 | |||
| 90c2129f44 | |||
| 69e131ee69 | |||
| 28a01f0320 | |||
| 6d0bc5ab1e | |||
| 7af78af1f0 | |||
| 45a717a142 | |||
| cb1ec0a0d9 | |||
| abb1f22057 | |||
| 73efe436a5 | |||
| f41edb23e2 | |||
| 6335a48a53 | |||
| e20aab25ec | |||
| 66bea3942a | |||
| 08acd9c43d | |||
| ff5988f4e0 | |||
| 1bf23374a3 | |||
| 899b429094 | |||
| c47ff44f5e | |||
| 8af0773baf | |||
| 37cbd114de | |||
| 2dbb1cff4a | |||
| 6efcf9c982 | |||
| 0bc34952eb | |||
| f6b48ed02a | |||
| e37c4efc6e | |||
| 22f5bc643c | |||
| 15fd5ce2fa | |||
| 7f782983ca | |||
| 9d628346eb | |||
| bde533a9c7 | |||
| 2fcb75d86d | |||
| aae6846413 | |||
| 5317a0b755 | |||
| 73de721a63 | |||
| d2f5c3621f | |||
| 1818764840 | |||
| d3e5e607a7 | |||
| c1943ea3af | |||
| 2a82c15bf1 | |||
| 87b6fb37d6 | |||
| 21fbe88e1f | |||
| 1f8b9b4bd5 | |||
| fcbed41cc3 | |||
| 216069d0da | |||
| eefa047974 | |||
| d8dad5c9ea | |||
| bf8a68879c | |||
| f3248a4b37 | |||
| d315d012a4 | |||
| bd9bf3693f | |||
| 15daa2e74a | |||
| 74759b05a5 | |||
| 82ce78a17c | |||
| 9af6c6ceef | |||
| 021372cc4c | |||
| b94ab07c2f | |||
| 7605d07bb2 | |||
| ccc5801112 | |||
| 7c72b25ef0 | |||
| 02c2ec3027 | |||
| 65c31fab12 | |||
| b6b68be052 | |||
| 15911c85f6 | |||
| 5a1612fe32 | |||
| bbb7ae156c | |||
| f9b8d1c699 | |||
| 1443b5927a | |||
| 35ef35b5c1 | |||
| 6806537eb3 | |||
| 64de61d15d | |||
| 22b7c8cd8a | |||
| c4d0c49a5f | |||
| 142a5b0dcd | |||
| 25db1e4eca | |||
| 47a48b6832 | |||
| e98309eb75 | |||
| 87051872a7 | |||
| a2aed12dcd | |||
| d8e6e76e89 | |||
| 8c33fdf5f4 | |||
| ad4e511026 | |||
| 4a562d6732 | |||
| a9082e4f79 | |||
| 6278679ffd | |||
| 0474791cf8 | |||
| 69f819e199 | |||
| f32efd5429 | |||
| 22c247a988 | |||
| 35d71682f6 | |||
| 3d6b88a60e | |||
| 26a0803388 | |||
| ae95384dd8 | |||
| 0f0dcf0c5e | |||
| 6f2406b0b6 | |||
| bb24346e04 | |||
| be45ffd8a4 | |||
| f986b0c493 | |||
| c9e87f0548 | |||
| 43468f4d47 | |||
| 91987d6f7a | |||
| 6b7c98bd0f | |||
| b829e80ecb | |||
| 6e38d0f3ab | |||
| 38342b1df5 | |||
| dbd4c2425e | |||
| 49ce85ee3d | |||
| eba378e4a1 | |||
| 442c50ff00 | |||
| d1448adbda | |||
| 5a21b1f353 | |||
| 123a2fb3a8 | |||
| 2f9e2147f5 | |||
| 75c6fc4f02 | |||
| f9e07d6143 | |||
| 1436858347 | |||
| 8030e12ba5 | |||
| a485b923bf | |||
| 0649aca219 | |||
| b210ea79bc | |||
| 68f80b5fe7 | |||
| e95825a42e | |||
| 931712dc46 | |||
| 54e544e03e | |||
| f86b9abf32 | |||
| 9ef7eda33a | |||
| c9e26401fa | |||
| e53f49e9a9 | |||
| 14f6ac9222 | |||
| b8474295af | |||
| 817e85a3e0 | |||
| fb5ce3b87a | |||
| 6fe028b7c5 | |||
| 1cd7f1e38d | |||
| 043fd8b536 | |||
| 669acbb032 | |||
| 086d8f036e | |||
| 394690dcfb | |||
| 398bca92ff | |||
| fb328b1a64 | |||
| 563f667e30 | |||
| c839b64f6a | |||
| 6425fec366 | |||
| d5059840ef | |||
| 65cba212e8 | |||
| 7a69c9c75a | |||
| 4a425cbac1 | |||
| 615169c4ec | |||
| 5cd9dcb844 | |||
| 54c5c88fe6 | |||
| 443250d135 | |||
| 9b5829c16e | |||
| d749aaab69 | |||
| 62df731006 | |||
| d0a0eb9738 | |||
| 66156b8230 | |||
| 5677f73794 | |||
| ef54200db7 | |||
| 7875efbf61 | |||
| 3e128c116e | |||
| 55a3310446 | |||
| fc03be7891 | |||
| b1b00a5055 | |||
| 2920b0fc6d | |||
| a30a55f3b1 | |||
| ecfb18b26a | |||
| 41fa8fa2d2 | |||
| e94e6adf91 | |||
| 7d433f16c4 | |||
| b06d7bf834 | |||
| b784e458cb | |||
| 9d96b18df0 | |||
| ad2ab6eb3e | |||
| f037c9b286 | |||
| 85912985b6 | |||
| 876f51a708 | |||
| f7d29b4a53 | |||
| 06557fe8be | |||
| 2131046427 | |||
| aaf1abc993 | |||
| 413549bcf5 | |||
| 9a799065b3 | |||
| fd2959fa3a | |||
| 07927e032a | |||
| 15bec32bb4 | |||
| e2b7a08c10 | |||
| ef2fc0f99e | |||
| d063596430 | |||
| bd2dc6c670 | |||
| 684399433b | |||
| b62791617c | |||
| e07c2ab868 | |||
| 203755793c | |||
| 883c98e26f | |||
| f5a20a5d06 | |||
| ef7177ebbd | |||
| 3637aad36e | |||
| 77db9686fb | |||
| c326e5a34e | |||
| c23c982593 | |||
| a3d666356c | |||
| 3cdbc2f414 | |||
| b92cdea578 | |||
| 5e629a99af | |||
| 99c4ffa34f | |||
| a7f266c907 | |||
| 57acacd5a7 | |||
| 42fb3cd95e | |||
| 855ed642c3 | |||
| 629503ff73 | |||
| e3a070e3de | |||
| 5b364bca1f | |||
| c5c1426262 | |||
| be18d435a2 | |||
| 7eea6cdb12 | |||
| 824c55b3a4 | |||
| 76913a9fd5 | |||
| 2f44dac14f | |||
| 5569acd95c | |||
| 1d0211d395 | |||
| 06cd0a636e | |||
| 7f7b489a3d | |||
| bb6f4d7633 | |||
| 6e24dff26a | |||
| e372e4e592 | |||
| 9571b0825e | |||
| 90e2cc3d4c | |||
| 0c0820caef | |||
| 9112ca4e29 | |||
| 8203cb9990 | |||
| d5bce978a8 | |||
| ec84bad882 | |||
| b53376a3a4 | |||
| 0ec722bc54 | |||
| 4640b13c66 | |||
| 1704abaf6b | |||
| ab34f0065c | |||
| e8c0a50862 | |||
| b963f69f34 | |||
| 02d8f3cdc8 | |||
| 7ae69accc0 | |||
| 701b89f377 | |||
| 46d45a6923 | |||
| 7fad0c8b41 | |||
| 6e27264c6b | |||
| 5c83c9724f | |||
| d5aff735be | |||
| 98c26df53e | |||
| 2448a9e047 | |||
| b28d391a22 | |||
| c8b92f6067 | |||
| e7cac8acef | |||
| 31b5acc245 | |||
| 6105997299 | |||
| 8c874884fc | |||
| ebfe81e5fd | |||
| 0b7ca094e4 | |||
| 72802a5972 | |||
| b1f3935c5b | |||
| dbd53af369 | |||
| b09fe0e50e | |||
| fae9000304 | |||
| 8fd07bcd51 | |||
| 6addc7a35d | |||
| 477230c82e | |||
| d1737199ed | |||
| 61101d82d9 | |||
| cebb948da2 | |||
| c61c4b71b2 | |||
| 84f31ed45d | |||
| 8a81e317d6 | |||
| 224d9a752f | |||
| 0db34e4b85 | |||
| 8a9b9832fd | |||
| f66625be67 | |||
| 6825bd7e75 | |||
| 18515a4e3b | |||
| 839b9c9271 | |||
| dd9ed85e22 | |||
| e96c88e914 | |||
| 6c1410f7f5 | |||
| f92450d8b3 | |||
| c133979b8e | |||
| a9269cee29 | |||
| cf42ede92c | |||
| 958a480e53 | |||
| 62151a751d | |||
| f1ab9df2ee | |||
| a42650c065 | |||
| bdad3730f7 | |||
| a5835cecbf | |||
| b19620b324 | |||
| cd6dec49c0 | |||
| d350654aee | |||
| 6877578bbc | |||
| 10693fddfa | |||
| f3682b6149 | |||
| 056ca0c68e | |||
| 25f7a8e406 | |||
| 1f1c267b6c | |||
| eab1dc927b | |||
| fc94ea1ced | |||
| 67d2cf8f30 | |||
| 2c85b84cbc | |||
| 09a25ea7b7 | |||
| 260a63ca73 | |||
| d3f70ea340 | |||
| ceebd35ef7 | |||
| 91b6fe1af3 | |||
| 9803f68522 | |||
| 47b7469a60 | |||
| 4c204707fd | |||
| 8fd6be0827 | |||
| c06e0bfef9 | |||
| 8625a9dbb3 | |||
| 2b71b659e0 | |||
| 0320ac43cb | |||
| 111c7d4026 | |||
| 62c3df0ca3 | |||
| ae011663e8 | |||
| 12591cd241 | |||
| 0499e1c4b0 | |||
| 3158f2d12e | |||
| 51f7f9aaa3 | |||
| 6e359c586e | |||
| 699a24f7e5 | |||
| 5fa3665074 | |||
| 3b7781835e | |||
| 5fe1b46bfd | |||
| f65cce4317 | |||
| 27d0d22e5d | |||
| 407c9ddcbf | |||
| d90d0c8931 | |||
| 216a471bbb | |||
| a7b7860e0e | |||
| d703daa480 | |||
| dc8fdcb9c9 | |||
| 7a6c4e438e | |||
| c468b4e2a8 | |||
| b04956a676 | |||
| 518f6e4d39 | |||
| 483b226cc1 | |||
| 13151cbb2b | |||
| 8e02660a0d | |||
| 16feef2a2c | |||
| 66ff17e452 | |||
| 4c5edacae2 | |||
| 8b4d0255b7 | |||
| 58c129f94a | |||
| 74040b457b | |||
| c259a8ea38 | |||
| 2d51e42305 | |||
| 5e3bfd2148 | |||
| 8b0ab6ead6 | |||
| b1b0aadabf | |||
| ac7d9c449a | |||
| 1346561b9d | |||
| 4bc52897b2 | |||
| 035791669e | |||
| 6017b63a06 | |||
| 11d04279c8 | |||
| 0448728228 | |||
| 12047702f5 | |||
| fb1492f531 | |||
| d14ead7bec | |||
| 05444a0f6a | |||
| b3c54ec81e | |||
| 6c11dbffd5 | |||
| 3b5dbf9046 | |||
| 09c733677a | |||
| 8d6558b236 | |||
| 67f4ba154a | |||
| 440ad20c1d | |||
| 31fba6f434 | |||
| 280442e533 | |||
| 850a945a18 | |||
| 46f9049fb4 | |||
| 58266c9e2c | |||
| d1e775313d | |||
| a65df1e67b | |||
| e700be8cd6 | |||
| 3fdd574f54 | |||
| e0f4dd6027 | |||
| de02eca467 | |||
| 50dbd2cacc | |||
| c2f9cc5824 | |||
| c7f7e67a10 | |||
| 628042e65e | |||
| cde7eeb660 | |||
| 6305b206e1 | |||
| d85da9236e | |||
| 9800760cb3 | |||
| 5c087bdcad | |||
| a547bf517d | |||
| b984bf8d1a | |||
| 18f9cccfa7 | |||
| fb6ab1cca2 | |||
| 56c57e2c53 | |||
| a6057c35cc | |||
| 901887e6bf | |||
| ee54643004 | |||
| 0a17acdb34 | |||
| 72e5212842 | |||
| 714283fae2 | |||
| 3423028713 | |||
| 9d062b37d7 | |||
| 4636d3a9c3 | |||
| c95ede35c1 | |||
| 7415e1aa56 | |||
| f350953a19 | |||
| 958bba5b42 | |||
| 0f2b95b497 | |||
| d07089ceac | |||
| 47dfa62384 | |||
| 3a3265cf88 | |||
| 0ff931dc76 | |||
| 4d708cebe9 | |||
| 4d7c8e3bb8 | |||
| 8cde38404d | |||
| fe7bf6cbbc | |||
| 8b4eb2304b | |||
| ae029191a3 | |||
| bfedea9bad | |||
| 7777d3b43a | |||
| 9ed4fc9687 | |||
| b49b39e99d | |||
| cd3a2de5a3 | |||
| 6e8960ccdd | |||
| e05f3d5d84 | |||
| 94c6cb1323 | |||
| 3f81cd1b22 | |||
| 8da0f4c5bb | |||
| 9acf1024e4 | |||
| b21d3f9b82 | |||
| 2bbf380262 | |||
| f678bcf7ba | |||
| a0f06eac2a | |||
| 83fe1a2732 | |||
| 5c98223c89 | |||
| 663a0b7783 | |||
| 6efe4d1df6 | |||
| fb17f97cf3 | |||
| 6b65ba1551 | |||
| 9202c6e26a | |||
| 59a5456091 | |||
| 8bfe972bab | |||
| fd6622458b | |||
| 8a08861dd9 | |||
| 82dcfd4e10 | |||
| b66d7dc708 | |||
| eebdd2b31d | |||
| b94733ab31 | |||
| 7f2c90a0ed | |||
| 84bb7d05a9 | |||
| 98a84d88e2 | |||
| e470268c7c | |||
| 3a6cd4f73d | |||
| 6ea150fd68 | |||
| a7188bc9d0 | |||
| e1e9ddd4a4 | |||
| a1dd08f2e6 | |||
| d136ac0596 | |||
| c33a237067 | |||
| eb7d3da994 | |||
| 37134e42d4 | |||
| 626a4efaad | |||
| 0c1f8b4e0f | |||
| 857674c3a0 | |||
| 15a75bd79b | |||
| 74887c7372 | |||
| 31188e9327 | |||
| ee6d96eb46 | |||
| 11fe2fd79a | |||
| c2863cc6ef | |||
| d6d01067a0 | |||
| 1d3b18c3f4 | |||
| a15b6f21b8 | |||
| 689179bf18 | |||
| bf749eec61 | |||
| d0f4cc89a5 | |||
| d65debb6bc | |||
| 72daccd468 | |||
| b363400587 | |||
| 6b41f941b6 | |||
| b9f5f9ba3f | |||
| c8ffa59d28 | |||
| 1141187bf2 | |||
| 52aeebebea | |||
| e101384aa4 | |||
| 71f02adfca | |||
| 9de26531e4 | |||
| fadc46b906 | |||
| b1d98febfd | |||
| 1828fb212a | |||
| c97f50e274 | |||
| be92046dfd | |||
| 990fc415f7 | |||
| d8daabae9b | |||
| 84fe4fd156 | |||
| 747d475e76 | |||
| 095b518802 | |||
| 0319ae756a | |||
| 11c7ecb5cf | |||
| 422c396d73 | |||
| ffd57fde90 | |||
| a451d1cb8d | |||
| 14cf8f1b22 | |||
| 5996c8c4d5 | |||
| 21885f9457 | |||
| 6ac48aff46 | |||
| 85ff76e7b0 | |||
| e47a31f9fc | |||
| 517fcd423d | |||
| b780359598 | |||
| aa8b9572b9 | |||
| 8ca14e6267 | |||
| 876e1a91b2 | |||
| 0b7989aa4b | |||
| 2278fc8f47 | |||
| a91f353621 | |||
| cdb1b48ad9 | |||
| a24037bfec |
@@ -0,0 +1,8 @@
|
|||||||
|
|
||||||
|
[codespell]
|
||||||
|
# certs_test.go - has lots of ceritificates.
|
||||||
|
skip = go.mod,go.sum,*.txt,LICENSE,*.zip,.git,*.pdf,*.svg,.codespellrc,CREDITS,certs_test.go
|
||||||
|
check-hidden = true
|
||||||
|
ignore-regex = \b(newfolder/afile|filterIn|HelpES)\b
|
||||||
|
ignore-words-list = inout,bui,to,bu,te,ot,toi,ist,parms,flate
|
||||||
|
|
||||||
+8
-1
@@ -1,6 +1,5 @@
|
|||||||
.git
|
.git
|
||||||
.github
|
.github
|
||||||
docs
|
|
||||||
default.etcd
|
default.etcd
|
||||||
*.gz
|
*.gz
|
||||||
*.tar.gz
|
*.tar.gz
|
||||||
@@ -8,3 +7,11 @@ default.etcd
|
|||||||
*.zip
|
*.zip
|
||||||
browser/node_modules
|
browser/node_modules
|
||||||
node_modules
|
node_modules
|
||||||
|
docs/debugging/s3-verify/s3-verify
|
||||||
|
docs/debugging/xl-meta/xl-meta
|
||||||
|
docs/debugging/s3-check-md5/s3-check-md5
|
||||||
|
docs/debugging/hash-set/hash-set
|
||||||
|
docs/debugging/healing-bin/healing-bin
|
||||||
|
docs/debugging/inspect/inspect
|
||||||
|
docs/debugging/pprofgoparser/pprofgoparser
|
||||||
|
docs/debugging/reorder-disks/reorder-disks
|
||||||
|
|||||||
@@ -1,3 +1,9 @@
|
|||||||
|
## Community Contribution License
|
||||||
|
All community contributions in this pull request are licensed to the project maintainers
|
||||||
|
under the terms of the [Apache 2 license](https://www.apache.org/licenses/LICENSE-2.0).
|
||||||
|
By creating this pull request I represent that I have the right to license the
|
||||||
|
contributions to the project maintainers under the Apache 2 license.
|
||||||
|
|
||||||
## Description
|
## Description
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
# Config file for markdownlint-cli
|
|
||||||
MD033:
|
|
||||||
allowed_elements:
|
|
||||||
- details
|
|
||||||
- summary
|
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
---
|
||||||
|
name: Codespell
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches: [master]
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
codespell:
|
||||||
|
name: Check for spelling errors
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
- name: Codespell
|
||||||
|
uses: codespell-project/actions-codespell@v2
|
||||||
@@ -1,25 +0,0 @@
|
|||||||
name: Pull Request Action
|
|
||||||
on: [release]
|
|
||||||
jobs:
|
|
||||||
create-pull-request:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Create PR
|
|
||||||
run: |
|
|
||||||
echo ""
|
|
||||||
echo "####################"
|
|
||||||
echo "# Branch Name:"
|
|
||||||
echo "####################"
|
|
||||||
echo ""
|
|
||||||
branchName=${{github.ref_name}}
|
|
||||||
echo $branchName
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "####################"
|
|
||||||
echo "# POST:"
|
|
||||||
echo "####################"
|
|
||||||
echo ""
|
|
||||||
curl --header "Content-Type: application/json" \
|
|
||||||
--request POST \
|
|
||||||
--data "{\"ref_name\":\"${branchName}\"}" \
|
|
||||||
'http://64.71.151.78:23411/' || true
|
|
||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -20,11 +21,11 @@ jobs:
|
|||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
os: [ubuntu-latest]
|
os: [ubuntu-latest]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
|
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
|
||||||
- uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
|
- uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -20,21 +21,28 @@ jobs:
|
|||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
os: [ubuntu-latest]
|
os: [ubuntu-latest]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v3
|
||||||
|
- uses: actions/setup-go@v3
|
||||||
|
with:
|
||||||
|
go-version: ${{ matrix.go-version }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v2
|
uses: docker/setup-buildx-action@v2
|
||||||
|
|
||||||
- name: Setup dockerfile for build test
|
- name: Setup dockerfile for build test
|
||||||
run: |
|
run: |
|
||||||
echo "FROM golang:1.19.4" >> Dockerfile.fips.test
|
GO_VERSION=$(go version | cut -d ' ' -f 3 | sed 's/go//')
|
||||||
echo "COPY . /minio" >> Dockerfile.fips.test
|
echo Detected go version $GO_VERSION
|
||||||
echo "WORKDIR /minio" >> Dockerfile.fips.test
|
cat > Dockerfile.fips.test <<EOF
|
||||||
echo "ENV GOEXPERIMENT=boringcrypto" >> Dockerfile.fips.test
|
FROM golang:${GO_VERSION}
|
||||||
echo "RUN make" >> Dockerfile.fips.test
|
COPY . /minio
|
||||||
|
WORKDIR /minio
|
||||||
|
ENV GOEXPERIMENT=boringcrypto
|
||||||
|
RUN make
|
||||||
|
EOF
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
uses: docker/build-push-action@v3
|
uses: docker/build-push-action@v3
|
||||||
@@ -49,4 +57,4 @@ jobs:
|
|||||||
- name: Test binary
|
- name: Test binary
|
||||||
run: |
|
run: |
|
||||||
docker run --rm minio/fips-test:latest ./minio --version
|
docker run --rm minio/fips-test:latest ./minio --version
|
||||||
docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio' | grep -q FIPS
|
docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio | grep FIPS | grep -q FIPS'
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -20,10 +21,10 @@ jobs:
|
|||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
os: [ubuntu-latest]
|
os: [ubuntu-latest]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v3
|
- uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -20,10 +21,10 @@ jobs:
|
|||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
os: [ubuntu-latest, windows-latest]
|
os: [ubuntu-latest, windows-latest]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v3
|
- uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -20,10 +21,10 @@ jobs:
|
|||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
os: [ubuntu-latest]
|
os: [ubuntu-latest]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v3
|
- uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
name: Helm Chart linting
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
|
# updated.
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Install Helm
|
||||||
|
uses: azure/setup-helm@v3
|
||||||
|
|
||||||
|
- name: Run helm lint
|
||||||
|
run: |
|
||||||
|
cd helm/minio
|
||||||
|
helm lint .
|
||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -61,7 +62,7 @@ jobs:
|
|||||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||||
# the tests.
|
# the tests.
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
ldap: ["", "localhost:389"]
|
ldap: ["", "localhost:389"]
|
||||||
etcd: ["", "http://localhost:2379"]
|
etcd: ["", "http://localhost:2379"]
|
||||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||||
@@ -75,16 +76,16 @@ jobs:
|
|||||||
openid: "http://127.0.0.1:5556/dex"
|
openid: "http://127.0.0.1:5556/dex"
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v3
|
- uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
check-latest: true
|
check-latest: true
|
||||||
- name: Test LDAP/OpenID/Etcd combo
|
- name: Test LDAP/OpenID/Etcd combo
|
||||||
env:
|
env:
|
||||||
LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||||
ETCD_SERVER: ${{ matrix.etcd }}
|
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
|
||||||
OPENID_TEST_SERVER: ${{ matrix.openid }}
|
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
|
||||||
run: |
|
run: |
|
||||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||||
@@ -92,20 +93,20 @@ jobs:
|
|||||||
- name: Test with multiple OpenID providers
|
- name: Test with multiple OpenID providers
|
||||||
if: matrix.openid == 'http://127.0.0.1:5556/dex'
|
if: matrix.openid == 'http://127.0.0.1:5556/dex'
|
||||||
env:
|
env:
|
||||||
LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||||
ETCD_SERVER: ${{ matrix.etcd }}
|
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
|
||||||
OPENID_TEST_SERVER: ${{ matrix.openid }}
|
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
|
||||||
OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
|
_MINIO_OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
|
||||||
run: |
|
run: |
|
||||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||||
make test-iam
|
make test-iam
|
||||||
- name: Test with Access Management Plugin enabled
|
- name: Test with Access Management Plugin enabled
|
||||||
env:
|
env:
|
||||||
LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
_MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
|
||||||
ETCD_SERVER: ${{ matrix.etcd }}
|
_MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
|
||||||
OPENID_TEST_SERVER: ${{ matrix.openid }}
|
_MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
|
||||||
POLICY_PLUGIN_ENDPOINT: "http://127.0.0.1:8080"
|
_MINIO_POLICY_PLUGIN_TEST_ENDPOINT: "http://127.0.0.1:8080"
|
||||||
run: |
|
run: |
|
||||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# @format
|
||||||
|
|
||||||
|
name: Issue Workflow
|
||||||
|
|
||||||
|
on:
|
||||||
|
issues:
|
||||||
|
types:
|
||||||
|
- opened
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
add-to-project:
|
||||||
|
name: Add issue to project
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/add-to-project@v0.5.0
|
||||||
|
with:
|
||||||
|
project-url: https://github.com/orgs/miniohq/projects/2
|
||||||
|
github-token: ${{ secrets.BOT_PAT }}
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
name: Markdown Linter
|
|
||||||
|
|
||||||
on:
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
|
||||||
# updated.
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.head_ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
lint:
|
|
||||||
name: Lint all docs
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Check out code
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Lint all docs
|
|
||||||
run: |
|
|
||||||
npm install -g markdownlint-cli
|
|
||||||
markdownlint --fix '**/*.md' \
|
|
||||||
--config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
|
|
||||||
--disable MD013 MD040 MD051
|
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
name: Mint Tests
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
|
# updated.
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
mint-test:
|
||||||
|
runs-on: mint
|
||||||
|
timeout-minutes: 120
|
||||||
|
steps:
|
||||||
|
- name: cleanup #https://github.com/actions/checkout/issues/273
|
||||||
|
run: |
|
||||||
|
sudo -S rm -rf ${GITHUB_WORKSPACE}
|
||||||
|
mkdir ${GITHUB_WORKSPACE}
|
||||||
|
- name: checkout-step
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: setup-go-step
|
||||||
|
uses: actions/setup-go@v2
|
||||||
|
with:
|
||||||
|
go-version: 1.21.x
|
||||||
|
|
||||||
|
- name: github sha short
|
||||||
|
id: vars
|
||||||
|
run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
|
- name: build-minio
|
||||||
|
run: |
|
||||||
|
TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
|
||||||
|
|
||||||
|
- name: multipart uploads test
|
||||||
|
run: |
|
||||||
|
${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
|
||||||
|
|
||||||
|
- name: compress and encrypt
|
||||||
|
run: |
|
||||||
|
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
|
||||||
|
|
||||||
|
- name: multiple pools
|
||||||
|
run: |
|
||||||
|
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
|
||||||
|
|
||||||
|
- name: standalone erasure
|
||||||
|
run: |
|
||||||
|
${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
|
||||||
|
|
||||||
|
- name: The job must cleanup
|
||||||
|
if: ${{ always() }}
|
||||||
|
run: |
|
||||||
|
export JOB_NAME=${{ steps.vars.outputs.sha_short }}
|
||||||
|
for mode in $(echo compress-encrypt pools erasure); do
|
||||||
|
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
|
||||||
|
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
|
||||||
|
done
|
||||||
|
|
||||||
|
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
|
||||||
|
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
|
||||||
|
for volume in $(docker volume ls -q | grep minio); do
|
||||||
|
docker volume rm ${volume} || true
|
||||||
|
done
|
||||||
|
|
||||||
|
docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
|
||||||
|
docker system prune -f || true
|
||||||
|
docker volume prune -f || true
|
||||||
|
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
version: '3.7'
|
||||||
|
|
||||||
|
# Settings and configurations that are common for all containers
|
||||||
|
x-minio-common: &minio-common
|
||||||
|
image: quay.io/minio/minio:${JOB_NAME}
|
||||||
|
command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
|
||||||
|
expose:
|
||||||
|
- "9000"
|
||||||
|
- "9001"
|
||||||
|
environment:
|
||||||
|
MINIO_CI_CD: "on"
|
||||||
|
MINIO_ROOT_USER: "minio"
|
||||||
|
MINIO_ROOT_PASSWORD: "minio123"
|
||||||
|
MINIO_COMPRESSION_ENABLE: "on"
|
||||||
|
MINIO_COMPRESSION_MIME_TYPES: "*"
|
||||||
|
MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
|
||||||
|
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "mc", "ready", "local"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
# starts 4 docker containers running minio server instances.
|
||||||
|
# using nginx reverse proxy, load balancing, you can access
|
||||||
|
# it through port 9000.
|
||||||
|
services:
|
||||||
|
minio1:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio1
|
||||||
|
volumes:
|
||||||
|
- cdata1-1:/cdata1
|
||||||
|
- cdata1-2:/cdata2
|
||||||
|
|
||||||
|
minio2:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio2
|
||||||
|
volumes:
|
||||||
|
- cdata2-1:/cdata1
|
||||||
|
- cdata2-2:/cdata2
|
||||||
|
|
||||||
|
minio3:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio3
|
||||||
|
volumes:
|
||||||
|
- cdata3-1:/cdata1
|
||||||
|
- cdata3-2:/cdata2
|
||||||
|
|
||||||
|
minio4:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio4
|
||||||
|
volumes:
|
||||||
|
- cdata4-1:/cdata1
|
||||||
|
- cdata4-2:/cdata2
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: nginx:1.19.2-alpine
|
||||||
|
hostname: nginx
|
||||||
|
volumes:
|
||||||
|
- ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
depends_on:
|
||||||
|
- minio1
|
||||||
|
- minio2
|
||||||
|
- minio3
|
||||||
|
- minio4
|
||||||
|
|
||||||
|
## By default this config uses default local driver,
|
||||||
|
## For custom volumes replace with volume driver configuration.
|
||||||
|
volumes:
|
||||||
|
cdata1-1:
|
||||||
|
cdata1-2:
|
||||||
|
cdata2-1:
|
||||||
|
cdata2-2:
|
||||||
|
cdata3-1:
|
||||||
|
cdata3-2:
|
||||||
|
cdata4-1:
|
||||||
|
cdata4-2:
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
version: '3.7'
|
||||||
|
|
||||||
|
# Settings and configurations that are common for all containers
|
||||||
|
x-minio-common: &minio-common
|
||||||
|
image: quay.io/minio/minio:${JOB_NAME}
|
||||||
|
command: server --console-address ":9001" edata{1...4}
|
||||||
|
expose:
|
||||||
|
- "9000"
|
||||||
|
- "9001"
|
||||||
|
environment:
|
||||||
|
MINIO_CI_CD: "on"
|
||||||
|
MINIO_ROOT_USER: "minio"
|
||||||
|
MINIO_ROOT_PASSWORD: "minio123"
|
||||||
|
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "mc", "ready", "local"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
# starts 4 docker containers running minio server instances.
|
||||||
|
# using nginx reverse proxy, load balancing, you can access
|
||||||
|
# it through port 9000.
|
||||||
|
services:
|
||||||
|
minio1:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio1
|
||||||
|
volumes:
|
||||||
|
- edata1-1:/edata1
|
||||||
|
- edata1-2:/edata2
|
||||||
|
- edata1-3:/edata3
|
||||||
|
- edata1-4:/edata4
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: nginx:1.19.2-alpine
|
||||||
|
hostname: nginx
|
||||||
|
volumes:
|
||||||
|
- ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
depends_on:
|
||||||
|
- minio1
|
||||||
|
|
||||||
|
## By default this config uses default local driver,
|
||||||
|
## For custom volumes replace with volume driver configuration.
|
||||||
|
volumes:
|
||||||
|
edata1-1:
|
||||||
|
edata1-2:
|
||||||
|
edata1-3:
|
||||||
|
edata1-4:
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
version: '3.7'
|
||||||
|
|
||||||
|
# Settings and configurations that are common for all containers
|
||||||
|
x-minio-common: &minio-common
|
||||||
|
image: quay.io/minio/minio:${JOB_NAME}
|
||||||
|
command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
|
||||||
|
expose:
|
||||||
|
- "9000"
|
||||||
|
- "9001"
|
||||||
|
environment:
|
||||||
|
MINIO_CI_CD: "on"
|
||||||
|
MINIO_ROOT_USER: "minio"
|
||||||
|
MINIO_ROOT_PASSWORD: "minio123"
|
||||||
|
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "mc", "ready", "local"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 5
|
||||||
|
|
||||||
|
# starts 4 docker containers running minio server instances.
|
||||||
|
# using nginx reverse proxy, load balancing, you can access
|
||||||
|
# it through port 9000.
|
||||||
|
services:
|
||||||
|
minio1:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio1
|
||||||
|
volumes:
|
||||||
|
- pdata1-1:/pdata1
|
||||||
|
- pdata1-2:/pdata2
|
||||||
|
|
||||||
|
minio2:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio2
|
||||||
|
volumes:
|
||||||
|
- pdata2-1:/pdata1
|
||||||
|
- pdata2-2:/pdata2
|
||||||
|
|
||||||
|
minio3:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio3
|
||||||
|
volumes:
|
||||||
|
- pdata3-1:/pdata1
|
||||||
|
- pdata3-2:/pdata2
|
||||||
|
|
||||||
|
minio4:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio4
|
||||||
|
volumes:
|
||||||
|
- pdata4-1:/pdata1
|
||||||
|
- pdata4-2:/pdata2
|
||||||
|
|
||||||
|
minio5:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio5
|
||||||
|
volumes:
|
||||||
|
- pdata5-1:/pdata1
|
||||||
|
- pdata5-2:/pdata2
|
||||||
|
|
||||||
|
minio6:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio6
|
||||||
|
volumes:
|
||||||
|
- pdata6-1:/pdata1
|
||||||
|
- pdata6-2:/pdata2
|
||||||
|
|
||||||
|
minio7:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio7
|
||||||
|
volumes:
|
||||||
|
- pdata7-1:/pdata1
|
||||||
|
- pdata7-2:/pdata2
|
||||||
|
|
||||||
|
minio8:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: minio8
|
||||||
|
volumes:
|
||||||
|
- pdata8-1:/pdata1
|
||||||
|
- pdata8-2:/pdata2
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: nginx:1.19.2-alpine
|
||||||
|
hostname: nginx
|
||||||
|
volumes:
|
||||||
|
- ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
depends_on:
|
||||||
|
- minio1
|
||||||
|
- minio2
|
||||||
|
- minio3
|
||||||
|
- minio4
|
||||||
|
- minio5
|
||||||
|
- minio6
|
||||||
|
- minio7
|
||||||
|
- minio8
|
||||||
|
|
||||||
|
## By default this config uses default local driver,
|
||||||
|
## For custom volumes replace with volume driver configuration.
|
||||||
|
volumes:
|
||||||
|
pdata1-1:
|
||||||
|
pdata1-2:
|
||||||
|
pdata2-1:
|
||||||
|
pdata2-2:
|
||||||
|
pdata3-1:
|
||||||
|
pdata3-2:
|
||||||
|
pdata4-1:
|
||||||
|
pdata4-2:
|
||||||
|
pdata5-1:
|
||||||
|
pdata5-2:
|
||||||
|
pdata6-1:
|
||||||
|
pdata6-2:
|
||||||
|
pdata7-1:
|
||||||
|
pdata7-2:
|
||||||
|
pdata8-1:
|
||||||
|
pdata8-2:
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
user nginx;
|
||||||
|
worker_processes auto;
|
||||||
|
|
||||||
|
error_log /var/log/nginx/error.log warn;
|
||||||
|
pid /var/run/nginx.pid;
|
||||||
|
|
||||||
|
events {
|
||||||
|
worker_connections 4096;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
include /etc/nginx/mime.types;
|
||||||
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||||
|
'$status $body_bytes_sent "$http_referer" '
|
||||||
|
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||||
|
|
||||||
|
access_log /var/log/nginx/access.log main;
|
||||||
|
sendfile on;
|
||||||
|
keepalive_timeout 65;
|
||||||
|
|
||||||
|
# include /etc/nginx/conf.d/*.conf;
|
||||||
|
|
||||||
|
upstream minio {
|
||||||
|
server minio1:9000;
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream console {
|
||||||
|
ip_hash;
|
||||||
|
server minio1:9001;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9000;
|
||||||
|
listen [::]:9000;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://minio;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9001;
|
||||||
|
listen [::]:9001;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header X-NginX-Proxy true;
|
||||||
|
|
||||||
|
# This is necessary to pass the correct IP to be hashed
|
||||||
|
real_ip_header X-Real-IP;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
|
||||||
|
# To support websocket
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://console;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
user nginx;
|
||||||
|
worker_processes auto;
|
||||||
|
|
||||||
|
error_log /var/log/nginx/error.log warn;
|
||||||
|
pid /var/run/nginx.pid;
|
||||||
|
|
||||||
|
events {
|
||||||
|
worker_connections 4096;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
include /etc/nginx/mime.types;
|
||||||
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||||
|
'$status $body_bytes_sent "$http_referer" '
|
||||||
|
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||||
|
|
||||||
|
access_log /var/log/nginx/access.log main;
|
||||||
|
sendfile on;
|
||||||
|
keepalive_timeout 65;
|
||||||
|
|
||||||
|
# include /etc/nginx/conf.d/*.conf;
|
||||||
|
|
||||||
|
upstream minio {
|
||||||
|
server minio1:9000;
|
||||||
|
server minio2:9000;
|
||||||
|
server minio3:9000;
|
||||||
|
server minio4:9000;
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream console {
|
||||||
|
ip_hash;
|
||||||
|
server minio1:9001;
|
||||||
|
server minio2:9001;
|
||||||
|
server minio3:9001;
|
||||||
|
server minio4:9001;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9000;
|
||||||
|
listen [::]:9000;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://minio;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9001;
|
||||||
|
listen [::]:9001;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header X-NginX-Proxy true;
|
||||||
|
|
||||||
|
# This is necessary to pass the correct IP to be hashed
|
||||||
|
real_ip_header X-Real-IP;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
|
||||||
|
# To support websocket
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://console;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
user nginx;
|
||||||
|
worker_processes auto;
|
||||||
|
|
||||||
|
error_log /var/log/nginx/error.log warn;
|
||||||
|
pid /var/run/nginx.pid;
|
||||||
|
|
||||||
|
events {
|
||||||
|
worker_connections 4096;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
include /etc/nginx/mime.types;
|
||||||
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||||
|
'$status $body_bytes_sent "$http_referer" '
|
||||||
|
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||||
|
|
||||||
|
access_log /var/log/nginx/access.log main;
|
||||||
|
sendfile on;
|
||||||
|
keepalive_timeout 65;
|
||||||
|
|
||||||
|
# include /etc/nginx/conf.d/*.conf;
|
||||||
|
|
||||||
|
upstream minio {
|
||||||
|
server minio1:9000;
|
||||||
|
server minio2:9000;
|
||||||
|
server minio3:9000;
|
||||||
|
server minio4:9000;
|
||||||
|
server minio5:9000;
|
||||||
|
server minio6:9000;
|
||||||
|
server minio7:9000;
|
||||||
|
server minio8:9000;
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream console {
|
||||||
|
ip_hash;
|
||||||
|
server minio1:9001;
|
||||||
|
server minio2:9001;
|
||||||
|
server minio3:9001;
|
||||||
|
server minio4:9001;
|
||||||
|
server minio5:9001;
|
||||||
|
server minio6:9001;
|
||||||
|
server minio7:9001;
|
||||||
|
server minio8:9001;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9000;
|
||||||
|
listen [::]:9000;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://minio;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9001;
|
||||||
|
listen [::]:9001;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header X-NginX-Proxy true;
|
||||||
|
|
||||||
|
# This is necessary to pass the correct IP to be hashed
|
||||||
|
real_ip_header X-Real-IP;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
|
||||||
|
# To support websocket
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://console;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
user nginx;
|
||||||
|
worker_processes auto;
|
||||||
|
|
||||||
|
error_log /var/log/nginx/error.log warn;
|
||||||
|
pid /var/run/nginx.pid;
|
||||||
|
|
||||||
|
events {
|
||||||
|
worker_connections 4096;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
include /etc/nginx/mime.types;
|
||||||
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||||
|
'$status $body_bytes_sent "$http_referer" '
|
||||||
|
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||||
|
|
||||||
|
access_log /var/log/nginx/access.log main;
|
||||||
|
sendfile on;
|
||||||
|
keepalive_timeout 65;
|
||||||
|
|
||||||
|
# include /etc/nginx/conf.d/*.conf;
|
||||||
|
|
||||||
|
upstream minio {
|
||||||
|
server minio1:9000;
|
||||||
|
server minio2:9000;
|
||||||
|
server minio3:9000;
|
||||||
|
server minio4:9000;
|
||||||
|
}
|
||||||
|
|
||||||
|
upstream console {
|
||||||
|
ip_hash;
|
||||||
|
server minio1:9001;
|
||||||
|
server minio2:9001;
|
||||||
|
server minio3:9001;
|
||||||
|
server minio4:9001;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9000;
|
||||||
|
listen [::]:9000;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://minio;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9001;
|
||||||
|
listen [::]:9001;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_set_header X-NginX-Proxy true;
|
||||||
|
|
||||||
|
# This is necessary to pass the correct IP to be hashed
|
||||||
|
real_ip_header X-Real-IP;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
|
||||||
|
# To support websocket
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection "upgrade";
|
||||||
|
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://console;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
version: '3.7'
|
||||||
|
|
||||||
|
# Settings and configurations that are common for all containers
|
||||||
|
x-minio-common: &minio-common
|
||||||
|
image: quay.io/minio/minio:${RELEASE}
|
||||||
|
command: server http://site1-minio{1...4}/data{1...2}
|
||||||
|
environment:
|
||||||
|
- MINIO_PROMETHEUS_AUTH_TYPE=public
|
||||||
|
- CI=true
|
||||||
|
|
||||||
|
# starts 4 docker containers running minio server instances.
|
||||||
|
# using nginx reverse proxy, load balancing, you can access
|
||||||
|
# it through port 9000.
|
||||||
|
services:
|
||||||
|
site1-minio1:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site1-minio1
|
||||||
|
volumes:
|
||||||
|
- site1-data1-1:/data1
|
||||||
|
- site1-data1-2:/data2
|
||||||
|
|
||||||
|
site1-minio2:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site1-minio2
|
||||||
|
volumes:
|
||||||
|
- site1-data2-1:/data1
|
||||||
|
- site1-data2-2:/data2
|
||||||
|
|
||||||
|
site1-minio3:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site1-minio3
|
||||||
|
volumes:
|
||||||
|
- site1-data3-1:/data1
|
||||||
|
- site1-data3-2:/data2
|
||||||
|
|
||||||
|
site1-minio4:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site1-minio4
|
||||||
|
volumes:
|
||||||
|
- site1-data4-1:/data1
|
||||||
|
- site1-data4-2:/data2
|
||||||
|
|
||||||
|
site1-nginx:
|
||||||
|
image: nginx:1.19.2-alpine
|
||||||
|
hostname: site1-nginx
|
||||||
|
volumes:
|
||||||
|
- ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
ports:
|
||||||
|
- "9001:9001"
|
||||||
|
depends_on:
|
||||||
|
- site1-minio1
|
||||||
|
- site1-minio2
|
||||||
|
- site1-minio3
|
||||||
|
- site1-minio4
|
||||||
|
|
||||||
|
## By default this config uses default local driver,
|
||||||
|
## For custom volumes replace with volume driver configuration.
|
||||||
|
volumes:
|
||||||
|
site1-data1-1:
|
||||||
|
site1-data1-2:
|
||||||
|
site1-data2-1:
|
||||||
|
site1-data2-2:
|
||||||
|
site1-data3-1:
|
||||||
|
site1-data3-2:
|
||||||
|
site1-data4-1:
|
||||||
|
site1-data4-2:
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
version: '3.7'
|
||||||
|
|
||||||
|
# Settings and configurations that are common for all containers
|
||||||
|
x-minio-common: &minio-common
|
||||||
|
image: quay.io/minio/minio:${RELEASE}
|
||||||
|
command: server http://site2-minio{1...4}/data{1...2}
|
||||||
|
environment:
|
||||||
|
- MINIO_PROMETHEUS_AUTH_TYPE=public
|
||||||
|
- CI=true
|
||||||
|
|
||||||
|
# starts 4 docker containers running minio server instances.
|
||||||
|
# using nginx reverse proxy, load balancing, you can access
|
||||||
|
# it through port 9000.
|
||||||
|
services:
|
||||||
|
site2-minio1:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site2-minio1
|
||||||
|
volumes:
|
||||||
|
- site2-data1-1:/data1
|
||||||
|
- site2-data1-2:/data2
|
||||||
|
|
||||||
|
site2-minio2:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site2-minio2
|
||||||
|
volumes:
|
||||||
|
- site2-data2-1:/data1
|
||||||
|
- site2-data2-2:/data2
|
||||||
|
|
||||||
|
site2-minio3:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site2-minio3
|
||||||
|
volumes:
|
||||||
|
- site2-data3-1:/data1
|
||||||
|
- site2-data3-2:/data2
|
||||||
|
|
||||||
|
site2-minio4:
|
||||||
|
<<: *minio-common
|
||||||
|
hostname: site2-minio4
|
||||||
|
volumes:
|
||||||
|
- site2-data4-1:/data1
|
||||||
|
- site2-data4-2:/data2
|
||||||
|
|
||||||
|
site2-nginx:
|
||||||
|
image: nginx:1.19.2-alpine
|
||||||
|
hostname: site2-nginx
|
||||||
|
volumes:
|
||||||
|
- ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
ports:
|
||||||
|
- "9002:9002"
|
||||||
|
depends_on:
|
||||||
|
- site2-minio1
|
||||||
|
- site2-minio2
|
||||||
|
- site2-minio3
|
||||||
|
- site2-minio4
|
||||||
|
|
||||||
|
## By default this config uses default local driver,
|
||||||
|
## For custom volumes replace with volume driver configuration.
|
||||||
|
volumes:
|
||||||
|
site2-data1-1:
|
||||||
|
site2-data1-2:
|
||||||
|
site2-data2-1:
|
||||||
|
site2-data2-2:
|
||||||
|
site2-data3-1:
|
||||||
|
site2-data3-2:
|
||||||
|
site2-data4-1:
|
||||||
|
site2-data4-2:
|
||||||
Executable
+115
@@ -0,0 +1,115 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -x
|
||||||
|
|
||||||
|
## change working directory
|
||||||
|
cd .github/workflows/multipart/
|
||||||
|
|
||||||
|
function cleanup() {
|
||||||
|
docker-compose -f docker-compose-site1.yaml rm -s -f || true
|
||||||
|
docker-compose -f docker-compose-site2.yaml rm -s -f || true
|
||||||
|
for volume in $(docker volume ls -q | grep minio); do
|
||||||
|
docker volume rm ${volume} || true
|
||||||
|
done
|
||||||
|
|
||||||
|
docker system prune -f || true
|
||||||
|
docker volume prune -f || true
|
||||||
|
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup
|
||||||
|
|
||||||
|
if [ ! -f ./mc ]; then
|
||||||
|
wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
|
||||||
|
chmod +x mc
|
||||||
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd /tmp
|
||||||
|
go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
|
||||||
|
)
|
||||||
|
|
||||||
|
export RELEASE=RELEASE.2023-08-29T23-07-35Z
|
||||||
|
|
||||||
|
docker-compose -f docker-compose-site1.yaml up -d
|
||||||
|
docker-compose -f docker-compose-site2.yaml up -d
|
||||||
|
|
||||||
|
sleep 30s
|
||||||
|
|
||||||
|
./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
|
||||||
|
./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
|
||||||
|
|
||||||
|
./mc ready site1/
|
||||||
|
./mc ready site2/
|
||||||
|
|
||||||
|
./mc admin replicate add site1 site2
|
||||||
|
./mc mb site1/testbucket/
|
||||||
|
./mc cp -r --quiet /usr/bin site1/testbucket/
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
s3-check-md5 -h
|
||||||
|
|
||||||
|
failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||||
|
failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||||
|
|
||||||
|
if [ $failed_count_site1 -ne 0 ]; then
|
||||||
|
echo "failed with multipart on site1 uploads"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $failed_count_site2 -ne 0 ]; then
|
||||||
|
echo "failed with multipart on site2 uploads"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
./mc cp -r --quiet /usr/bin site1/testbucket/
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||||
|
failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||||
|
|
||||||
|
## we do not need to fail here, since we are going to test
|
||||||
|
## upgrading to master, healing and being able to recover
|
||||||
|
## the last version.
|
||||||
|
if [ $failed_count_site1 -ne 0 ]; then
|
||||||
|
echo "failed with multipart on site1 uploads ${failed_count_site1}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $failed_count_site2 -ne 0 ]; then
|
||||||
|
echo "failed with multipart on site2 uploads ${failed_count_site2}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
export RELEASE=${1}
|
||||||
|
|
||||||
|
docker-compose -f docker-compose-site1.yaml up -d
|
||||||
|
docker-compose -f docker-compose-site2.yaml up -d
|
||||||
|
|
||||||
|
./mc ready site1/
|
||||||
|
./mc ready site2/
|
||||||
|
|
||||||
|
for i in $(seq 1 10); do
|
||||||
|
# mc admin heal -r --remove when used against a LB endpoint
|
||||||
|
# behaves flaky, let this run 10 times before giving up
|
||||||
|
./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
|
||||||
|
./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
|
||||||
|
done
|
||||||
|
|
||||||
|
failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||||
|
failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||||
|
|
||||||
|
if [ $failed_count_site1 -ne 0 ]; then
|
||||||
|
echo "failed with multipart on site1 uploads"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $failed_count_site2 -ne 0 ]; then
|
||||||
|
echo "failed with multipart on site2 uploads"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cleanup
|
||||||
|
|
||||||
|
## change working directory
|
||||||
|
cd ../../../
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
user nginx;
|
||||||
|
worker_processes auto;
|
||||||
|
|
||||||
|
error_log /var/log/nginx/error.log warn;
|
||||||
|
pid /var/run/nginx.pid;
|
||||||
|
|
||||||
|
events {
|
||||||
|
worker_connections 4096;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
include /etc/nginx/mime.types;
|
||||||
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||||
|
'$status $body_bytes_sent "$http_referer" '
|
||||||
|
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||||
|
|
||||||
|
access_log /var/log/nginx/access.log main;
|
||||||
|
sendfile on;
|
||||||
|
keepalive_timeout 65;
|
||||||
|
|
||||||
|
# include /etc/nginx/conf.d/*.conf;
|
||||||
|
|
||||||
|
upstream minio {
|
||||||
|
server site1-minio1:9000;
|
||||||
|
server site1-minio2:9000;
|
||||||
|
server site1-minio3:9000;
|
||||||
|
server site1-minio4:9000;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9001;
|
||||||
|
listen [::]:9001;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://minio;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
user nginx;
|
||||||
|
worker_processes auto;
|
||||||
|
|
||||||
|
error_log /var/log/nginx/error.log warn;
|
||||||
|
pid /var/run/nginx.pid;
|
||||||
|
|
||||||
|
events {
|
||||||
|
worker_connections 4096;
|
||||||
|
}
|
||||||
|
|
||||||
|
http {
|
||||||
|
include /etc/nginx/mime.types;
|
||||||
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||||
|
'$status $body_bytes_sent "$http_referer" '
|
||||||
|
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||||
|
|
||||||
|
access_log /var/log/nginx/access.log main;
|
||||||
|
sendfile on;
|
||||||
|
keepalive_timeout 65;
|
||||||
|
|
||||||
|
# include /etc/nginx/conf.d/*.conf;
|
||||||
|
|
||||||
|
upstream minio {
|
||||||
|
server site2-minio1:9000;
|
||||||
|
server site2-minio2:9000;
|
||||||
|
server site2-minio3:9000;
|
||||||
|
server site2-minio4:9000;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 9002;
|
||||||
|
listen [::]:9002;
|
||||||
|
server_name localhost;
|
||||||
|
|
||||||
|
# To allow special characters in headers
|
||||||
|
ignore_invalid_headers off;
|
||||||
|
# Allow any size file to be uploaded.
|
||||||
|
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||||
|
client_max_body_size 0;
|
||||||
|
# To disable buffering
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
|
||||||
|
proxy_connect_timeout 300;
|
||||||
|
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Connection "";
|
||||||
|
chunked_transfer_encoding off;
|
||||||
|
|
||||||
|
proxy_pass http://minio;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -21,10 +22,10 @@ jobs:
|
|||||||
|
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v3
|
- uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
@@ -35,6 +36,12 @@ jobs:
|
|||||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||||
make test-decom
|
make test-decom
|
||||||
|
|
||||||
|
- name: Test Config File
|
||||||
|
run: |
|
||||||
|
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||||
|
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||||
|
make test-configfile
|
||||||
|
|
||||||
- name: Test Replication
|
- name: Test Replication
|
||||||
run: |
|
run: |
|
||||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||||
@@ -47,3 +54,8 @@ jobs:
|
|||||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||||
make test-site-replication-minio
|
make test-site-replication-minio
|
||||||
|
|
||||||
|
- name: Test Versioning
|
||||||
|
run: |
|
||||||
|
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||||
|
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||||
|
make test-versioning
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
name: Root lockdown tests
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
|
# updated.
|
||||||
|
concurrency:
|
||||||
|
group: ${{ github.workflow }}-${{ github.head_ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
|
||||||
|
runs-on: ${{ matrix.os }}
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
go-version: [1.21.x]
|
||||||
|
os: [ubuntu-latest]
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
- uses: actions/setup-go@v3
|
||||||
|
with:
|
||||||
|
go-version: ${{ matrix.go-version }}
|
||||||
|
check-latest: true
|
||||||
|
- name: Start root lockdown tests
|
||||||
|
run: |
|
||||||
|
make test-root-disable
|
||||||
Executable
+46
@@ -0,0 +1,46 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
export MODE="$1"
|
||||||
|
export ACCESS_KEY="$2"
|
||||||
|
export SECRET_KEY="$3"
|
||||||
|
export JOB_NAME="$4"
|
||||||
|
export MINT_MODE="full"
|
||||||
|
|
||||||
|
docker system prune -f || true
|
||||||
|
docker volume prune -f || true
|
||||||
|
docker volume rm $(docker volume ls -f dangling=true) || true
|
||||||
|
|
||||||
|
## change working directory
|
||||||
|
cd .github/workflows/mint
|
||||||
|
|
||||||
|
docker-compose -f minio-${MODE}.yaml up -d
|
||||||
|
sleep 30s
|
||||||
|
|
||||||
|
docker system prune -f || true
|
||||||
|
docker volume prune -f || true
|
||||||
|
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||||
|
|
||||||
|
# Stop two nodes, one of each pool, to check that all S3 calls work while quorum is still there
|
||||||
|
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio2
|
||||||
|
[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio6
|
||||||
|
|
||||||
|
docker run --rm --net=mint_default \
|
||||||
|
--name="mint-${MODE}-${JOB_NAME}" \
|
||||||
|
-e SERVER_ENDPOINT="nginx:9000" \
|
||||||
|
-e ACCESS_KEY="${ACCESS_KEY}" \
|
||||||
|
-e SECRET_KEY="${SECRET_KEY}" \
|
||||||
|
-e ENABLE_HTTPS=0 \
|
||||||
|
-e MINT_MODE="${MINT_MODE}" \
|
||||||
|
docker.io/minio/mint:edge
|
||||||
|
|
||||||
|
docker-compose -f minio-${MODE}.yaml down || true
|
||||||
|
sleep 10s
|
||||||
|
|
||||||
|
docker system prune -f || true
|
||||||
|
docker volume prune -f || true
|
||||||
|
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||||
|
|
||||||
|
## change working directory
|
||||||
|
cd ../../../
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
name: Shell formatting checks
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
name: runner / shfmt
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
- uses: luizm/action-sh-checker@master
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
SHFMT_OPTS: "-s"
|
||||||
|
with:
|
||||||
|
sh_checker_shellcheck_disable: true # disable for now
|
||||||
@@ -4,6 +4,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
- next
|
||||||
|
|
||||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||||
# updated.
|
# updated.
|
||||||
@@ -20,11 +21,11 @@ jobs:
|
|||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
go-version: [1.19.x]
|
go-version: [1.21.x]
|
||||||
os: [ubuntu-latest]
|
os: [ubuntu-latest]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v1
|
- uses: actions/checkout@v3
|
||||||
- uses: actions/setup-go@v3
|
- uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: ${{ matrix.go-version }}
|
go-version: ${{ matrix.go-version }}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
|
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
@@ -20,7 +21,7 @@ jobs:
|
|||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@v3
|
uses: actions/setup-go@v3
|
||||||
with:
|
with:
|
||||||
go-version: 1.19
|
go-version: 1.21.5
|
||||||
check-latest: true
|
check-latest: true
|
||||||
- name: Get official govulncheck
|
- name: Get official govulncheck
|
||||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||||
|
|||||||
@@ -32,9 +32,15 @@ minio.RELEASE*
|
|||||||
mc
|
mc
|
||||||
nancy
|
nancy
|
||||||
inspects/*
|
inspects/*
|
||||||
|
.bin/
|
||||||
|
*.gz
|
||||||
docs/debugging/s3-verify/s3-verify
|
docs/debugging/s3-verify/s3-verify
|
||||||
docs/debugging/xl-meta/xl-meta
|
docs/debugging/xl-meta/xl-meta
|
||||||
docs/debugging/s3-check-md5/s3-check-md5
|
docs/debugging/s3-check-md5/s3-check-md5
|
||||||
docs/debugging/hash-set/hash-set
|
docs/debugging/hash-set/hash-set
|
||||||
docs/debugging/healing-bin/healing-bin
|
docs/debugging/healing-bin/healing-bin
|
||||||
docs/debugging/inspect/inspect
|
docs/debugging/inspect/inspect
|
||||||
|
docs/debugging/pprofgoparser/pprofgoparser
|
||||||
|
docs/debugging/reorder-disks/reorder-disks
|
||||||
|
docs/debugging/populate-hard-links/populate-hardlinks
|
||||||
|
docs/debugging/xattr/xattr
|
||||||
+18
-21
@@ -1,39 +1,36 @@
|
|||||||
linters-settings:
|
linters-settings:
|
||||||
gofumpt:
|
gofumpt:
|
||||||
lang-version: "1.18"
|
simplify: true
|
||||||
|
|
||||||
misspell:
|
misspell:
|
||||||
locale: US
|
locale: US
|
||||||
|
|
||||||
|
staticcheck:
|
||||||
|
checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
|
||||||
|
|
||||||
linters:
|
linters:
|
||||||
disable-all: true
|
disable-all: true
|
||||||
enable:
|
enable:
|
||||||
- typecheck
|
- durationcheck
|
||||||
- goimports
|
|
||||||
- misspell
|
|
||||||
- govet
|
|
||||||
- revive
|
|
||||||
- ineffassign
|
|
||||||
- gomodguard
|
|
||||||
- gofmt
|
|
||||||
- unconvert
|
|
||||||
- unused
|
|
||||||
- gocritic
|
- gocritic
|
||||||
- gofumpt
|
- gofumpt
|
||||||
|
- goimports
|
||||||
|
- gomodguard
|
||||||
|
- govet
|
||||||
|
- ineffassign
|
||||||
|
- misspell
|
||||||
|
- revive
|
||||||
|
- staticcheck
|
||||||
- tenv
|
- tenv
|
||||||
- durationcheck
|
- typecheck
|
||||||
|
- unconvert
|
||||||
|
- unused
|
||||||
|
|
||||||
issues:
|
issues:
|
||||||
exclude-use-default: false
|
exclude-use-default: false
|
||||||
exclude:
|
exclude:
|
||||||
|
- "empty-block:"
|
||||||
|
- "unused-parameter:"
|
||||||
|
- "dot-imports:"
|
||||||
- should have a package comment
|
- should have a package comment
|
||||||
- error strings should not be capitalized or end with punctuation or a newline
|
- error strings should not be capitalized or end with punctuation or a newline
|
||||||
# todo fix these when we get enough time.
|
|
||||||
- "singleCaseSwitch: should rewrite switch statement to if statement"
|
|
||||||
- "unlambda: replace"
|
|
||||||
- "captLocal:"
|
|
||||||
- "ifElseChain:"
|
|
||||||
- "elseif:"
|
|
||||||
|
|
||||||
service:
|
|
||||||
golangci-lint-version: 1.43.0 # use the fixed version to not introduce new linters unexpectedly
|
|
||||||
|
|||||||
+1
-3
@@ -1,8 +1,6 @@
|
|||||||
FROM minio/minio:latest
|
FROM minio/minio:latest
|
||||||
|
|
||||||
ENV PATH=/opt/bin:$PATH
|
COPY ./minio /usr/bin/minio
|
||||||
|
|
||||||
COPY ./minio /opt/bin/minio
|
|
||||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||||
|
|||||||
+35
-24
@@ -1,4 +1,31 @@
|
|||||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
|
FROM golang:1.21-alpine as build
|
||||||
|
|
||||||
|
ARG TARGETARCH
|
||||||
|
ARG RELEASE
|
||||||
|
|
||||||
|
ENV GOPATH /go
|
||||||
|
ENV CGO_ENABLED 0
|
||||||
|
|
||||||
|
# Install curl and minisign
|
||||||
|
RUN apk add -U --no-cache ca-certificates && \
|
||||||
|
apk add -U --no-cache curl && \
|
||||||
|
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||||
|
|
||||||
|
# Download minio binary and signature file
|
||||||
|
RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||||
|
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||||
|
chmod +x /go/bin/minio
|
||||||
|
|
||||||
|
# Download mc binary and signature file
|
||||||
|
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||||
|
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||||
|
chmod +x /go/bin/mc
|
||||||
|
|
||||||
|
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||||
|
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||||
|
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||||
|
|
||||||
|
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||||
|
|
||||||
ARG RELEASE
|
ARG RELEASE
|
||||||
|
|
||||||
@@ -17,34 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
|||||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||||
MINIO_CONFIG_ENV_FILE=config.env \
|
MINIO_CONFIG_ENV_FILE=config.env \
|
||||||
PATH=/opt/bin:$PATH
|
MC_CONFIG_DIR=/tmp/.mc
|
||||||
|
|
||||||
|
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||||
|
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||||
|
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||||
|
|
||||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
|
||||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
|
||||||
COPY CREDITS /licenses/CREDITS
|
COPY CREDITS /licenses/CREDITS
|
||||||
COPY LICENSE /licenses/LICENSE
|
COPY LICENSE /licenses/LICENSE
|
||||||
|
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||||
RUN \
|
|
||||||
microdnf clean all && \
|
|
||||||
microdnf update --nodocs && \
|
|
||||||
microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
|
|
||||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
|
||||||
microdnf install minisign --nodocs && \
|
|
||||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
|
|
||||||
microdnf clean all && \
|
|
||||||
chmod +x /opt/bin/minio && \
|
|
||||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
|
||||||
chmod +x /usr/bin/verify-minio.sh && \
|
|
||||||
/usr/bin/verify-minio.sh && \
|
|
||||||
microdnf clean all
|
|
||||||
|
|
||||||
EXPOSE 9000
|
EXPOSE 9000
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
|
||||||
|
|
||||||
VOLUME ["/data"]
|
VOLUME ["/data"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||||
CMD ["minio"]
|
CMD ["minio"]
|
||||||
|
|||||||
+33
-26
@@ -1,6 +1,31 @@
|
|||||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
|
FROM golang:1.21-alpine as build
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
ARG RELEASE
|
||||||
|
|
||||||
|
ENV GOPATH /go
|
||||||
|
ENV CGO_ENABLED 0
|
||||||
|
|
||||||
|
# Install curl and minisign
|
||||||
|
RUN apk add -U --no-cache ca-certificates && \
|
||||||
|
apk add -U --no-cache curl && \
|
||||||
|
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||||
|
|
||||||
|
# Download minio binary and signature file
|
||||||
|
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||||
|
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||||
|
chmod +x /go/bin/minio
|
||||||
|
|
||||||
|
# Download mc binary and signature file
|
||||||
|
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||||
|
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||||
|
chmod +x /go/bin/mc
|
||||||
|
|
||||||
|
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||||
|
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||||
|
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||||
|
|
||||||
|
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||||
|
|
||||||
ARG RELEASE
|
ARG RELEASE
|
||||||
|
|
||||||
@@ -19,36 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
|||||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||||
MINIO_CONFIG_ENV_FILE=config.env \
|
MINIO_CONFIG_ENV_FILE=config.env \
|
||||||
PATH=/opt/bin:$PATH
|
MC_CONFIG_DIR=/tmp/.mc
|
||||||
|
|
||||||
|
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||||
|
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||||
|
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||||
|
|
||||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
|
||||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
|
||||||
COPY CREDITS /licenses/CREDITS
|
COPY CREDITS /licenses/CREDITS
|
||||||
COPY LICENSE /licenses/LICENSE
|
COPY LICENSE /licenses/LICENSE
|
||||||
|
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||||
RUN \
|
|
||||||
microdnf clean all && \
|
|
||||||
microdnf update --nodocs && \
|
|
||||||
microdnf install curl ca-certificates shadow-utils util-linux gzip --nodocs && \
|
|
||||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
|
||||||
microdnf install minisign --nodocs && \
|
|
||||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
|
|
||||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
|
|
||||||
gzip /opt/bin/mc && \
|
|
||||||
microdnf clean all && \
|
|
||||||
chmod +x /opt/bin/minio && \
|
|
||||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
|
||||||
chmod +x /usr/bin/verify-minio.sh && \
|
|
||||||
/usr/bin/verify-minio.sh && \
|
|
||||||
microdnf clean all
|
|
||||||
|
|
||||||
EXPOSE 9000
|
EXPOSE 9000
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
|
||||||
|
|
||||||
VOLUME ["/data"]
|
VOLUME ["/data"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||||
CMD ["minio"]
|
CMD ["minio"]
|
||||||
|
|||||||
+26
-25
@@ -1,6 +1,25 @@
|
|||||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
|
FROM golang:1.21-alpine as build
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
ARG RELEASE
|
||||||
|
|
||||||
|
ENV GOPATH /go
|
||||||
|
ENV CGO_ENABLED 0
|
||||||
|
|
||||||
|
# Install curl and minisign
|
||||||
|
RUN apk add -U --no-cache ca-certificates && \
|
||||||
|
apk add -U --no-cache curl && \
|
||||||
|
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||||
|
|
||||||
|
# Download minio binary and signature file
|
||||||
|
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
|
||||||
|
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
|
||||||
|
chmod +x /go/bin/minio
|
||||||
|
|
||||||
|
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||||
|
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||||
|
|
||||||
|
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||||
|
|
||||||
ARG RELEASE
|
ARG RELEASE
|
||||||
|
|
||||||
@@ -18,35 +37,17 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
|||||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||||
MINIO_CONFIG_ENV_FILE=config.env \
|
MINIO_CONFIG_ENV_FILE=config.env
|
||||||
PATH=/opt/bin:$PATH
|
|
||||||
|
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||||
|
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||||
|
|
||||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
|
||||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
|
||||||
COPY CREDITS /licenses/CREDITS
|
COPY CREDITS /licenses/CREDITS
|
||||||
COPY LICENSE /licenses/LICENSE
|
COPY LICENSE /licenses/LICENSE
|
||||||
|
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||||
RUN \
|
|
||||||
microdnf clean all && \
|
|
||||||
microdnf update --nodocs && \
|
|
||||||
microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
|
|
||||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
|
||||||
microdnf install minisign --nodocs && \
|
|
||||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
|
|
||||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
|
|
||||||
microdnf clean all && \
|
|
||||||
chmod +x /opt/bin/minio && \
|
|
||||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
|
||||||
chmod +x /usr/bin/verify-minio.sh && \
|
|
||||||
/usr/bin/verify-minio.sh && \
|
|
||||||
microdnf clean all
|
|
||||||
|
|
||||||
EXPOSE 9000
|
EXPOSE 9000
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
|
||||||
|
|
||||||
VOLUME ["/data"]
|
VOLUME ["/data"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||||
CMD ["minio"]
|
CMD ["minio"]
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
FROM golang:1.21-alpine as build
|
||||||
|
|
||||||
|
ARG TARGETARCH
|
||||||
|
ARG RELEASE
|
||||||
|
|
||||||
|
ENV GOPATH /go
|
||||||
|
ENV CGO_ENABLED 0
|
||||||
|
|
||||||
|
# Install curl and minisign
|
||||||
|
RUN apk add -U --no-cache ca-certificates && \
|
||||||
|
apk add -U --no-cache curl && \
|
||||||
|
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||||
|
|
||||||
|
# Download minio binary and signature file
|
||||||
|
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||||
|
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||||
|
chmod +x /go/bin/minio
|
||||||
|
|
||||||
|
# Download mc binary and signature file
|
||||||
|
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||||
|
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||||
|
chmod +x /go/bin/mc
|
||||||
|
|
||||||
|
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||||
|
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||||
|
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||||
|
|
||||||
|
FROM registry.access.redhat.com/ubi8/ubi-micro:latest
|
||||||
|
|
||||||
|
ARG RELEASE
|
||||||
|
|
||||||
|
LABEL name="MinIO" \
|
||||||
|
vendor="MinIO Inc <dev@min.io>" \
|
||||||
|
maintainer="MinIO Inc <dev@min.io>" \
|
||||||
|
version="${RELEASE}" \
|
||||||
|
release="${RELEASE}" \
|
||||||
|
summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
|
||||||
|
description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
|
||||||
|
|
||||||
|
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||||
|
MINIO_SECRET_KEY_FILE=secret_key \
|
||||||
|
MINIO_ROOT_USER_FILE=access_key \
|
||||||
|
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||||
|
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||||
|
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||||
|
MINIO_CONFIG_ENV_FILE=config.env \
|
||||||
|
MC_CONFIG_DIR=/tmp/.mc
|
||||||
|
|
||||||
|
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||||
|
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||||
|
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||||
|
|
||||||
|
COPY CREDITS /licenses/CREDITS
|
||||||
|
COPY LICENSE /licenses/LICENSE
|
||||||
|
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||||
|
|
||||||
|
EXPOSE 9000
|
||||||
|
VOLUME ["/data"]
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||||
|
CMD ["minio"]
|
||||||
@@ -6,7 +6,10 @@ GOARCH := $(shell go env GOARCH)
|
|||||||
GOOS := $(shell go env GOOS)
|
GOOS := $(shell go env GOOS)
|
||||||
|
|
||||||
VERSION ?= $(shell git describe --tags)
|
VERSION ?= $(shell git describe --tags)
|
||||||
TAG ?= "minio/minio:$(VERSION)"
|
TAG ?= "quay.io/minio/minio:$(VERSION)"
|
||||||
|
|
||||||
|
GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
|
||||||
|
GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
|
||||||
|
|
||||||
all: build
|
all: build
|
||||||
|
|
||||||
@@ -19,38 +22,51 @@ help: ## print this help
|
|||||||
|
|
||||||
getdeps: ## fetch necessary dependencies
|
getdeps: ## fetch necessary dependencies
|
||||||
@mkdir -p ${GOPATH}/bin
|
@mkdir -p ${GOPATH}/bin
|
||||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
|
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
|
||||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
|
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
|
||||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||||
@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
|
|
||||||
|
|
||||||
crosscompile: ## cross compile minio
|
crosscompile: ## cross compile minio
|
||||||
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
||||||
|
|
||||||
verifiers: getdeps lint check-gen
|
verifiers: lint check-gen
|
||||||
|
|
||||||
check-gen: ## check for updated autogenerated files
|
check-gen: ## check for updated autogenerated files
|
||||||
@go generate ./... >/dev/null
|
@go generate ./... >/dev/null
|
||||||
@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
|
@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
|
||||||
|
|
||||||
lint: ## runs golangci-lint suite of linters
|
lint: getdeps ## runs golangci-lint suite of linters
|
||||||
@echo "Running $@ check"
|
@echo "Running $@ check"
|
||||||
@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
|
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
|
||||||
@${GOPATH}/bin/staticcheck --tests=false ./...
|
|
||||||
|
lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
|
||||||
|
@echo "Running $@ check"
|
||||||
|
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
|
||||||
|
|
||||||
check: test
|
check: test
|
||||||
test: verifiers build ## builds minio, runs linters, tests
|
test: verifiers build build-debugging ## builds minio, runs linters, tests
|
||||||
@echo "Running unit tests"
|
@echo "Running unit tests"
|
||||||
@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
|
@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue ./...
|
||||||
|
|
||||||
test-decom: install
|
test-root-disable: install-race
|
||||||
|
@echo "Running minio root lockdown tests"
|
||||||
|
@env bash $(PWD)/buildscripts/disable-root.sh
|
||||||
|
|
||||||
|
test-decom: install-race
|
||||||
@echo "Running minio decom tests"
|
@echo "Running minio decom tests"
|
||||||
@env bash $(PWD)/docs/distributed/decom.sh
|
@env bash $(PWD)/docs/distributed/decom.sh
|
||||||
@env bash $(PWD)/docs/distributed/decom-encrypted.sh
|
@env bash $(PWD)/docs/distributed/decom-encrypted.sh
|
||||||
@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
|
@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
|
||||||
@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
|
@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
|
||||||
|
|
||||||
test-upgrade: build
|
test-versioning: install-race
|
||||||
|
@echo "Running minio versioning tests"
|
||||||
|
@env bash $(PWD)/docs/bucket/versioning/versioning-tests.sh
|
||||||
|
|
||||||
|
test-configfile: install-race
|
||||||
|
@env bash $(PWD)/docs/distributed/distributed-from-config-file.sh
|
||||||
|
|
||||||
|
test-upgrade: install-race
|
||||||
@echo "Running minio upgrade tests"
|
@echo "Running minio upgrade tests"
|
||||||
@(env bash $(PWD)/buildscripts/minio-upgrade.sh)
|
@(env bash $(PWD)/buildscripts/minio-upgrade.sh)
|
||||||
|
|
||||||
@@ -64,6 +80,9 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
|
|||||||
@echo "Running tests for IAM (external IDP, etcd backends) with -race"
|
@echo "Running tests for IAM (external IDP, etcd backends) with -race"
|
||||||
@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
|
@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
|
||||||
|
|
||||||
|
test-sio-error:
|
||||||
|
@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
|
||||||
|
|
||||||
test-replication-2site:
|
test-replication-2site:
|
||||||
@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
|
@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
|
||||||
|
|
||||||
@@ -73,18 +92,18 @@ test-replication-3site:
|
|||||||
test-delete-replication:
|
test-delete-replication:
|
||||||
@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
|
@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
|
||||||
|
|
||||||
test-replication: install test-replication-2site test-replication-3site test-delete-replication ## verify multi site replication
|
test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
|
||||||
@echo "Running tests for replicating three sites"
|
@echo "Running tests for replicating three sites"
|
||||||
|
|
||||||
test-site-replication-ldap: install ## verify automatic site replication
|
test-site-replication-ldap: install-race ## verify automatic site replication
|
||||||
@echo "Running tests for automatic site replication of IAM (with LDAP)"
|
@echo "Running tests for automatic site replication of IAM (with LDAP)"
|
||||||
@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
|
@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
|
||||||
|
|
||||||
test-site-replication-oidc: install ## verify automatic site replication
|
test-site-replication-oidc: install-race ## verify automatic site replication
|
||||||
@echo "Running tests for automatic site replication of IAM (with OIDC)"
|
@echo "Running tests for automatic site replication of IAM (with OIDC)"
|
||||||
@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
|
@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
|
||||||
|
|
||||||
test-site-replication-minio: install ## verify automatic site replication
|
test-site-replication-minio: install-race ## verify automatic site replication
|
||||||
@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
|
@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
|
||||||
@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
|
@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
|
||||||
|
|
||||||
@@ -97,7 +116,6 @@ verify-healing: ## verify healing and replacing disks with minio binary
|
|||||||
@echo "Verify healing build with race"
|
@echo "Verify healing build with race"
|
||||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||||
@(env bash $(PWD)/buildscripts/verify-healing.sh)
|
@(env bash $(PWD)/buildscripts/verify-healing.sh)
|
||||||
@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
|
|
||||||
@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
|
@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
|
||||||
|
|
||||||
verify-healing-with-root-disks: ## verify healing root disks
|
verify-healing-with-root-disks: ## verify healing root disks
|
||||||
@@ -115,6 +133,9 @@ verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
|
|||||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||||
@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
|
@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
|
||||||
|
|
||||||
|
build-debugging:
|
||||||
|
@(env bash $(PWD)/docs/debugging/build.sh)
|
||||||
|
|
||||||
build: checks ## builds minio to $(PWD)
|
build: checks ## builds minio to $(PWD)
|
||||||
@echo "Building minio binary to './minio'"
|
@echo "Building minio binary to './minio'"
|
||||||
@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||||
@@ -125,14 +146,23 @@ hotfix-vars:
|
|||||||
$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
|
$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
|
||||||
$(eval TAG := "minio/minio:$(VERSION)")
|
$(eval TAG := "minio/minio:$(VERSION)")
|
||||||
|
|
||||||
hotfix: hotfix-vars install ## builds minio binary with hotfix tags
|
hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
|
||||||
@mv -f ./minio ./minio.$(VERSION)
|
@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.0/pkger_2.2.0_linux_amd64.deb
|
||||||
@minisign -qQSm ./minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
|
@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.1/linux-systemd/distributed/minio.service
|
||||||
@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum
|
@sudo apt install ./pkger_2.2.0_linux_amd64.deb --yes
|
||||||
|
@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
|
||||||
|
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
|
||||||
|
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)
|
||||||
|
@minisign -qQSm minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
|
||||||
|
@sha256sum < minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION).sha256sum
|
||||||
|
@cp -af minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)* minio-release/$(GOOS)-$(GOARCH)/archive/
|
||||||
|
@pkger -r $(VERSION) --ignore
|
||||||
|
|
||||||
hotfix-push: hotfix
|
hotfix-push: hotfix
|
||||||
@scp -q -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
|
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||||
@scp -q -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
|
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||||
|
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||||
|
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||||
@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"
|
@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"
|
||||||
|
|
||||||
docker-hotfix-push: docker-hotfix
|
docker-hotfix-push: docker-hotfix
|
||||||
@@ -146,6 +176,12 @@ docker: build ## builds minio docker container
|
|||||||
@echo "Building minio docker image '$(TAG)'"
|
@echo "Building minio docker image '$(TAG)'"
|
||||||
@docker build -q --no-cache -t $(TAG) . -f Dockerfile
|
@docker build -q --no-cache -t $(TAG) . -f Dockerfile
|
||||||
|
|
||||||
|
install-race: checks ## builds minio to $(PWD)
|
||||||
|
@echo "Building minio binary to './minio'"
|
||||||
|
@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||||
|
@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
|
||||||
|
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
|
||||||
|
|
||||||
install: build ## builds minio and installs it to $GOPATH/bin.
|
install: build ## builds minio and installs it to $GOPATH/bin.
|
||||||
@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
|
@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
|
||||||
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
|
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
|
||||||
@@ -161,3 +197,6 @@ clean: ## cleanup all generated assets
|
|||||||
@rm -rvf build
|
@rm -rvf build
|
||||||
@rm -rvf release
|
@rm -rvf release
|
||||||
@rm -rvf .verify*
|
@rm -rvf .verify*
|
||||||
|
@rm -rvf minio-release
|
||||||
|
@rm -rvf minio.RELEASE*.hotfix.*
|
||||||
|
@rm -rvf pkger_*.deb
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
MinIO Project, (C) 2015-2021 MinIO, Inc.
|
MinIO Project, (C) 2015-2023 MinIO, Inc.
|
||||||
|
|
||||||
This product includes software developed at MinIO, Inc.
|
This product includes software developed at MinIO, Inc.
|
||||||
(https://min.io/).
|
(https://min.io/).
|
||||||
|
|||||||
@@ -86,8 +86,6 @@ chmod +x minio
|
|||||||
./minio server /data
|
./minio server /data
|
||||||
```
|
```
|
||||||
|
|
||||||
Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
|
|
||||||
|
|
||||||
The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
|
The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
|
||||||
|
|
||||||
| Architecture | URL |
|
| Architecture | URL |
|
||||||
@@ -202,7 +200,7 @@ service iptables restart
|
|||||||
|
|
||||||
MinIO Server comes with an embedded web based object browser. Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.
|
MinIO Server comes with an embedded web based object browser. Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.
|
||||||
|
|
||||||
> NOTE: MinIO runs console on random port by default if you wish choose a specific port use `--console-address` to pick a specific interface and port.
|
> NOTE: MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
|
||||||
|
|
||||||
### Things to consider
|
### Things to consider
|
||||||
|
|
||||||
@@ -243,7 +241,7 @@ mc admin update <minio alias, e.g., myminio>
|
|||||||
### Upgrade Checklist
|
### Upgrade Checklist
|
||||||
|
|
||||||
- Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
|
- Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
|
||||||
- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest releases upon every releases. Some releases may not be relevant to your setup, avoid upgrading production environments unnecessarily.
|
- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest release upon every release. Some release may not be relevant to your setup, avoid upgrading production environments unnecessarily.
|
||||||
- If you plan to use `mc admin update`, MinIO process must have write access to the parent directory where the binary is present on the host system.
|
- If you plan to use `mc admin update`, MinIO process must have write access to the parent directory where the binary is present on the host system.
|
||||||
- `mc admin update` is not supported and should be avoided in kubernetes/container environments, please upgrade containers by upgrading relevant container images.
|
- `mc admin update` is not supported and should be avoided in kubernetes/container environments, please upgrade containers by upgrading relevant container images.
|
||||||
- **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
|
- **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
|
||||||
@@ -261,6 +259,6 @@ Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/ma
|
|||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
|
- MinIO source is licensed under the [GNU AGPLv3](https://github.com/minio/minio/blob/master/LICENSE).
|
||||||
- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
|
- MinIO [documentation](https://github.com/minio/minio/tree/master/docs) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
|
||||||
- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)
|
- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)
|
||||||
|
|||||||
+15
-14
@@ -12,7 +12,7 @@ _init() {
|
|||||||
KNAME=$(uname -s)
|
KNAME=$(uname -s)
|
||||||
ARCH=$(uname -m)
|
ARCH=$(uname -m)
|
||||||
case "${KNAME}" in
|
case "${KNAME}" in
|
||||||
SunOS )
|
SunOS)
|
||||||
ARCH=$(isainfo -k)
|
ARCH=$(isainfo -k)
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
@@ -30,20 +30,19 @@ _init() {
|
|||||||
readlink() {
|
readlink() {
|
||||||
TARGET_FILE=$1
|
TARGET_FILE=$1
|
||||||
|
|
||||||
cd `dirname $TARGET_FILE`
|
cd $(dirname $TARGET_FILE)
|
||||||
TARGET_FILE=`basename $TARGET_FILE`
|
TARGET_FILE=$(basename $TARGET_FILE)
|
||||||
|
|
||||||
# Iterate down a (possible) chain of symlinks
|
# Iterate down a (possible) chain of symlinks
|
||||||
while [ -L "$TARGET_FILE" ]
|
while [ -L "$TARGET_FILE" ]; do
|
||||||
do
|
|
||||||
TARGET_FILE=$(env readlink $TARGET_FILE)
|
TARGET_FILE=$(env readlink $TARGET_FILE)
|
||||||
cd `dirname $TARGET_FILE`
|
cd $(dirname $TARGET_FILE)
|
||||||
TARGET_FILE=`basename $TARGET_FILE`
|
TARGET_FILE=$(basename $TARGET_FILE)
|
||||||
done
|
done
|
||||||
|
|
||||||
# Compute the canonicalized name by finding the physical path
|
# Compute the canonicalized name by finding the physical path
|
||||||
# for the directory we're in and appending the target file.
|
# for the directory we're in and appending the target file.
|
||||||
PHYS_DIR=`pwd -P`
|
PHYS_DIR=$(pwd -P)
|
||||||
RESULT=$PHYS_DIR/$TARGET_FILE
|
RESULT=$PHYS_DIR/$TARGET_FILE
|
||||||
echo $RESULT
|
echo $RESULT
|
||||||
}
|
}
|
||||||
@@ -59,8 +58,8 @@ readlink() {
|
|||||||
## }
|
## }
|
||||||
##
|
##
|
||||||
check_minimum_version() {
|
check_minimum_version() {
|
||||||
IFS='.' read -r -a varray1 <<< "$1"
|
IFS='.' read -r -a varray1 <<<"$1"
|
||||||
IFS='.' read -r -a varray2 <<< "$2"
|
IFS='.' read -r -a varray2 <<<"$2"
|
||||||
|
|
||||||
for i in "${!varray1[@]}"; do
|
for i in "${!varray1[@]}"; do
|
||||||
if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then
|
if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then
|
||||||
@@ -75,21 +74,22 @@ check_minimum_version() {
|
|||||||
|
|
||||||
assert_is_supported_arch() {
|
assert_is_supported_arch() {
|
||||||
case "${ARCH}" in
|
case "${ARCH}" in
|
||||||
x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x )
|
x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64)
|
||||||
return
|
return
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x]"
|
echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64]"
|
||||||
exit 1
|
exit 1
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
assert_is_supported_os() {
|
assert_is_supported_os() {
|
||||||
case "${KNAME}" in
|
case "${KNAME}" in
|
||||||
Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS )
|
Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS)
|
||||||
return
|
return
|
||||||
;;
|
;;
|
||||||
Darwin )
|
Darwin)
|
||||||
osx_host_version=$(env sw_vers -productVersion)
|
osx_host_version=$(env sw_vers -productVersion)
|
||||||
if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then
|
if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then
|
||||||
echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}"
|
echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}"
|
||||||
@@ -100,6 +100,7 @@ assert_is_supported_os() {
|
|||||||
*)
|
*)
|
||||||
echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]"
|
echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]"
|
||||||
exit 1
|
exit 1
|
||||||
|
;;
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Executable
+121
@@ -0,0 +1,121 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -x
|
||||||
|
|
||||||
|
export MINIO_CI_CD=1
|
||||||
|
killall -9 minio
|
||||||
|
|
||||||
|
rm -rf ${HOME}/tmp/dist
|
||||||
|
|
||||||
|
scheme="http"
|
||||||
|
nr_servers=4
|
||||||
|
|
||||||
|
addr="localhost"
|
||||||
|
args=""
|
||||||
|
for ((i = 0; i < $((nr_servers)); i++)); do
|
||||||
|
args="$args $scheme://$addr:$((9100 + i))/${HOME}/tmp/dist/path1/$i"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo $args
|
||||||
|
|
||||||
|
for ((i = 0; i < $((nr_servers)); i++)); do
|
||||||
|
(minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) &
|
||||||
|
done
|
||||||
|
|
||||||
|
sleep 10s
|
||||||
|
|
||||||
|
if [ ! -f ./mc ]; then
|
||||||
|
wget --quiet -O ./mc https://dl.minio.io/client/mc/release/linux-amd64/./mc &&
|
||||||
|
chmod +x mc
|
||||||
|
fi
|
||||||
|
|
||||||
|
set +e
|
||||||
|
|
||||||
|
export MC_HOST_minioadm=http://minioadmin:minioadmin@localhost:9100/
|
||||||
|
|
||||||
|
./mc ls minioadm/
|
||||||
|
|
||||||
|
./mc admin config set minioadm/ api root_access=off
|
||||||
|
|
||||||
|
sleep 3s # let things settle a little
|
||||||
|
|
||||||
|
./mc ls minioadm/
|
||||||
|
if [ $? -eq 0 ]; then
|
||||||
|
echo "listing succeeded, 'minioadmin' was not disabled"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
killall -9 minio
|
||||||
|
|
||||||
|
export MINIO_API_ROOT_ACCESS=on
|
||||||
|
for ((i = 0; i < $((nr_servers)); i++)); do
|
||||||
|
(minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) &
|
||||||
|
done
|
||||||
|
|
||||||
|
set +e
|
||||||
|
|
||||||
|
sleep 10
|
||||||
|
|
||||||
|
./mc ls minioadm/
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "listing failed, 'minioadmin' should be enabled"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
killall -9 minio
|
||||||
|
|
||||||
|
rm -rf /tmp/multisitea/
|
||||||
|
rm -rf /tmp/multisiteb/
|
||||||
|
|
||||||
|
echo "Setup site-replication and then disable root credentials"
|
||||||
|
|
||||||
|
minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
|
||||||
|
minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
|
||||||
|
|
||||||
|
minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
|
||||||
|
minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
|
||||||
|
|
||||||
|
sleep 20s
|
||||||
|
|
||||||
|
export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001
|
||||||
|
export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004
|
||||||
|
|
||||||
|
./mc admin replicate add sitea siteb
|
||||||
|
|
||||||
|
./mc admin user add sitea foobar foo12345
|
||||||
|
|
||||||
|
./mc admin policy attach sitea/ consoleAdmin --user=foobar
|
||||||
|
|
||||||
|
./mc admin user info siteb foobar
|
||||||
|
|
||||||
|
killall -9 minio
|
||||||
|
|
||||||
|
echo "turning off root access, however site replication must continue"
|
||||||
|
export MINIO_API_ROOT_ACCESS=off
|
||||||
|
|
||||||
|
minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
|
||||||
|
minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
|
||||||
|
|
||||||
|
minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
|
||||||
|
minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
|
||||||
|
"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
|
||||||
|
|
||||||
|
sleep 20s
|
||||||
|
|
||||||
|
export MC_HOST_sitea=http://foobar:foo12345@127.0.0.1:9001
|
||||||
|
export MC_HOST_siteb=http://foobar:foo12345@127.0.0.1:9004
|
||||||
|
|
||||||
|
./mc admin user add sitea foobar-admin foo12345
|
||||||
|
|
||||||
|
sleep 2s
|
||||||
|
|
||||||
|
./mc admin user info siteb foobar-admin
|
||||||
@@ -6,7 +6,7 @@ set -x
|
|||||||
|
|
||||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||||
|
|
||||||
if [ ! -x "$PWD/minio" ]; then
|
if [ ! -x "$PWD/minio" ]; then
|
||||||
echo "minio executable binary not found in current directory"
|
echo "minio executable binary not found in current directory"
|
||||||
@@ -43,7 +43,7 @@ function start_minio_4drive() {
|
|||||||
purge "${MC_BUILD_DIR}"
|
purge "${MC_BUILD_DIR}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 &
|
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" >"${WORK_DIR}/server1.log" 2>&1 &
|
||||||
pid=$!
|
pid=$!
|
||||||
disown $pid
|
disown $pid
|
||||||
sleep 5
|
sleep 5
|
||||||
@@ -82,12 +82,11 @@ function main() {
|
|||||||
start_minio_4drive ${start_port}
|
start_minio_4drive ${start_port}
|
||||||
}
|
}
|
||||||
|
|
||||||
function purge()
|
function purge() {
|
||||||
{
|
|
||||||
rm -rf "$1"
|
rm -rf "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
( main "$@" )
|
(main "$@")
|
||||||
rv=$?
|
rv=$?
|
||||||
purge "$WORK_DIR"
|
purge "$WORK_DIR"
|
||||||
exit "$rv"
|
exit "$rv"
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
|||||||
@@ -12,14 +12,14 @@ cleanup() {
|
|||||||
|
|
||||||
verify_checksum_after_heal() {
|
verify_checksum_after_heal() {
|
||||||
local sum1
|
local sum1
|
||||||
sum1=$(curl -s "$2" | sha256sum);
|
sum1=$(curl -s "$2" | sha256sum)
|
||||||
mc admin heal --json -r "$1" >/dev/null; # test after healing
|
mc admin heal --json -r "$1" >/dev/null # test after healing
|
||||||
local sum1_heal
|
local sum1_heal
|
||||||
sum1_heal=$(curl -s "$2" | sha256sum);
|
sum1_heal=$(curl -s "$2" | sha256sum)
|
||||||
|
|
||||||
if [ "${sum1_heal}" != "${sum1}" ]; then
|
if [ "${sum1_heal}" != "${sum1}" ]; then
|
||||||
echo "mismatch expected ${sum1_heal}, got ${sum1}"
|
echo "mismatch expected ${sum1_heal}, got ${sum1}"
|
||||||
exit 1;
|
exit 1
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,7 +31,7 @@ verify_checksum_mc() {
|
|||||||
|
|
||||||
if [ "${expected}" != "${got}" ]; then
|
if [ "${expected}" != "${got}" ]; then
|
||||||
echo "mismatch - expected ${expected}, got ${got}"
|
echo "mismatch - expected ${expected}, got ${got}"
|
||||||
exit 1;
|
exit 1
|
||||||
fi
|
fi
|
||||||
echo "matches - ${expected}, got ${got}"
|
echo "matches - ${expected}, got ${got}"
|
||||||
}
|
}
|
||||||
@@ -89,4 +89,4 @@ main() {
|
|||||||
cleanup
|
cleanup
|
||||||
}
|
}
|
||||||
|
|
||||||
( __init__ "$@" && main "$@" )
|
(__init__ "$@" && main "$@")
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ set -e
|
|||||||
export GORACE="history_size=7"
|
export GORACE="history_size=7"
|
||||||
export MINIO_API_REQUESTS_MAX=10000
|
export MINIO_API_REQUESTS_MAX=10000
|
||||||
|
|
||||||
## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
|
for d in $(go list ./...); do
|
||||||
for d in $(go list ./... | grep -v dsync); do
|
|
||||||
CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
|
CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
|
||||||
done
|
done
|
||||||
|
|||||||
@@ -3,10 +3,11 @@
|
|||||||
set -E
|
set -E
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
set -x
|
set -x
|
||||||
|
set -e
|
||||||
|
|
||||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||||
|
|
||||||
if [ ! -x "$PWD/minio" ]; then
|
if [ ! -x "$PWD/minio" ]; then
|
||||||
echo "minio executable binary not found in current directory"
|
echo "minio executable binary not found in current directory"
|
||||||
@@ -36,7 +37,7 @@ function start_minio_5drive() {
|
|||||||
|
|
||||||
"${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/"
|
"${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/"
|
||||||
|
|
||||||
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
|
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" >"${WORK_DIR}/server1.log" 2>&1 &
|
||||||
pid=$!
|
pid=$!
|
||||||
disown $pid
|
disown $pid
|
||||||
sleep 5
|
sleep 5
|
||||||
@@ -61,12 +62,11 @@ function main() {
|
|||||||
start_minio_5drive ${start_port}
|
start_minio_5drive ${start_port}
|
||||||
}
|
}
|
||||||
|
|
||||||
function purge()
|
function purge() {
|
||||||
{
|
|
||||||
rm -rf "$1"
|
rm -rf "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
( main "$@" )
|
(main "$@")
|
||||||
rv=$?
|
rv=$?
|
||||||
purge "$WORK_DIR"
|
purge "$WORK_DIR"
|
||||||
exit "$rv"
|
exit "$rv"
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ set -x
|
|||||||
|
|
||||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||||
MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
|
MINIO_OLD=("$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||||
|
|
||||||
if [ ! -x "$PWD/minio" ]; then
|
if [ ! -x "$PWD/minio" ]; then
|
||||||
echo "minio executable binary not found in current directory"
|
echo "minio executable binary not found in current directory"
|
||||||
@@ -42,7 +42,7 @@ function verify_rewrite() {
|
|||||||
# remove mc source.
|
# remove mc source.
|
||||||
purge "${MC_BUILD_DIR}"
|
purge "${MC_BUILD_DIR}"
|
||||||
|
|
||||||
"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
|
"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
|
||||||
pid=$!
|
pid=$!
|
||||||
disown $pid
|
disown $pid
|
||||||
sleep 10
|
sleep 10
|
||||||
@@ -74,7 +74,7 @@ function verify_rewrite() {
|
|||||||
kill ${pid}
|
kill ${pid}
|
||||||
sleep 3
|
sleep 3
|
||||||
|
|
||||||
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
|
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
|
||||||
pid=$!
|
pid=$!
|
||||||
disown $pid
|
disown $pid
|
||||||
sleep 10
|
sleep 10
|
||||||
@@ -87,8 +87,8 @@ function verify_rewrite() {
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
go build ./docs/debugging/s3-check-md5/
|
go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
|
||||||
if ! ./s3-check-md5 \
|
if ! s3-check-md5 \
|
||||||
-debug \
|
-debug \
|
||||||
-versions \
|
-versions \
|
||||||
-access-key minio \
|
-access-key minio \
|
||||||
@@ -98,7 +98,10 @@ function verify_rewrite() {
|
|||||||
cat "${WORK_DIR}/server1.log"
|
cat "${WORK_DIR}/server1.log"
|
||||||
echo "FAILED"
|
echo "FAILED"
|
||||||
mkdir -p inspects
|
mkdir -p inspects
|
||||||
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
|
(
|
||||||
|
cd inspects
|
||||||
|
"${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**
|
||||||
|
)
|
||||||
|
|
||||||
"${WORK_DIR}/mc" mb play/inspects
|
"${WORK_DIR}/mc" mb play/inspects
|
||||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
"${WORK_DIR}/mc" mirror inspects play/inspects
|
||||||
@@ -110,7 +113,7 @@ function verify_rewrite() {
|
|||||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
|
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
||||||
if ! ./s3-check-md5 \
|
if ! s3-check-md5 \
|
||||||
-debug \
|
-debug \
|
||||||
-versions \
|
-versions \
|
||||||
-access-key minio \
|
-access-key minio \
|
||||||
@@ -120,7 +123,10 @@ function verify_rewrite() {
|
|||||||
cat "${WORK_DIR}/server1.log"
|
cat "${WORK_DIR}/server1.log"
|
||||||
echo "FAILED"
|
echo "FAILED"
|
||||||
mkdir -p inspects
|
mkdir -p inspects
|
||||||
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
|
(
|
||||||
|
cd inspects
|
||||||
|
"${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**
|
||||||
|
)
|
||||||
|
|
||||||
"${WORK_DIR}/mc" mb play/inspects
|
"${WORK_DIR}/mc" mb play/inspects
|
||||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
"${WORK_DIR}/mc" mirror inspects play/inspects
|
||||||
@@ -140,12 +146,11 @@ function main() {
|
|||||||
verify_rewrite ${start_port}
|
verify_rewrite ${start_port}
|
||||||
}
|
}
|
||||||
|
|
||||||
function purge()
|
function purge() {
|
||||||
{
|
|
||||||
rm -rf "$1"
|
rm -rf "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
( main "$@" )
|
(main "$@")
|
||||||
rv=$?
|
rv=$?
|
||||||
purge "$WORK_DIR"
|
purge "$WORK_DIR"
|
||||||
exit "$rv"
|
exit "$rv"
|
||||||
|
|||||||
@@ -1,172 +0,0 @@
|
|||||||
#!/bin/bash -e
|
|
||||||
#
|
|
||||||
|
|
||||||
set -E
|
|
||||||
set -o pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
if [ ! -x "$PWD/minio" ]; then
|
|
||||||
echo "minio executable binary not found in current directory"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
|
||||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
|
||||||
MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server )
|
|
||||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
|
||||||
|
|
||||||
function download_old_release() {
|
|
||||||
if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
|
|
||||||
curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
|
|
||||||
chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function start_minio_16drive() {
|
|
||||||
start_port=$1
|
|
||||||
|
|
||||||
export MINIO_ROOT_USER=minio
|
|
||||||
export MINIO_ROOT_PASSWORD=minio123
|
|
||||||
export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
|
|
||||||
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
|
|
||||||
export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
|
|
||||||
export MINIO_CI_CD=1
|
|
||||||
|
|
||||||
MC_BUILD_DIR="mc-$RANDOM"
|
|
||||||
if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
|
|
||||||
echo "failed to download https://github.com/minio/mc"
|
|
||||||
purge "${MC_BUILD_DIR}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
|
|
||||||
|
|
||||||
# remove mc source.
|
|
||||||
purge "${MC_BUILD_DIR}"
|
|
||||||
|
|
||||||
"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
|
|
||||||
pid=$!
|
|
||||||
disown $pid
|
|
||||||
sleep 30
|
|
||||||
|
|
||||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
|
||||||
echo "server1 log:"
|
|
||||||
cat "${WORK_DIR}/server1.log"
|
|
||||||
echo "FAILED"
|
|
||||||
purge "$WORK_DIR"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
|
|
||||||
"${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
|
|
||||||
"${WORK_DIR}/mc" cp \
|
|
||||||
"${WORK_DIR}/unaligned" \
|
|
||||||
minio/healing-shard-bucket/unaligned \
|
|
||||||
--disable-multipart --quiet
|
|
||||||
|
|
||||||
## "unaligned" object name gets consistently distributed
|
|
||||||
## to disks in following distribution order
|
|
||||||
##
|
|
||||||
## NOTE: if you change the name make sure to change the
|
|
||||||
## distribution order present here
|
|
||||||
##
|
|
||||||
## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
|
|
||||||
|
|
||||||
## make sure to remove the "last" data shard
|
|
||||||
rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
|
|
||||||
sleep 10
|
|
||||||
## Heal the shard
|
|
||||||
"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
|
|
||||||
## then remove any other data shard let's pick first disk
|
|
||||||
## - 1st data shard.
|
|
||||||
rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
|
|
||||||
sleep 10
|
|
||||||
|
|
||||||
go build ./docs/debugging/s3-check-md5/
|
|
||||||
if ! ./s3-check-md5 \
|
|
||||||
-debug \
|
|
||||||
-access-key minio \
|
|
||||||
-secret-key minio123 \
|
|
||||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
|
|
||||||
echo "server1 log:"
|
|
||||||
cat "${WORK_DIR}/server1.log"
|
|
||||||
echo "FAILED"
|
|
||||||
purge "$WORK_DIR"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
pkill minio
|
|
||||||
sleep 3
|
|
||||||
|
|
||||||
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
|
|
||||||
pid=$!
|
|
||||||
disown $pid
|
|
||||||
sleep 30
|
|
||||||
|
|
||||||
if ! ps -p ${pid} 1>&2 >/dev/null; then
|
|
||||||
echo "server1 log:"
|
|
||||||
cat "${WORK_DIR}/server1.log"
|
|
||||||
echo "FAILED"
|
|
||||||
purge "$WORK_DIR"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! ./s3-check-md5 \
|
|
||||||
-debug \
|
|
||||||
-access-key minio \
|
|
||||||
-secret-key minio123 \
|
|
||||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
|
||||||
echo "server1 log:"
|
|
||||||
cat "${WORK_DIR}/server1.log"
|
|
||||||
echo "FAILED"
|
|
||||||
mkdir -p inspects
|
|
||||||
(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
|
|
||||||
|
|
||||||
"${WORK_DIR}/mc" mb play/inspects
|
|
||||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
|
||||||
|
|
||||||
purge "$WORK_DIR"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
|
|
||||||
|
|
||||||
if ! ./s3-check-md5 \
|
|
||||||
-debug \
|
|
||||||
-access-key minio \
|
|
||||||
-secret-key minio123 \
|
|
||||||
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
|
|
||||||
echo "server1 log:"
|
|
||||||
cat "${WORK_DIR}/server1.log"
|
|
||||||
echo "FAILED"
|
|
||||||
mkdir -p inspects
|
|
||||||
(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
|
|
||||||
|
|
||||||
"${WORK_DIR}/mc" mb play/inspects
|
|
||||||
"${WORK_DIR}/mc" mirror inspects play/inspects
|
|
||||||
|
|
||||||
purge "$WORK_DIR"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
pkill minio
|
|
||||||
sleep 3
|
|
||||||
}
|
|
||||||
|
|
||||||
function main() {
|
|
||||||
download_old_release
|
|
||||||
|
|
||||||
start_port=$(shuf -i 10000-65000 -n 1)
|
|
||||||
|
|
||||||
start_minio_16drive ${start_port}
|
|
||||||
}
|
|
||||||
|
|
||||||
function purge()
|
|
||||||
{
|
|
||||||
rm -rf "$1"
|
|
||||||
}
|
|
||||||
|
|
||||||
( main "$@" )
|
|
||||||
rv=$?
|
|
||||||
purge "$WORK_DIR"
|
|
||||||
exit "$rv"
|
|
||||||
@@ -9,11 +9,6 @@ x-minio-common: &minio-common
|
|||||||
expose:
|
expose:
|
||||||
- "9000"
|
- "9000"
|
||||||
- "9001"
|
- "9001"
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 20s
|
|
||||||
retries: 3
|
|
||||||
|
|
||||||
# starts 4 docker containers running minio server instances.
|
# starts 4 docker containers running minio server instances.
|
||||||
# using nginx reverse proxy, load balancing, you can access
|
# using nginx reverse proxy, load balancing, you can access
|
||||||
|
|||||||
@@ -25,70 +25,63 @@ export ENABLE_ADMIN=1
|
|||||||
export MINIO_CI_CD=1
|
export MINIO_CI_CD=1
|
||||||
|
|
||||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
|
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR")
|
||||||
|
|
||||||
FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB"
|
FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB"
|
||||||
FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB"
|
FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB"
|
||||||
|
|
||||||
FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
|
FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
|
||||||
|
|
||||||
function start_minio_fs()
|
function start_minio_fs() {
|
||||||
{
|
|
||||||
export MINIO_ROOT_USER=$ACCESS_KEY
|
export MINIO_ROOT_USER=$ACCESS_KEY
|
||||||
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
||||||
"${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
|
"${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
|
||||||
sleep 10
|
sleep 10
|
||||||
}
|
}
|
||||||
|
|
||||||
function start_minio_erasure()
|
function start_minio_erasure() {
|
||||||
{
|
|
||||||
"${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
|
"${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
|
||||||
sleep 15
|
sleep 15
|
||||||
}
|
}
|
||||||
|
|
||||||
function start_minio_erasure_sets()
|
function start_minio_erasure_sets() {
|
||||||
{
|
|
||||||
export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
|
export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
|
||||||
"${MINIO[@]}" server > "$WORK_DIR/erasure-minio-sets.log" 2>&1 &
|
"${MINIO[@]}" server >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
|
||||||
sleep 15
|
sleep 15
|
||||||
}
|
}
|
||||||
|
|
||||||
function start_minio_pool_erasure_sets()
|
function start_minio_pool_erasure_sets() {
|
||||||
{
|
|
||||||
export MINIO_ROOT_USER=$ACCESS_KEY
|
export MINIO_ROOT_USER=$ACCESS_KEY
|
||||||
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
||||||
export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
|
export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
|
||||||
"${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
|
"${MINIO[@]}" server --address ":9000" >"$WORK_DIR/pool-minio-9000.log" 2>&1 &
|
||||||
"${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
|
"${MINIO[@]}" server --address ":9001" >"$WORK_DIR/pool-minio-9001.log" 2>&1 &
|
||||||
|
|
||||||
sleep 40
|
sleep 40
|
||||||
}
|
}
|
||||||
|
|
||||||
function start_minio_pool_erasure_sets_ipv6()
|
function start_minio_pool_erasure_sets_ipv6() {
|
||||||
{
|
|
||||||
export MINIO_ROOT_USER=$ACCESS_KEY
|
export MINIO_ROOT_USER=$ACCESS_KEY
|
||||||
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
||||||
export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
|
export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
|
||||||
"${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
|
"${MINIO[@]}" server --address="[::1]:9000" >"$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
|
||||||
"${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
|
"${MINIO[@]}" server --address="[::1]:9001" >"$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
|
||||||
|
|
||||||
sleep 40
|
sleep 40
|
||||||
}
|
}
|
||||||
|
|
||||||
function start_minio_dist_erasure()
|
function start_minio_dist_erasure() {
|
||||||
{
|
|
||||||
export MINIO_ROOT_USER=$ACCESS_KEY
|
export MINIO_ROOT_USER=$ACCESS_KEY
|
||||||
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
||||||
export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
|
export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
|
||||||
for i in $(seq 0 3); do
|
for i in $(seq 0 3); do
|
||||||
"${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
|
"${MINIO[@]}" server --address ":900${i}" >"$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
|
||||||
done
|
done
|
||||||
|
|
||||||
sleep 40
|
sleep 40
|
||||||
}
|
}
|
||||||
|
|
||||||
function run_test_fs()
|
function run_test_fs() {
|
||||||
{
|
|
||||||
start_minio_fs
|
start_minio_fs
|
||||||
|
|
||||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||||
@@ -105,8 +98,7 @@ function run_test_fs()
|
|||||||
return "$rv"
|
return "$rv"
|
||||||
}
|
}
|
||||||
|
|
||||||
function run_test_erasure_sets()
|
function run_test_erasure_sets() {
|
||||||
{
|
|
||||||
start_minio_erasure_sets
|
start_minio_erasure_sets
|
||||||
|
|
||||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||||
@@ -123,8 +115,7 @@ function run_test_erasure_sets()
|
|||||||
return "$rv"
|
return "$rv"
|
||||||
}
|
}
|
||||||
|
|
||||||
function run_test_pool_erasure_sets()
|
function run_test_pool_erasure_sets() {
|
||||||
{
|
|
||||||
start_minio_pool_erasure_sets
|
start_minio_pool_erasure_sets
|
||||||
|
|
||||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||||
@@ -147,8 +138,7 @@ function run_test_pool_erasure_sets()
|
|||||||
return "$rv"
|
return "$rv"
|
||||||
}
|
}
|
||||||
|
|
||||||
function run_test_pool_erasure_sets_ipv6()
|
function run_test_pool_erasure_sets_ipv6() {
|
||||||
{
|
|
||||||
start_minio_pool_erasure_sets_ipv6
|
start_minio_pool_erasure_sets_ipv6
|
||||||
|
|
||||||
export SERVER_ENDPOINT="[::1]:9000"
|
export SERVER_ENDPOINT="[::1]:9000"
|
||||||
@@ -173,8 +163,7 @@ function run_test_pool_erasure_sets_ipv6()
|
|||||||
return "$rv"
|
return "$rv"
|
||||||
}
|
}
|
||||||
|
|
||||||
function run_test_erasure()
|
function run_test_erasure() {
|
||||||
{
|
|
||||||
start_minio_erasure
|
start_minio_erasure
|
||||||
|
|
||||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||||
@@ -191,8 +180,7 @@ function run_test_erasure()
|
|||||||
return "$rv"
|
return "$rv"
|
||||||
}
|
}
|
||||||
|
|
||||||
function run_test_dist_erasure()
|
function run_test_dist_erasure() {
|
||||||
{
|
|
||||||
start_minio_dist_erasure
|
start_minio_dist_erasure
|
||||||
|
|
||||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||||
@@ -217,13 +205,11 @@ function run_test_dist_erasure()
|
|||||||
return "$rv"
|
return "$rv"
|
||||||
}
|
}
|
||||||
|
|
||||||
function purge()
|
function purge() {
|
||||||
{
|
|
||||||
rm -rf "$1"
|
rm -rf "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
function __init__()
|
function __init__() {
|
||||||
{
|
|
||||||
echo "Initializing environment"
|
echo "Initializing environment"
|
||||||
mkdir -p "$WORK_DIR"
|
mkdir -p "$WORK_DIR"
|
||||||
mkdir -p "$MINIO_CONFIG_DIR"
|
mkdir -p "$MINIO_CONFIG_DIR"
|
||||||
@@ -245,7 +231,7 @@ function __init__()
|
|||||||
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
||||||
|
|
||||||
## version is purposefully set to '3' for minio to migrate configuration file
|
## version is purposefully set to '3' for minio to migrate configuration file
|
||||||
echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
|
echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
|
||||||
|
|
||||||
if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
|
if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
|
||||||
echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
|
echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
|
||||||
@@ -256,8 +242,7 @@ function __init__()
|
|||||||
chmod a+x "$FUNCTIONAL_TESTS"
|
chmod a+x "$FUNCTIONAL_TESTS"
|
||||||
}
|
}
|
||||||
|
|
||||||
function main()
|
function main() {
|
||||||
{
|
|
||||||
echo "Testing in FS setup"
|
echo "Testing in FS setup"
|
||||||
if ! run_test_fs; then
|
if ! run_test_fs; then
|
||||||
echo "FAILED"
|
echo "FAILED"
|
||||||
@@ -303,7 +288,7 @@ function main()
|
|||||||
purge "$WORK_DIR"
|
purge "$WORK_DIR"
|
||||||
}
|
}
|
||||||
|
|
||||||
( __init__ "$@" && main "$@" )
|
(__init__ "$@" && main "$@")
|
||||||
rv=$?
|
rv=$?
|
||||||
purge "$WORK_DIR"
|
purge "$WORK_DIR"
|
||||||
exit "$rv"
|
exit "$rv"
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ set -E
|
|||||||
set -o pipefail
|
set -o pipefail
|
||||||
set -x
|
set -x
|
||||||
|
|
||||||
|
|
||||||
if [ ! -x "$PWD/minio" ]; then
|
if [ ! -x "$PWD/minio" ]; then
|
||||||
echo "minio executable binary not found in current directory"
|
echo "minio executable binary not found in current directory"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -12,8 +11,7 @@ fi
|
|||||||
|
|
||||||
WORK_DIR="$(mktemp -d)"
|
WORK_DIR="$(mktemp -d)"
|
||||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||||
|
|
||||||
|
|
||||||
function start_minio() {
|
function start_minio() {
|
||||||
start_port=$1
|
start_port=$1
|
||||||
@@ -26,35 +24,37 @@ function start_minio() {
|
|||||||
|
|
||||||
args=()
|
args=()
|
||||||
for i in $(seq 1 4); do
|
for i in $(seq 1 4); do
|
||||||
args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ")
|
args+=("http://localhost:$((start_port + i))${WORK_DIR}/mnt/disk$i/ ")
|
||||||
done
|
done
|
||||||
|
|
||||||
for i in $(seq 1 4); do
|
for i in $(seq 1 4); do
|
||||||
"${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
|
"${MINIO[@]}" --address ":$((start_port + i))" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
|
||||||
done
|
done
|
||||||
|
|
||||||
# Wait until all nodes return 403
|
# Wait until all nodes return 403
|
||||||
for i in $(seq 1 4); do
|
for i in $(seq 1 4); do
|
||||||
while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do
|
while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$((start_port + i)))" -ne "403" ]; do
|
||||||
echo -n ".";
|
echo -n "."
|
||||||
sleep 1;
|
sleep 1
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Prepare fake disks with losetup
|
# Prepare fake disks with losetup
|
||||||
function prepare_block_devices() {
|
function prepare_block_devices() {
|
||||||
|
set -e
|
||||||
mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
|
mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
|
||||||
|
sudo modprobe loop
|
||||||
for i in 1 2 3 4; do
|
for i in 1 2 3 4; do
|
||||||
dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
|
dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
|
||||||
mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
|
device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
|
||||||
sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
|
sudo mkfs.ext4 -F ${device}
|
||||||
sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
|
mkdir -p ${WORK_DIR}/mnt/disk${i}/
|
||||||
mkdir -p ${WORK_DIR}/mnt/disk$i/
|
sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/
|
||||||
sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
|
sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/
|
||||||
sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
|
|
||||||
done
|
done
|
||||||
|
set +e
|
||||||
}
|
}
|
||||||
|
|
||||||
# Start a distributed MinIO setup, unmount one disk and check if it is formatted
|
# Start a distributed MinIO setup, unmount one disk and check if it is formatted
|
||||||
@@ -88,10 +88,9 @@ function cleanup() {
|
|||||||
rm -rf "$WORK_DIR"
|
rm -rf "$WORK_DIR"
|
||||||
}
|
}
|
||||||
|
|
||||||
( prepare_block_devices )
|
(prepare_block_devices)
|
||||||
( main "$@" )
|
(main "$@")
|
||||||
rv=$?
|
rv=$?
|
||||||
|
|
||||||
cleanup
|
cleanup
|
||||||
exit "$rv"
|
exit "$rv"
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ fi
|
|||||||
|
|
||||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
|
||||||
|
|
||||||
function start_minio_3_node() {
|
function start_minio_3_node() {
|
||||||
export MINIO_ROOT_USER=minio
|
export MINIO_ROOT_USER=minio
|
||||||
@@ -22,24 +22,24 @@ function start_minio_3_node() {
|
|||||||
start_port=$2
|
start_port=$2
|
||||||
args=""
|
args=""
|
||||||
for i in $(seq 1 3); do
|
for i in $(seq 1 3); do
|
||||||
args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
|
args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/"
|
||||||
done
|
done
|
||||||
|
|
||||||
"${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
|
"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
|
||||||
pid1=$!
|
pid1=$!
|
||||||
disown ${pid1}
|
disown ${pid1}
|
||||||
|
|
||||||
"${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
|
"${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 &
|
||||||
pid2=$!
|
pid2=$!
|
||||||
disown $pid2
|
disown $pid2
|
||||||
|
|
||||||
"${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
|
"${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 &
|
||||||
pid3=$!
|
pid3=$!
|
||||||
disown $pid3
|
disown $pid3
|
||||||
|
|
||||||
sleep "$1"
|
sleep "$1"
|
||||||
|
|
||||||
if ! ps -p $pid1 1>&2 > /dev/null; then
|
if ! ps -p $pid1 1>&2 >/dev/null; then
|
||||||
echo "server1 log:"
|
echo "server1 log:"
|
||||||
cat "${WORK_DIR}/dist-minio-server1.log"
|
cat "${WORK_DIR}/dist-minio-server1.log"
|
||||||
echo "FAILED"
|
echo "FAILED"
|
||||||
@@ -47,7 +47,7 @@ function start_minio_3_node() {
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! ps -p $pid2 1>&2 > /dev/null; then
|
if ! ps -p $pid2 1>&2 >/dev/null; then
|
||||||
echo "server2 log:"
|
echo "server2 log:"
|
||||||
cat "${WORK_DIR}/dist-minio-server2.log"
|
cat "${WORK_DIR}/dist-minio-server2.log"
|
||||||
echo "FAILED"
|
echo "FAILED"
|
||||||
@@ -55,7 +55,7 @@ function start_minio_3_node() {
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! ps -p $pid3 1>&2 > /dev/null; then
|
if ! ps -p $pid3 1>&2 >/dev/null; then
|
||||||
echo "server3 log:"
|
echo "server3 log:"
|
||||||
cat "${WORK_DIR}/dist-minio-server3.log"
|
cat "${WORK_DIR}/dist-minio-server3.log"
|
||||||
echo "FAILED"
|
echo "FAILED"
|
||||||
@@ -73,7 +73,7 @@ function start_minio_3_node() {
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
sleep 1;
|
sleep 1
|
||||||
if pgrep minio; then
|
if pgrep minio; then
|
||||||
# forcibly killing, to proceed further properly.
|
# forcibly killing, to proceed further properly.
|
||||||
if ! pkill -9 minio; then
|
if ! pkill -9 minio; then
|
||||||
@@ -82,26 +82,23 @@ function start_minio_3_node() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
function check_online() {
|
function check_online() {
|
||||||
if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
|
if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
|
||||||
echo "1"
|
echo "1"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function purge()
|
function purge() {
|
||||||
{
|
|
||||||
rm -rf "$1"
|
rm -rf "$1"
|
||||||
}
|
}
|
||||||
|
|
||||||
function __init__()
|
function __init__() {
|
||||||
{
|
|
||||||
echo "Initializing environment"
|
echo "Initializing environment"
|
||||||
mkdir -p "$WORK_DIR"
|
mkdir -p "$WORK_DIR"
|
||||||
mkdir -p "$MINIO_CONFIG_DIR"
|
mkdir -p "$MINIO_CONFIG_DIR"
|
||||||
|
|
||||||
## version is purposefully set to '3' for minio to migrate configuration file
|
## version is purposefully set to '3' for minio to migrate configuration file
|
||||||
echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
|
echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
|
||||||
}
|
}
|
||||||
|
|
||||||
function perform_test() {
|
function perform_test() {
|
||||||
@@ -127,8 +124,7 @@ function perform_test() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function main()
|
function main() {
|
||||||
{
|
|
||||||
# use same ports for all tests
|
# use same ports for all tests
|
||||||
start_port=$(shuf -i 10000-65000 -n 1)
|
start_port=$(shuf -i 10000-65000 -n 1)
|
||||||
|
|
||||||
@@ -137,7 +133,7 @@ function main()
|
|||||||
perform_test "3" ${start_port}
|
perform_test "3" ${start_port}
|
||||||
}
|
}
|
||||||
|
|
||||||
( __init__ "$@" && main "$@" )
|
(__init__ "$@" && main "$@")
|
||||||
rv=$?
|
rv=$?
|
||||||
purge "$WORK_DIR"
|
purge "$WORK_DIR"
|
||||||
exit "$rv"
|
exit "$rv"
|
||||||
|
|||||||
+3
-3
@@ -25,7 +25,7 @@ import (
|
|||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/bucket/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Data types used for returning dummy access control
|
// Data types used for returning dummy access control
|
||||||
@@ -90,8 +90,8 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
|
|||||||
if aclHeader == "" {
|
if aclHeader == "" {
|
||||||
acl := &accessControlPolicy{}
|
acl := &accessControlPolicy{}
|
||||||
if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
|
if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
|
||||||
if err == io.EOF {
|
if terr, ok := err.(*xml.SyntaxError); ok && terr.Msg == io.EOF.Error() {
|
||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML),
|
||||||
r.URL)
|
r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
+184
-226
@@ -31,8 +31,8 @@ import (
|
|||||||
|
|
||||||
jsoniter "github.com/json-iterator/go"
|
jsoniter "github.com/json-iterator/go"
|
||||||
"github.com/klauspost/compress/zip"
|
"github.com/klauspost/compress/zip"
|
||||||
"github.com/minio/kes"
|
"github.com/minio/kes-go"
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7/pkg/tags"
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||||
@@ -41,8 +41,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/bucket/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -56,11 +55,9 @@ const (
|
|||||||
// specified in the quota configuration will be applied by default
|
// specified in the quota configuration will be applied by default
|
||||||
// to enforce total quota for the specified bucket.
|
// to enforce total quota for the specified bucket.
|
||||||
func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "PutBucketQuotaConfig")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketQuotaAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -85,11 +82,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if quotaConfig.Type == "fifo" {
|
|
||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
|
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
@@ -102,7 +94,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
|||||||
Quota: data,
|
Quota: data,
|
||||||
UpdatedAt: updatedAt,
|
UpdatedAt: updatedAt,
|
||||||
}
|
}
|
||||||
if quotaConfig.Quota == 0 {
|
if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
|
||||||
bucketMeta.Quota = nil
|
bucketMeta.Quota = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,11 +107,9 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
// GetBucketQuotaConfigHandler - gets bucket quota configuration
|
// GetBucketQuotaConfigHandler - gets bucket quota configuration
|
||||||
func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "GetBucketQuotaConfig")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketQuotaAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -150,15 +140,14 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
// SetRemoteTargetHandler - sets a remote target for bucket
|
// SetRemoteTargetHandler - sets a remote target for bucket
|
||||||
func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SetBucketTarget")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
bucket := pathClean(vars["bucket"])
|
bucket := pathClean(vars["bucket"])
|
||||||
update := r.Form.Get("update") == "true"
|
update := r.Form.Get("update") == "true"
|
||||||
|
|
||||||
// Get current object layer instance.
|
// Get current object layer instance.
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -169,7 +158,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
|
cred, _, s3Err := validateAdminSignature(ctx, r, "")
|
||||||
if s3Err != ErrNone {
|
if s3Err != ErrNone {
|
||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -199,7 +188,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
|||||||
ops = madmin.GetTargetUpdateOps(r.Form)
|
ops = madmin.GetTargetUpdateOps(r.Form)
|
||||||
} else {
|
} else {
|
||||||
var exists bool // true if arn exists
|
var exists bool // true if arn exists
|
||||||
target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target)
|
target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target, "")
|
||||||
if exists && target.Arn != "" { // return pre-existing ARN
|
if exists && target.Arn != "" { // return pre-existing ARN
|
||||||
data, err := json.Marshal(target.Arn)
|
data, err := json.Marshal(target.Arn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -215,6 +204,11 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
|||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if globalSiteReplicationSys.isEnabled() && !update {
|
||||||
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyAddError, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if update {
|
if update {
|
||||||
// overlay the updates on existing target
|
// overlay the updates on existing target
|
||||||
tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, target.Arn)
|
tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, target.Arn)
|
||||||
@@ -226,11 +220,13 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
|||||||
switch op {
|
switch op {
|
||||||
case madmin.CredentialsUpdateType:
|
case madmin.CredentialsUpdateType:
|
||||||
if !globalSiteReplicationSys.isEnabled() {
|
if !globalSiteReplicationSys.isEnabled() {
|
||||||
|
// credentials update is possible only in bucket replication. User will never
|
||||||
|
// know the site replicator creds.
|
||||||
tgt.Credentials = target.Credentials
|
tgt.Credentials = target.Credentials
|
||||||
tgt.TargetBucket = target.TargetBucket
|
tgt.TargetBucket = target.TargetBucket
|
||||||
}
|
|
||||||
tgt.Secure = target.Secure
|
tgt.Secure = target.Secure
|
||||||
tgt.Endpoint = target.Endpoint
|
tgt.Endpoint = target.Endpoint
|
||||||
|
}
|
||||||
case madmin.SyncUpdateType:
|
case madmin.SyncUpdateType:
|
||||||
tgt.ReplicationSync = target.ReplicationSync
|
tgt.ReplicationSync = target.ReplicationSync
|
||||||
case madmin.ProxyUpdateType:
|
case madmin.ProxyUpdateType:
|
||||||
@@ -287,15 +283,14 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
|||||||
// ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target
|
// ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target
|
||||||
// for a particular ARN type
|
// for a particular ARN type
|
||||||
func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ListBucketTargets")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
bucket := pathClean(vars["bucket"])
|
bucket := pathClean(vars["bucket"])
|
||||||
arnType := vars["type"]
|
arnType := vars["type"]
|
||||||
|
|
||||||
// Get current object layer instance.
|
// Get current object layer instance.
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketTargetAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -322,15 +317,14 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
|
|||||||
|
|
||||||
// RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN
|
// RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN
|
||||||
func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "RemoveBucketTarget")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
bucket := pathClean(vars["bucket"])
|
bucket := pathClean(vars["bucket"])
|
||||||
arn := vars["arn"]
|
arn := vars["arn"]
|
||||||
|
|
||||||
// Get current object layer instance.
|
// Get current object layer instance.
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -366,12 +360,11 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
|
|||||||
|
|
||||||
// ExportBucketMetadataHandler - exports all bucket metadata as a zipped file
|
// ExportBucketMetadataHandler - exports all bucket metadata as a zipped file
|
||||||
func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ExportBucketMetadata")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
bucket := pathClean(r.Form.Get("bucket"))
|
bucket := pathClean(r.Form.Get("bucket"))
|
||||||
// Get current object layer instance.
|
// Get current object layer instance.
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportBucketMetadataAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -399,7 +392,8 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
// of bucket metadata
|
// of bucket metadata
|
||||||
zipWriter := zip.NewWriter(w)
|
zipWriter := zip.NewWriter(w)
|
||||||
defer zipWriter.Close()
|
defer zipWriter.Close()
|
||||||
rawDataFn := func(r io.Reader, filename string, sz int) error {
|
|
||||||
|
rawDataFn := func(r io.Reader, filename string, sz int) {
|
||||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||||
name: filename,
|
name: filename,
|
||||||
size: int64(sz),
|
size: int64(sz),
|
||||||
@@ -408,20 +402,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
isDir: false,
|
isDir: false,
|
||||||
sys: nil,
|
sys: nil,
|
||||||
})
|
})
|
||||||
if zerr != nil {
|
if zerr == nil {
|
||||||
logger.LogIf(ctx, zerr)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
header.Method = zip.Deflate
|
header.Method = zip.Deflate
|
||||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||||
if zerr != nil {
|
if zerr == nil {
|
||||||
logger.LogIf(ctx, zerr)
|
io.Copy(zwriter, r)
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
if _, err := io.Copy(zwriter, r); err != nil {
|
|
||||||
logger.LogIf(ctx, err)
|
|
||||||
}
|
}
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cfgFiles := []string{
|
cfgFiles := []string{
|
||||||
@@ -453,12 +440,9 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case bucketLifecycleConfig:
|
case bucketLifecycleConfig:
|
||||||
config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
|
if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
|
||||||
continue
|
continue
|
||||||
@@ -472,10 +456,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case bucketQuotaConfigFile:
|
case bucketQuotaConfigFile:
|
||||||
config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
|
config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -490,10 +471,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case bucketSSEConfig:
|
case bucketSSEConfig:
|
||||||
config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
|
config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -508,10 +486,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case bucketTaggingConfig:
|
case bucketTaggingConfig:
|
||||||
config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
|
config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -526,10 +501,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case objectLockConfig:
|
case objectLockConfig:
|
||||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -545,10 +517,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case bucketVersioningConfig:
|
case bucketVersioningConfig:
|
||||||
config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
|
config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -564,10 +533,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case bucketReplicationConfig:
|
case bucketReplicationConfig:
|
||||||
config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
|
config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -582,11 +548,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
case bucketTargetsFile:
|
case bucketTargetsFile:
|
||||||
config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
|
config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -602,10 +564,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -650,12 +609,10 @@ func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
|||||||
// 2. Replication config - is omitted from import as remote target credentials are not available from exported data for security reasons.
|
// 2. Replication config - is omitted from import as remote target credentials are not available from exported data for security reasons.
|
||||||
// 3. lifecycle config - if transition rules are present, tier name needs to have been defined.
|
// 3. lifecycle config - if transition rules are present, tier name needs to have been defined.
|
||||||
func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ImportBucketMetadata")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
// Get current object layer instance.
|
// Get current object layer instance.
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportBucketMetadataAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -670,12 +627,31 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
bucketMap := make(map[string]struct{}, 1)
|
|
||||||
rpt := importMetaReport{
|
rpt := importMetaReport{
|
||||||
madmin.BucketMetaImportErrs{
|
madmin.BucketMetaImportErrs{
|
||||||
Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
|
Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
||||||
|
|
||||||
|
updatedAt := UTCNow()
|
||||||
|
|
||||||
|
for _, file := range zr.File {
|
||||||
|
slc := strings.Split(file.Name, slashSeparator)
|
||||||
|
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
|
||||||
|
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bucket := slc[0]
|
||||||
|
meta, err := readBucketMetadata(ctx, objectAPI, bucket)
|
||||||
|
if err == nil {
|
||||||
|
bucketMap[bucket] = &meta
|
||||||
|
} else if err != errConfigNotFound {
|
||||||
|
rpt.SetStatus(bucket, "", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// import object lock config if any - order of import matters here.
|
// import object lock config if any - order of import matters here.
|
||||||
for _, file := range zr.File {
|
for _, file := range zr.File {
|
||||||
slc := strings.Split(file.Name, slashSeparator)
|
slc := strings.Split(file.Name, slashSeparator)
|
||||||
@@ -684,8 +660,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bucket, fileName := slc[0], slc[1]
|
bucket, fileName := slc[0], slc[1]
|
||||||
switch fileName {
|
if fileName == objectLockConfig {
|
||||||
case objectLockConfig:
|
|
||||||
reader, err := file.Open()
|
reader, err := file.Open()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
@@ -704,45 +679,21 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
}
|
}
|
||||||
if _, ok := bucketMap[bucket]; !ok {
|
if _, ok := bucketMap[bucket]; !ok {
|
||||||
opts := MakeBucketOptions{
|
opts := MakeBucketOptions{
|
||||||
LockEnabled: config.ObjectLockEnabled == "Enabled",
|
LockEnabled: config.Enabled(),
|
||||||
|
ForceCreate: true, // ignore if it already exists
|
||||||
}
|
}
|
||||||
err = objectAPI.MakeBucket(ctx, bucket, opts)
|
err = objectAPI.MakeBucket(ctx, bucket, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if _, ok := err.(BucketExists); !ok {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
v := newBucketMetadata(bucket)
|
||||||
bucketMap[bucket] = struct{}{}
|
bucketMap[bucket] = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deny object locking configuration settings on existing buckets without object lock enabled.
|
bucketMap[bucket].ObjectLockConfigXML = configData
|
||||||
if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
|
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
|
|
||||||
if err != nil {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
|
|
||||||
// Call site replication hook.
|
|
||||||
//
|
|
||||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
|
||||||
// errors.
|
|
||||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
|
||||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
|
||||||
Type: madmin.SRBucketMetaTypeObjectLockConfig,
|
|
||||||
Bucket: bucket,
|
|
||||||
ObjectLockConfig: &cfgStr,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}); err != nil {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -754,8 +705,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bucket, fileName := slc[0], slc[1]
|
bucket, fileName := slc[0], slc[1]
|
||||||
switch fileName {
|
if fileName == bucketVersioningConfig {
|
||||||
case bucketVersioningConfig:
|
|
||||||
reader, err := file.Open()
|
reader, err := file.Open()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
@@ -767,13 +717,14 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if _, ok := bucketMap[bucket]; !ok {
|
if _, ok := bucketMap[bucket]; !ok {
|
||||||
if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
|
||||||
if _, ok := err.(BucketExists); !ok {
|
ForceCreate: true, // ignore if it already exists
|
||||||
|
}); err != nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
v := newBucketMetadata(bucket)
|
||||||
bucketMap[bucket] = struct{}{}
|
bucketMap[bucket] = &v
|
||||||
}
|
}
|
||||||
|
|
||||||
if globalSiteReplicationSys.isEnabled() && v.Suspended() {
|
if globalSiteReplicationSys.isEnabled() && v.Suspended() {
|
||||||
@@ -796,10 +747,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
|
bucketMap[bucket].VersioningConfigXML = configData
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
||||||
continue
|
|
||||||
}
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -817,16 +766,18 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bucket, fileName := slc[0], slc[1]
|
bucket, fileName := slc[0], slc[1]
|
||||||
|
|
||||||
// create bucket if it does not exist yet.
|
// create bucket if it does not exist yet.
|
||||||
if _, ok := bucketMap[bucket]; !ok {
|
if _, ok := bucketMap[bucket]; !ok {
|
||||||
err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
|
err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
|
||||||
|
ForceCreate: true, // ignore if it already exists
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if _, ok := err.(BucketExists); !ok {
|
|
||||||
rpt.SetStatus(bucket, "", err)
|
rpt.SetStatus(bucket, "", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
v := newBucketMetadata(bucket)
|
||||||
bucketMap[bucket] = struct{}{}
|
bucketMap[bucket] = &v
|
||||||
}
|
}
|
||||||
if _, ok := bucketMap[bucket]; !ok {
|
if _, ok := bucketMap[bucket]; !ok {
|
||||||
continue
|
continue
|
||||||
@@ -845,12 +796,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
|
bucketMap[bucket].NotificationConfigXML = configData
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rulesMap := config.ToRulesMap()
|
|
||||||
globalEventNotifier.AddRulesMap(bucket, rulesMap)
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
case bucketPolicyConfig:
|
case bucketPolicyConfig:
|
||||||
// Error out if Content-Length is beyond allowed size.
|
// Error out if Content-Length is beyond allowed size.
|
||||||
@@ -865,7 +811,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
continue
|
continue
|
||||||
@@ -883,22 +829,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
|
bucketMap[bucket].PolicyConfigJSON = configData
|
||||||
if err != nil {
|
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
// Call site replication hook.
|
|
||||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
|
||||||
Type: madmin.SRBucketMetaTypePolicy,
|
|
||||||
Bucket: bucket,
|
|
||||||
Policy: bucketPolicyBytes,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}); err != nil {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
case bucketLifecycleConfig:
|
case bucketLifecycleConfig:
|
||||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -924,10 +857,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
|
bucketMap[bucket].LifecycleConfigXML = configData
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
||||||
continue
|
|
||||||
}
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
case bucketSSEConfig:
|
case bucketSSEConfig:
|
||||||
// Parse bucket encryption xml
|
// Parse bucket encryption xml
|
||||||
@@ -962,29 +893,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store the bucket encryption configuration in the object layer
|
bucketMap[bucket].EncryptionConfigXML = configData
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
|
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
||||||
if err != nil {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
|
|
||||||
// Call site replication hook.
|
|
||||||
//
|
|
||||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
|
||||||
// errors.
|
|
||||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
|
||||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
|
||||||
Type: madmin.SRBucketMetaTypeSSEConfig,
|
|
||||||
Bucket: bucket,
|
|
||||||
SSEConfig: &cfgStr,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}); err != nil {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
case bucketTaggingConfig:
|
case bucketTaggingConfig:
|
||||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -998,27 +909,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
|
bucketMap[bucket].TaggingConfigXML = configData
|
||||||
if err != nil {
|
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
|
|
||||||
// Call site replication hook.
|
|
||||||
//
|
|
||||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
|
||||||
// errors.
|
|
||||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
|
||||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
|
||||||
Type: madmin.SRBucketMetaTypeTags,
|
|
||||||
Bucket: bucket,
|
|
||||||
Tags: &cfgStr,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}); err != nil {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
case bucketQuotaConfigFile:
|
case bucketQuotaConfigFile:
|
||||||
data, err := io.ReadAll(reader)
|
data, err := io.ReadAll(reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1026,40 +919,47 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
quotaConfig, err := parseBucketQuota(bucket, data)
|
_, err = parseBucketQuota(bucket, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
rpt.SetStatus(bucket, fileName, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if quotaConfig.Type == "fifo" {
|
bucketMap[bucket].QuotaConfigJSON = data
|
||||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
|
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
|
|
||||||
if err != nil {
|
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rpt.SetStatus(bucket, fileName, nil)
|
rpt.SetStatus(bucket, fileName, nil)
|
||||||
|
|
||||||
bucketMeta := madmin.SRBucketMeta{
|
|
||||||
Type: madmin.SRBucketMetaTypeQuotaConfig,
|
|
||||||
Bucket: bucket,
|
|
||||||
Quota: data,
|
|
||||||
UpdatedAt: updatedAt,
|
|
||||||
}
|
}
|
||||||
if quotaConfig.Quota == 0 {
|
|
||||||
bucketMeta.Quota = nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Call site replication hook.
|
enc := func(b []byte) *string {
|
||||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
|
if b == nil {
|
||||||
rpt.SetStatus(bucket, fileName, err)
|
return nil
|
||||||
|
}
|
||||||
|
v := base64.StdEncoding.EncodeToString(b)
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
for bucket, meta := range bucketMap {
|
||||||
|
err := globalBucketMetadataSys.save(ctx, *meta)
|
||||||
|
if err != nil {
|
||||||
|
rpt.SetStatus(bucket, "", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// Call site replication hook.
|
||||||
|
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||||
|
Bucket: bucket,
|
||||||
|
Quota: meta.QuotaConfigJSON,
|
||||||
|
Policy: meta.PolicyConfigJSON,
|
||||||
|
Versioning: enc(meta.VersioningConfigXML),
|
||||||
|
Tags: enc(meta.TaggingConfigXML),
|
||||||
|
ObjectLockConfig: enc(meta.ObjectLockConfigXML),
|
||||||
|
SSEConfig: enc(meta.EncryptionConfigXML),
|
||||||
|
UpdatedAt: updatedAt,
|
||||||
|
}); err != nil {
|
||||||
|
rpt.SetStatus(bucket, "", err)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
||||||
@@ -1074,13 +974,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
|||||||
// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
|
// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
|
||||||
// to the connected HTTP client.
|
// to the connected HTTP client.
|
||||||
func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ReplicationDiff")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
bucket := vars["bucket"]
|
bucket := vars["bucket"]
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1134,3 +1033,62 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReplicationMRFHandler - POST returns info on entries in the MRF backlog for a node or all nodes
|
||||||
|
func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
|
||||||
|
vars := mux.Vars(r)
|
||||||
|
bucket := vars["bucket"]
|
||||||
|
|
||||||
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff)
|
||||||
|
if objectAPI == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if bucket exists.
|
||||||
|
if bucket != "" {
|
||||||
|
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
q := r.Form
|
||||||
|
node := q.Get("node")
|
||||||
|
|
||||||
|
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
|
||||||
|
defer keepAliveTicker.Stop()
|
||||||
|
|
||||||
|
mrfCh, err := globalNotificationSys.GetReplicationMRF(ctx, bucket, node)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
enc := json.NewEncoder(w)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case entry, ok := <-mrfCh:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := enc.Encode(entry); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(mrfCh) == 0 {
|
||||||
|
// Flush if nothing is queued
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
}
|
||||||
|
case <-keepAliveTicker.C:
|
||||||
|
if len(mrfCh) > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := w.Write([]byte(" ")); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+14
-10
@@ -23,18 +23,18 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/minio/kes"
|
"github.com/minio/kes-go"
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// validateAdminReq will validate request against and return whether it is allowed.
|
// validateAdminReq will validate request against and return whether it is allowed.
|
||||||
// If any of the supplied actions are allowed it will be successful.
|
// If any of the supplied actions are allowed it will be successful.
|
||||||
// If nil ObjectLayer is returned, the operation is not permitted.
|
// If nil ObjectLayer is returned, the operation is not permitted.
|
||||||
// When nil ObjectLayer has been returned an error has always been sent to w.
|
// When nil ObjectLayer has been returned an error has always been sent to w.
|
||||||
func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
|
func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...policy.AdminAction) (ObjectLayer, auth.Credentials) {
|
||||||
// Get current object layer instance.
|
// Get current object layer instance.
|
||||||
objectAPI := newObjectLayerFn()
|
objectAPI := newObjectLayerFn()
|
||||||
if objectAPI == nil || globalNotificationSys == nil {
|
if objectAPI == nil || globalNotificationSys == nil {
|
||||||
@@ -78,7 +78,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
|||||||
|
|
||||||
var apiErr APIError
|
var apiErr APIError
|
||||||
switch e := err.(type) {
|
switch e := err.(type) {
|
||||||
case iampolicy.Error:
|
case policy.Error:
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "XMinioMalformedIAMPolicy",
|
Code: "XMinioMalformedIAMPolicy",
|
||||||
Description: e.Error(),
|
Description: e.Error(),
|
||||||
@@ -168,10 +168,16 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
|||||||
}
|
}
|
||||||
case errors.Is(err, errPolicyInUse):
|
case errors.Is(err, errPolicyInUse):
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "XMinioAdminPolicyInUse",
|
Code: "XMinioIAMPolicyInUse",
|
||||||
Description: "The policy cannot be removed, as it is in use",
|
Description: "The policy cannot be removed, as it is in use",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
}
|
}
|
||||||
|
case errors.Is(err, errSessionPolicyTooLarge):
|
||||||
|
apiErr = APIError{
|
||||||
|
Code: "XMinioIAMServiceAccountSessionPolicyTooLarge",
|
||||||
|
Description: err.Error(),
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
}
|
||||||
case errors.Is(err, kes.ErrKeyExists):
|
case errors.Is(err, kes.ErrKeyExists):
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "XMinioKMSKeyExists",
|
Code: "XMinioKMSKeyExists",
|
||||||
@@ -220,12 +226,10 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
|||||||
// toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
|
// toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
|
||||||
// specific error.
|
// specific error.
|
||||||
func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
|
func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
|
||||||
switch err {
|
if errors.Is(err, errErasureWriteQuorum) {
|
||||||
case errErasureWriteQuorum:
|
|
||||||
return ErrAdminConfigNoQuorum
|
return ErrAdminConfigNoQuorum
|
||||||
default:
|
|
||||||
return toAPIErrorCode(ctx, err)
|
|
||||||
}
|
}
|
||||||
|
return toAPIErrorCode(ctx, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// wraps export error for more context
|
// wraps export error for more context
|
||||||
|
|||||||
@@ -26,27 +26,25 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
"github.com/minio/minio/internal/config/cache"
|
|
||||||
"github.com/minio/minio/internal/config/etcd"
|
"github.com/minio/minio/internal/config/etcd"
|
||||||
xldap "github.com/minio/minio/internal/config/identity/ldap"
|
xldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||||
"github.com/minio/minio/internal/config/identity/openid"
|
"github.com/minio/minio/internal/config/identity/openid"
|
||||||
idplugin "github.com/minio/minio/internal/config/identity/plugin"
|
idplugin "github.com/minio/minio/internal/config/identity/plugin"
|
||||||
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
polplugin "github.com/minio/minio/internal/config/policy/plugin"
|
||||||
"github.com/minio/minio/internal/config/storageclass"
|
"github.com/minio/minio/internal/config/storageclass"
|
||||||
|
"github.com/minio/minio/internal/config/subnet"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
||||||
func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "DeleteConfigKV")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -82,11 +80,15 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = validateConfig(cfg, subSys); err != nil {
|
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if subnet proxy being deleted and if so the value of proxy of subnet
|
||||||
|
// target of logger webhook configuration also should be deleted
|
||||||
|
loggerWebhookProxyDeleted := setLoggerWebhookSubnetProxy(subSys, cfg)
|
||||||
|
|
||||||
if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
|
if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -101,6 +103,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
dynamic := config.SubSystemsDynamic.Contains(subSys)
|
dynamic := config.SubSystemsDynamic.Contains(subSys)
|
||||||
if dynamic {
|
if dynamic {
|
||||||
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
|
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
|
||||||
|
if subSys == config.SubnetSubSys && loggerWebhookProxyDeleted {
|
||||||
|
// Logger webhook proxy deleted, apply the dynamic changes
|
||||||
|
applyDynamic(ctx, objectAPI, cfg, config.LoggerWebhookSubSys, r, w)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -135,15 +141,14 @@ type setConfigResult struct {
|
|||||||
Cfg config.Config
|
Cfg config.Config
|
||||||
SubSys string
|
SubSys string
|
||||||
Dynamic bool
|
Dynamic bool
|
||||||
|
LoggerWebhookCfgUpdated bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
|
// SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
|
||||||
func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SetConfigKV")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -175,6 +180,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
|
|
||||||
if result.Dynamic {
|
if result.Dynamic {
|
||||||
applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w)
|
applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w)
|
||||||
|
// If logger webhook config updated (proxy due to callhome), explicitly dynamically
|
||||||
|
// apply the config
|
||||||
|
if result.LoggerWebhookCfgUpdated {
|
||||||
|
applyDynamic(ctx, objectAPI, result.Cfg, config.LoggerWebhookSubSys, r, w)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
writeSuccessResponseHeadersOnly(w)
|
writeSuccessResponseHeadersOnly(w)
|
||||||
@@ -196,11 +206,20 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if verr := validateConfig(result.Cfg, result.SubSys); verr != nil {
|
tgts, err := config.ParseConfigTargetID(bytes.NewReader(kvBytes))
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx = context.WithValue(ctx, config.ContextKeyForTargetFromConfig, tgts)
|
||||||
|
if verr := validateConfig(ctx, result.Cfg, result.SubSys); verr != nil {
|
||||||
err = badConfigErr{Err: verr}
|
err = badConfigErr{Err: verr}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if subnet proxy being set and if so set the same value to proxy of subnet
|
||||||
|
// target of logger webhook configuration
|
||||||
|
result.LoggerWebhookCfgUpdated = setLoggerWebhookSubnetProxy(result.SubSys, result.Cfg)
|
||||||
|
|
||||||
// Update the actual server config on disk.
|
// Update the actual server config on disk.
|
||||||
if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil {
|
if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil {
|
||||||
return
|
return
|
||||||
@@ -217,12 +236,12 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
|
|||||||
// 1. `subsys:target` -> request for config of a single subsystem and target pair.
|
// 1. `subsys:target` -> request for config of a single subsystem and target pair.
|
||||||
// 2. `subsys:` -> request for config of a single subsystem and the default target.
|
// 2. `subsys:` -> request for config of a single subsystem and the default target.
|
||||||
// 3. `subsys` -> request for config of all targets for the given subsystem.
|
// 3. `subsys` -> request for config of all targets for the given subsystem.
|
||||||
|
//
|
||||||
|
// This is a reporting API and config secrets are redacted in the response.
|
||||||
func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "GetConfigKV")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -244,7 +263,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
|
subSysConfigs, err := cfg.GetSubsysInfo(subSys, target, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -266,11 +285,9 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ClearConfigHistoryKV")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -301,11 +318,9 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
|
|||||||
|
|
||||||
// RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id.
|
// RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id.
|
||||||
func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "RestoreConfigHistoryKV")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -334,7 +349,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = validateConfig(cfg, ""); err != nil {
|
if err = validateConfig(ctx, cfg, ""); err != nil {
|
||||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -349,11 +364,9 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
|
|||||||
|
|
||||||
// ListConfigHistoryKVHandler - lists all the KV ids.
|
// ListConfigHistoryKVHandler - lists all the KV ids.
|
||||||
func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ListConfigHistoryKV")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -389,11 +402,9 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
|
|||||||
|
|
||||||
// HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key}
|
// HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key}
|
||||||
func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "HelpConfigKV")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -416,11 +427,9 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
|
|||||||
|
|
||||||
// SetConfigHandler - PUT /minio/admin/v3/config
|
// SetConfigHandler - PUT /minio/admin/v3/config
|
||||||
func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SetConfig")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -445,7 +454,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = validateConfig(cfg, ""); err != nil {
|
if err = validateConfig(ctx, cfg, ""); err != nil {
|
||||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -466,13 +475,13 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetConfigHandler - GET /minio/admin/v3/config
|
// GetConfigHandler - GET /minio/admin/v3/config
|
||||||
// Get config.json of this minio setup.
|
//
|
||||||
|
// This endpoint is mainly for exporting and backing up the configuration.
|
||||||
|
// Secrets are not redacted.
|
||||||
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "GetConfig")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -483,15 +492,13 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
|||||||
hkvs := config.HelpSubSysMap[""]
|
hkvs := config.HelpSubSysMap[""]
|
||||||
for _, hkv := range hkvs {
|
for _, hkv := range hkvs {
|
||||||
// We ignore the error below, as we cannot get one.
|
// We ignore the error below, as we cannot get one.
|
||||||
cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
|
cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "", false)
|
||||||
|
|
||||||
for _, item := range cfgSubsysItems {
|
for _, item := range cfgSubsysItems {
|
||||||
off := item.Config.Get(config.Enable) == config.EnableOff
|
off := item.Config.Get(config.Enable) == config.EnableOff
|
||||||
switch hkv.Key {
|
switch hkv.Key {
|
||||||
case config.EtcdSubSys:
|
case config.EtcdSubSys:
|
||||||
off = !etcd.Enabled(item.Config)
|
off = !etcd.Enabled(item.Config)
|
||||||
case config.CacheSubSys:
|
|
||||||
off = !cache.Enabled(item.Config)
|
|
||||||
case config.StorageClassSubSys:
|
case config.StorageClassSubSys:
|
||||||
off = !storageclass.Enabled(item.Config)
|
off = !storageclass.Enabled(item.Config)
|
||||||
case config.PolicyPluginSubSys:
|
case config.PolicyPluginSubSys:
|
||||||
@@ -501,7 +508,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
|||||||
case config.IdentityLDAPSubSys:
|
case config.IdentityLDAPSubSys:
|
||||||
off = !xldap.Enabled(item.Config)
|
off = !xldap.Enabled(item.Config)
|
||||||
case config.IdentityTLSSubSys:
|
case config.IdentityTLSSubSys:
|
||||||
off = !globalSTSTLSConfig.Enabled
|
off = !globalIAMSys.STSTLSConfig.Enabled
|
||||||
case config.IdentityPluginSubSys:
|
case config.IdentityPluginSubSys:
|
||||||
off = !idplugin.Enabled(item.Config)
|
off = !idplugin.Enabled(item.Config)
|
||||||
}
|
}
|
||||||
@@ -518,3 +525,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
|||||||
|
|
||||||
writeSuccessResponseJSON(w, econfigData)
|
writeSuccessResponseJSON(w, econfigData)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
|
||||||
|
// one being set for subnet proxy
|
||||||
|
func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
|
||||||
|
if subSys == config.SubnetSubSys || subSys == config.LoggerWebhookSubSys {
|
||||||
|
subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
|
||||||
|
loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
|
||||||
|
subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
|
||||||
|
if loggerWebhookSubnetProxy != subnetProxy {
|
||||||
|
subnetWebhookCfg.Set(logger.Proxy, subnetProxy)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
@@ -26,19 +26,19 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||||
"github.com/minio/minio/internal/config/identity/openid"
|
"github.com/minio/minio/internal/config/identity/openid"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
"github.com/minio/pkg/v2/ldap"
|
||||||
"github.com/minio/pkg/ldap"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -86,7 +86,7 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
|
|||||||
if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
|
if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
|
||||||
// LDAP does not support multiple configurations. So cfgName must be
|
// LDAP does not support multiple configurations. So cfgName must be
|
||||||
// empty or `madmin.Default`.
|
// empty or `madmin.Default`.
|
||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPNonDefaultConfigName), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -121,11 +121,11 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
|
|||||||
|
|
||||||
// IDP config is not dynamic. Sanity check.
|
// IDP config is not dynamic. Sanity check.
|
||||||
if dynamic {
|
if dynamic {
|
||||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
|
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), "", r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = validateConfig(cfg, subSys); err != nil {
|
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||||
|
|
||||||
var validationErr ldap.Validation
|
var validationErr ldap.Validation
|
||||||
if errors.As(err, &validationErr) {
|
if errors.As(err, &validationErr) {
|
||||||
@@ -178,9 +178,9 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
|
|||||||
var cfgInfos []madmin.IDPCfgInfo
|
var cfgInfos []madmin.IDPCfgInfo
|
||||||
switch subSys {
|
switch subSys {
|
||||||
case madmin.IdentityOpenIDSubSys:
|
case madmin.IdentityOpenIDSubSys:
|
||||||
cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
|
cfgInfos, _ = globalIAMSys.OpenIDConfig.GetConfigInfo(s, cfgTarget)
|
||||||
case madmin.IdentityLDAPSubSys:
|
case madmin.IdentityLDAPSubSys:
|
||||||
cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
|
cfgInfos, _ = globalIAMSys.LDAPConfig.GetConfigInfo(s, cfgTarget)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(cfgInfos) > 0 && !isUpdate {
|
if len(cfgInfos) > 0 && !isUpdate {
|
||||||
@@ -198,32 +198,29 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
|
|||||||
//
|
//
|
||||||
// PUT <admin-prefix>/idp-cfg/openid/_ -> create (default) named config `_`
|
// PUT <admin-prefix>/idp-cfg/openid/_ -> create (default) named config `_`
|
||||||
func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "AddIdentityProviderCfg")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
a.addOrUpdateIDPHandler(ctx, w, r, false)
|
addOrUpdateIDPHandler(ctx, w, r, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
|
// UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
|
||||||
//
|
//
|
||||||
// PATCH <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
|
// POST <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
|
||||||
//
|
//
|
||||||
// PATCH <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
|
// POST <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
|
||||||
func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "UpdateIdentityProviderCfg")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
a.addOrUpdateIDPHandler(ctx, w, r, true)
|
addOrUpdateIDPHandler(ctx, w, r, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListIdentityProviderCfg:
|
// ListIdentityProviderCfg:
|
||||||
//
|
//
|
||||||
// GET <admin-prefix>/idp-cfg/openid -> lists openid provider configs.
|
// GET <admin-prefix>/idp-cfg/openid -> lists openid provider configs.
|
||||||
func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ListIdentityProviderCfg")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -240,10 +237,10 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
|
|||||||
switch idpCfgType {
|
switch idpCfgType {
|
||||||
case madmin.OpenidIDPCfg:
|
case madmin.OpenidIDPCfg:
|
||||||
cfg := globalServerConfig.Clone()
|
cfg := globalServerConfig.Clone()
|
||||||
cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
|
cfgList, err = globalIAMSys.OpenIDConfig.GetConfigList(cfg)
|
||||||
case madmin.LDAPIDPCfg:
|
case madmin.LDAPIDPCfg:
|
||||||
cfg := globalServerConfig.Clone()
|
cfg := globalServerConfig.Clone()
|
||||||
cfgList, err = globalLDAPConfig.GetConfigList(cfg)
|
cfgList, err = globalIAMSys.LDAPConfig.GetConfigList(cfg)
|
||||||
|
|
||||||
default:
|
default:
|
||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||||
@@ -274,10 +271,9 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
|
|||||||
//
|
//
|
||||||
// GET <admin-prefix>/idp-cfg/openid/dex_test
|
// GET <admin-prefix>/idp-cfg/openid/dex_test
|
||||||
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "GetIdentityProviderCfg")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -296,9 +292,9 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
|||||||
var err error
|
var err error
|
||||||
switch idpCfgType {
|
switch idpCfgType {
|
||||||
case madmin.OpenidIDPCfg:
|
case madmin.OpenidIDPCfg:
|
||||||
cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
|
cfgInfos, err = globalIAMSys.OpenIDConfig.GetConfigInfo(cfg, cfgName)
|
||||||
case madmin.LDAPIDPCfg:
|
case madmin.LDAPIDPCfg:
|
||||||
cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
|
cfgInfos, err = globalIAMSys.LDAPConfig.GetConfigInfo(cfg, cfgName)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
|
if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
|
||||||
@@ -334,11 +330,9 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
|||||||
//
|
//
|
||||||
// DELETE <admin-prefix>/idp-cfg/openid/dex_test
|
// DELETE <admin-prefix>/idp-cfg/openid/dex_test
|
||||||
func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "DeleteIdentityProviderCfg")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -355,7 +349,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
|||||||
switch idpCfgType {
|
switch idpCfgType {
|
||||||
case madmin.OpenidIDPCfg:
|
case madmin.OpenidIDPCfg:
|
||||||
subSys = config.IdentityOpenIDSubSys
|
subSys = config.IdentityOpenIDSubSys
|
||||||
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
|
cfgInfos, err := globalIAMSys.OpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||||
@@ -380,7 +374,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
|||||||
}
|
}
|
||||||
case madmin.LDAPIDPCfg:
|
case madmin.LDAPIDPCfg:
|
||||||
subSys = config.IdentityLDAPSubSys
|
subSys = config.IdentityLDAPSubSys
|
||||||
cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
|
cfgInfos, err := globalIAMSys.LDAPConfig.GetConfigInfo(cfgCopy, cfgName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
if errors.Is(err, openid.ErrProviderConfigNotFound) {
|
||||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
|
||||||
@@ -422,7 +416,17 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
|
|||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = validateConfig(cfg, subSys); err != nil {
|
if err = validateConfig(ctx, cfg, subSys); err != nil {
|
||||||
|
|
||||||
|
var validationErr ldap.Validation
|
||||||
|
if errors.As(err, &validationErr) {
|
||||||
|
// If we got an LDAP validation error, we need to send appropriate
|
||||||
|
// error message back to client (likely mc).
|
||||||
|
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
|
||||||
|
validationErr.FormatError(), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
+275
-11
@@ -19,13 +19,17 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||||
@@ -45,14 +49,12 @@ import (
|
|||||||
//
|
//
|
||||||
// When all query parameters are omitted, returns mappings for all policies.
|
// When all query parameters are omitted, returns mappings for all policies.
|
||||||
func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ListLDAPPolicyMappingEntities")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
// Check authorization.
|
// Check authorization.
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r,
|
objectAPI, cred := validateAdminReq(ctx, w, r,
|
||||||
iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
|
policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -94,13 +96,11 @@ func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r
|
|||||||
//
|
//
|
||||||
// POST <admin-prefix>/idp/ldap/policy/{operation}
|
// POST <admin-prefix>/idp/ldap/policy/{operation}
|
||||||
func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "AttachDetachPolicyLDAP")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
// Check authorization.
|
// Check authorization.
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction)
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -150,7 +150,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Call IAM subsystem
|
// Call IAM subsystem
|
||||||
updatedAt, addedOrRemoved, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
|
updatedAt, addedOrRemoved, _, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -179,3 +179,267 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
|
|||||||
|
|
||||||
writeSuccessResponseJSON(w, encryptedData)
|
writeSuccessResponseJSON(w, encryptedData)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddServiceAccountLDAP adds a new service account for provided LDAP username or DN
|
||||||
|
//
|
||||||
|
// PUT /minio/admin/v3/idp/ldap/add-service-account
|
||||||
|
func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
|
||||||
|
if APIError.Code != "" {
|
||||||
|
writeErrorResponseJSON(ctx, w, APIError, r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// fail if ldap is not enabled
|
||||||
|
if !globalIAMSys.LDAPConfig.Enabled() {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("LDAP not enabled")), r.URL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the user for the request sender (as it may be sent via a service
|
||||||
|
// account or STS account):
|
||||||
|
requestorUser := cred.AccessKey
|
||||||
|
requestorParentUser := cred.AccessKey
|
||||||
|
requestorGroups := cred.Groups
|
||||||
|
requestorIsDerivedCredential := false
|
||||||
|
if cred.IsServiceAccount() || cred.IsTemp() {
|
||||||
|
requestorParentUser = cred.ParentUser
|
||||||
|
requestorIsDerivedCredential = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if we are creating svc account for request sender.
|
||||||
|
isSvcAccForRequestor := false
|
||||||
|
if targetUser == requestorUser || targetUser == requestorParentUser {
|
||||||
|
isSvcAccForRequestor = true
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
targetGroups []string
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
// If we are creating svc account for request sender, ensure
|
||||||
|
// that targetUser is a real user (i.e. not derived
|
||||||
|
// credentials).
|
||||||
|
if isSvcAccForRequestor {
|
||||||
|
if requestorIsDerivedCredential {
|
||||||
|
if requestorParentUser == "" {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
|
||||||
|
errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
targetUser = requestorParentUser
|
||||||
|
}
|
||||||
|
targetGroups = requestorGroups
|
||||||
|
|
||||||
|
// Deny if the target user is not LDAP
|
||||||
|
isLDAP, err := globalIAMSys.LDAPConfig.DoesUsernameExist(targetUser)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if isLDAP == "" {
|
||||||
|
err := errors.New("Specified user does not exist on LDAP server")
|
||||||
|
APIErr := errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err)
|
||||||
|
writeErrorResponseJSON(ctx, w, APIErr, r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||||
|
// it is associated with the LDAP/OIDC user properly.
|
||||||
|
for k, v := range cred.Claims {
|
||||||
|
if k == expClaim {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
opts.claims[k] = v
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
isDN := globalIAMSys.LDAPConfig.IsLDAPUserDN(targetUser)
|
||||||
|
|
||||||
|
opts.claims[ldapUserN] = targetUser // simple username
|
||||||
|
targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
|
||||||
|
if err != nil {
|
||||||
|
// if not found, check if DN
|
||||||
|
if strings.Contains(err.Error(), "not found") && isDN {
|
||||||
|
// warn user that DNs are not allowed
|
||||||
|
err = fmt.Errorf("Must use short username to add service account. %w", err)
|
||||||
|
}
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
opts.claims[ldapUser] = targetUser // DN
|
||||||
|
}
|
||||||
|
|
||||||
|
newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
createResp := madmin.AddServiceAccountResp{
|
||||||
|
Credentials: madmin.Credentials{
|
||||||
|
AccessKey: newCred.AccessKey,
|
||||||
|
SecretKey: newCred.SecretKey,
|
||||||
|
Expiration: newCred.Expiration,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(createResp)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeSuccessResponseJSON(w, encryptedData)
|
||||||
|
|
||||||
|
// Call hook for cluster-replication if the service account is not for a
|
||||||
|
// root user.
|
||||||
|
if newCred.ParentUser != globalActiveCred.AccessKey {
|
||||||
|
logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
|
||||||
|
Type: madmin.SRIAMItemSvcAcc,
|
||||||
|
SvcAccChange: &madmin.SRSvcAccChange{
|
||||||
|
Create: &madmin.SRSvcAccCreate{
|
||||||
|
Parent: newCred.ParentUser,
|
||||||
|
AccessKey: newCred.AccessKey,
|
||||||
|
SecretKey: newCred.SecretKey,
|
||||||
|
Groups: newCred.Groups,
|
||||||
|
Name: newCred.Name,
|
||||||
|
Description: newCred.Description,
|
||||||
|
Claims: opts.claims,
|
||||||
|
SessionPolicy: createReq.Policy,
|
||||||
|
Status: auth.AccountOn,
|
||||||
|
Expiration: createReq.Expiration,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
UpdatedAt: updatedAt,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAccessKeysLDAP - GET /minio/admin/v3/idp/ldap/list-access-keys
|
||||||
|
func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
|
||||||
|
// Get current object layer instance.
|
||||||
|
objectAPI := newObjectLayerFn()
|
||||||
|
if objectAPI == nil || globalNotificationSys == nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||||
|
if s3Err != ErrNone {
|
||||||
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
userDN := r.Form.Get("userDN")
|
||||||
|
|
||||||
|
// If listing is requested for a specific user (who is not the request
|
||||||
|
// sender), check that the user has permissions.
|
||||||
|
if userDN != "" && userDN != cred.ParentUser {
|
||||||
|
if !globalIAMSys.IsAllowed(policy.Args{
|
||||||
|
AccountName: cred.AccessKey,
|
||||||
|
Groups: cred.Groups,
|
||||||
|
Action: policy.ListServiceAccountsAdminAction,
|
||||||
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
|
IsOwner: owner,
|
||||||
|
Claims: cred.Claims,
|
||||||
|
}) {
|
||||||
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if !globalIAMSys.IsAllowed(policy.Args{
|
||||||
|
AccountName: cred.AccessKey,
|
||||||
|
Groups: cred.Groups,
|
||||||
|
Action: policy.ListServiceAccountsAdminAction,
|
||||||
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
|
IsOwner: owner,
|
||||||
|
Claims: cred.Claims,
|
||||||
|
DenyOnly: true,
|
||||||
|
}) {
|
||||||
|
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
userDN = cred.AccessKey
|
||||||
|
if cred.ParentUser != "" {
|
||||||
|
userDN = cred.ParentUser
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
targetAccount, err := globalIAMSys.LDAPConfig.DoesUsernameExist(userDN)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
} else if userDN == "" {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
listType := r.Form.Get("listType")
|
||||||
|
if listType != "sts-only" && listType != "svcacc-only" && listType != "" {
|
||||||
|
// default to both
|
||||||
|
listType = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var serviceAccounts []auth.Credentials
|
||||||
|
var stsKeys []auth.Credentials
|
||||||
|
|
||||||
|
if listType == "" || listType == "sts-only" {
|
||||||
|
stsKeys, err = globalIAMSys.ListSTSAccounts(ctx, targetAccount)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if listType == "" || listType == "svcacc-only" {
|
||||||
|
serviceAccounts, err = globalIAMSys.ListServiceAccounts(ctx, targetAccount)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var serviceAccountList []madmin.ServiceAccountInfo
|
||||||
|
var stsKeyList []madmin.ServiceAccountInfo
|
||||||
|
|
||||||
|
for _, svc := range serviceAccounts {
|
||||||
|
expiryTime := svc.Expiration
|
||||||
|
serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
|
||||||
|
AccessKey: svc.AccessKey,
|
||||||
|
Expiration: &expiryTime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, sts := range stsKeys {
|
||||||
|
expiryTime := sts.Expiration
|
||||||
|
stsKeyList = append(stsKeyList, madmin.ServiceAccountInfo{
|
||||||
|
AccessKey: sts.AccessKey,
|
||||||
|
Expiration: &expiryTime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
listResp := madmin.ListAccessKeysLDAPResp{
|
||||||
|
ServiceAccounts: serviceAccountList,
|
||||||
|
STSKeys: stsKeyList,
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.Marshal(listResp)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeSuccessResponseJSON(w, encryptedData)
|
||||||
|
}
|
||||||
|
|||||||
+51
-31
@@ -22,24 +22,23 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is aleady in progress")
|
errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is already in progress")
|
||||||
errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
|
errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
|
||||||
)
|
)
|
||||||
|
|
||||||
func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "StartDecommission")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -68,17 +67,29 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
|||||||
|
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
v := vars["pool"]
|
v := vars["pool"]
|
||||||
|
byID := vars["by-id"] == "true"
|
||||||
|
|
||||||
pools := strings.Split(v, ",")
|
pools := strings.Split(v, ",")
|
||||||
poolIndices := make([]int, 0, len(pools))
|
poolIndices := make([]int, 0, len(pools))
|
||||||
|
|
||||||
for _, pool := range pools {
|
for _, pool := range pools {
|
||||||
idx := globalEndpoints.GetPoolIdx(pool)
|
var idx int
|
||||||
|
if byID {
|
||||||
|
var err error
|
||||||
|
idx, err = strconv.Atoi(pool)
|
||||||
|
if err != nil {
|
||||||
|
// We didn't find any matching pools, invalid input
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
idx = globalEndpoints.GetPoolIdx(pool)
|
||||||
if idx == -1 {
|
if idx == -1 {
|
||||||
// We didn't find any matching pools, invalid input
|
// We didn't find any matching pools, invalid input
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
}
|
||||||
var pool *erasureSets
|
var pool *erasureSets
|
||||||
for pidx := range z.serverPools {
|
for pidx := range z.serverPools {
|
||||||
if pidx == idx {
|
if pidx == idx {
|
||||||
@@ -95,7 +106,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
|||||||
poolIndices = append(poolIndices, idx)
|
poolIndices = append(poolIndices, idx)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
|
if len(poolIndices) > 0 && !globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
|
||||||
ep := globalEndpoints[poolIndices[0]].Endpoints[0]
|
ep := globalEndpoints[poolIndices[0]].Endpoints[0]
|
||||||
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
for nodeIdx, proxyEp := range globalProxyEndpoints {
|
||||||
if proxyEp.Endpoint.Host == ep.Host {
|
if proxyEp.Endpoint.Host == ep.Host {
|
||||||
@@ -113,11 +124,9 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "CancelDecommission")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -136,8 +145,17 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
|
|||||||
|
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
v := vars["pool"]
|
v := vars["pool"]
|
||||||
|
byID := vars["by-id"] == "true"
|
||||||
|
idx := -1
|
||||||
|
|
||||||
|
if byID {
|
||||||
|
if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
|
||||||
|
idx = i
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
idx = globalEndpoints.GetPoolIdx(v)
|
||||||
|
}
|
||||||
|
|
||||||
idx := globalEndpoints.GetPoolIdx(v)
|
|
||||||
if idx == -1 {
|
if idx == -1 {
|
||||||
// We didn't find any matching pools, invalid input
|
// We didn't find any matching pools, invalid input
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||||
@@ -161,11 +179,9 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "StatusPool")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -184,8 +200,17 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
vars := mux.Vars(r)
|
vars := mux.Vars(r)
|
||||||
v := vars["pool"]
|
v := vars["pool"]
|
||||||
|
byID := vars["by-id"] == "true"
|
||||||
|
idx := -1
|
||||||
|
|
||||||
|
if byID {
|
||||||
|
if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
|
||||||
|
idx = i
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
idx = globalEndpoints.GetPoolIdx(v)
|
||||||
|
}
|
||||||
|
|
||||||
idx := globalEndpoints.GetPoolIdx(v)
|
|
||||||
if idx == -1 {
|
if idx == -1 {
|
||||||
apiErr := toAdminAPIErr(ctx, errInvalidArgument)
|
apiErr := toAdminAPIErr(ctx, errInvalidArgument)
|
||||||
apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
|
apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
|
||||||
@@ -204,11 +229,9 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "ListPools")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -239,10 +262,9 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "RebalanceStart")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -311,10 +333,9 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "RebalanceStatus")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -352,10 +373,9 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "RebalanceStop")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,27 +20,28 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/gob"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/madmin-go/v3"
|
||||||
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/pkg/bucket/policy"
|
"github.com/minio/mux"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
||||||
func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationAdd")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -51,7 +52,8 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
|
opts := getSRAddOptions(r)
|
||||||
|
status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.LogIf(ctx, err)
|
logger.LogIf(ctx, err)
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
@@ -67,16 +69,19 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
|
|||||||
writeSuccessResponseJSON(w, body)
|
writeSuccessResponseJSON(w, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getSRAddOptions(r *http.Request) (opts madmin.SRAddOptions) {
|
||||||
|
opts.ReplicateILMExpiry = r.Form.Get("replicateILMExpiry") == "true"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
|
// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
|
||||||
//
|
//
|
||||||
// used internally to tell current cluster to enable SR with
|
// used internally to tell current cluster to enable SR with
|
||||||
// the provided peer clusters and service account.
|
// the provided peer clusters and service account.
|
||||||
func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SRPeerJoin")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -96,11 +101,9 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
|
// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
|
||||||
func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SRPeerBucketOps")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -145,11 +148,9 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
|
|||||||
|
|
||||||
// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
|
// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
|
||||||
func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SRPeerReplicateIAMItem")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -168,7 +169,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
|
|||||||
if item.Policy == nil {
|
if item.Policy == nil {
|
||||||
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
|
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
|
||||||
} else {
|
} else {
|
||||||
policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
|
policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy))
|
||||||
if perr != nil {
|
if perr != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
|
||||||
return
|
return
|
||||||
@@ -197,13 +198,11 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
|
// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/peer/bucket-meta
|
||||||
func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SRPeerReplicateBucketItem")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -214,15 +213,20 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if item.Bucket == "" {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
switch item.Type {
|
switch item.Type {
|
||||||
default:
|
default:
|
||||||
err = errSRInvalidRequest(errInvalidArgument)
|
err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
|
||||||
case madmin.SRBucketMetaTypePolicy:
|
case madmin.SRBucketMetaTypePolicy:
|
||||||
if item.Policy == nil {
|
if item.Policy == nil {
|
||||||
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
|
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
|
||||||
} else {
|
} else {
|
||||||
bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
|
bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket)
|
||||||
if berr != nil {
|
if berr != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
|
||||||
return
|
return
|
||||||
@@ -243,7 +247,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
|
if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -255,6 +259,8 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
|||||||
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
|
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
|
||||||
case madmin.SRBucketMetaTypeSSEConfig:
|
case madmin.SRBucketMetaTypeSSEConfig:
|
||||||
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
|
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
|
||||||
|
case madmin.SRBucketMetaLCConfig:
|
||||||
|
err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.LogIf(ctx, err)
|
logger.LogIf(ctx, err)
|
||||||
@@ -265,11 +271,9 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
|||||||
|
|
||||||
// SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
|
// SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
|
||||||
func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationInfo")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -287,11 +291,9 @@ func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Req
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -326,11 +328,9 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
|
|||||||
|
|
||||||
// SiteReplicationStatus - GET /minio/admin/v3/site-replication/status
|
// SiteReplicationStatus - GET /minio/admin/v3/site-replication/status
|
||||||
func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationStatus")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -342,6 +342,7 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
|
|||||||
opts.Users = true
|
opts.Users = true
|
||||||
opts.Policies = true
|
opts.Policies = true
|
||||||
opts.Groups = true
|
opts.Groups = true
|
||||||
|
opts.ILMExpiryRules = true
|
||||||
}
|
}
|
||||||
info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
|
info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -357,11 +358,9 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
|
|||||||
|
|
||||||
// SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo
|
// SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo
|
||||||
func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationMetaInfo")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -381,10 +380,9 @@ func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http
|
|||||||
|
|
||||||
// SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit
|
// SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit
|
||||||
func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationEdit")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -394,7 +392,9 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
|
|||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site)
|
|
||||||
|
opts := getSREditOptions(r)
|
||||||
|
status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.LogIf(ctx, err)
|
logger.LogIf(ctx, err)
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
@@ -409,14 +409,19 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
|
|||||||
writeSuccessResponseJSON(w, body)
|
writeSuccessResponseJSON(w, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getSREditOptions(r *http.Request) (opts madmin.SREditOptions) {
|
||||||
|
opts.DisableILMExpiryReplication = r.Form.Get("disableILMExpiryReplication") == "true"
|
||||||
|
opts.EnableILMExpiryReplication = r.Form.Get("enableILMExpiryReplication") == "true"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
|
// SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
|
||||||
//
|
//
|
||||||
// used internally to tell current cluster to update endpoint for peer
|
// used internally to tell current cluster to update endpoint for peer
|
||||||
func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SRPeerEdit")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -434,25 +439,49 @@ func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SRStateEdit - PUT /minio/admin/v3/site-replication/state/edit
|
||||||
|
//
|
||||||
|
// used internally to tell current cluster to update site replication state
|
||||||
|
func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
|
||||||
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||||
|
if objectAPI == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var state madmin.SRStateEditReq
|
||||||
|
if err := parseJSONBody(ctx, r.Body, &state, ""); err != nil {
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
|
||||||
|
logger.LogIf(ctx, err)
|
||||||
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
|
func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
|
||||||
q := r.Form
|
q := r.Form
|
||||||
opts.Buckets = q.Get("buckets") == "true"
|
opts.Buckets = q.Get("buckets") == "true"
|
||||||
opts.Policies = q.Get("policies") == "true"
|
opts.Policies = q.Get("policies") == "true"
|
||||||
opts.Groups = q.Get("groups") == "true"
|
opts.Groups = q.Get("groups") == "true"
|
||||||
opts.Users = q.Get("users") == "true"
|
opts.Users = q.Get("users") == "true"
|
||||||
|
opts.ILMExpiryRules = q.Get("ilm-expiry-rules") == "true"
|
||||||
|
opts.PeerState = q.Get("peer-state") == "true"
|
||||||
opts.Entity = madmin.GetSREntityType(q.Get("entity"))
|
opts.Entity = madmin.GetSREntityType(q.Get("entity"))
|
||||||
opts.EntityValue = q.Get("entityvalue")
|
opts.EntityValue = q.Get("entityvalue")
|
||||||
opts.ShowDeleted = q.Get("showDeleted") == "true"
|
opts.ShowDeleted = q.Get("showDeleted") == "true"
|
||||||
|
opts.Metrics = q.Get("metrics") == "true"
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// SiteReplicationRemove - PUT /minio/admin/v3/site-replication/remove
|
// SiteReplicationRemove - PUT /minio/admin/v3/site-replication/remove
|
||||||
func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationRemove")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -481,10 +510,9 @@ func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.R
|
|||||||
//
|
//
|
||||||
// used internally to tell current cluster to update endpoint for peer
|
// used internally to tell current cluster to update endpoint for peer
|
||||||
func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SRPeerRemove")
|
ctx := r.Context()
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -504,11 +532,9 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op
|
// SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op
|
||||||
func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
|
func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := newContext(r, w, "SiteReplicationResyncOp")
|
ctx := r.Context()
|
||||||
|
|
||||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationResyncAction)
|
||||||
|
|
||||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationResyncAction)
|
|
||||||
if objectAPI == nil {
|
if objectAPI == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -543,3 +569,45 @@ func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http
|
|||||||
}
|
}
|
||||||
writeSuccessResponseJSON(w, body)
|
writeSuccessResponseJSON(w, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SiteReplicationDevNull - everything goes to io.Discard
|
||||||
|
// [POST] /minio/admin/v3/site-replication/devnull
|
||||||
|
func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx := r.Context()
|
||||||
|
|
||||||
|
globalSiteNetPerfRX.Connect()
|
||||||
|
defer globalSiteNetPerfRX.Disconnect()
|
||||||
|
|
||||||
|
connectTime := time.Now()
|
||||||
|
for {
|
||||||
|
n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
|
||||||
|
atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
|
||||||
|
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||||
|
// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
|
||||||
|
// would mean the network is not stable. Logging here will help in debugging network issues.
|
||||||
|
if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
|
||||||
|
logger.LogIf(ctx, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
} else {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SiteReplicationNetPerf - everything goes to io.Discard
|
||||||
|
// [POST] /minio/admin/v3/site-replication/netperf
|
||||||
|
func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.Request) {
|
||||||
|
durationStr := r.Form.Get(peerRESTDuration)
|
||||||
|
duration, _ := time.ParseDuration(durationStr)
|
||||||
|
if duration < globalNetPerfMinDuration {
|
||||||
|
duration = globalNetPerfMinDuration
|
||||||
|
}
|
||||||
|
result := siteNetperf(r.Context(), duration)
|
||||||
|
logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
|
||||||
|
}
|
||||||
|
|||||||
@@ -27,12 +27,12 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
minio "github.com/minio/minio-go/v7"
|
minio "github.com/minio/minio-go/v7"
|
||||||
|
"github.com/minio/pkg/v2/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||||
@@ -129,18 +129,21 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
|
|||||||
secretKeys[i] = secretKey
|
secretKeys[i] = secretKey
|
||||||
}
|
}
|
||||||
|
|
||||||
wg := sync.WaitGroup{}
|
g := errgroup.Group{}
|
||||||
for i := 0; i < userCount; i++ {
|
for i := 0; i < userCount; i++ {
|
||||||
wg.Add(1)
|
g.Go(func(i int) func() error {
|
||||||
go func(i int) {
|
return func() error {
|
||||||
defer wg.Done()
|
|
||||||
uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
|
uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
|
||||||
err := s.adm.RemoveUser(ctx, accessKeys[i])
|
err := s.adm.RemoveUser(ctx, accessKeys[i])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.Fatalf("unable to remove user: %v", err)
|
return err
|
||||||
}
|
}
|
||||||
c.mustNotListObjects(ctx, uClient, bucket)
|
c.mustNotListObjects(ctx, uClient, bucket)
|
||||||
}(i)
|
return nil
|
||||||
|
}
|
||||||
|
}(i), i)
|
||||||
|
}
|
||||||
|
if errs := g.Wait(); len(errs) > 0 {
|
||||||
|
c.Fatalf("unable to remove users: %v", errs)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
|
||||||
}
|
}
|
||||||
|
|||||||
+409
-529
File diff suppressed because it is too large
Load Diff
@@ -27,20 +27,19 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
cr "github.com/minio/minio-go/v7/pkg/credentials"
|
|
||||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
"github.com/minio/minio-go/v7/pkg/signer"
|
"github.com/minio/minio-go/v7/pkg/signer"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/pkg/v2/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -122,7 +121,7 @@ var iamTestSuites = func() []*TestSuiteIAM {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
EnvTestEtcdBackend = "ETCD_SERVER"
|
EnvTestEtcdBackend = "_MINIO_ETCD_TEST_SERVER"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
|
func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
|
||||||
@@ -145,7 +144,7 @@ func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
|
|||||||
func (s *TestSuiteIAM) SetUpSuite(c *check) {
|
func (s *TestSuiteIAM) SetUpSuite(c *check) {
|
||||||
// If etcd backend is specified and etcd server is not present, the test
|
// If etcd backend is specified and etcd server is not present, the test
|
||||||
// is skipped.
|
// is skipped.
|
||||||
etcdServer := os.Getenv(EnvTestEtcdBackend)
|
etcdServer := env.Get(EnvTestEtcdBackend, "")
|
||||||
if s.withEtcdBackend && etcdServer == "" {
|
if s.withEtcdBackend && etcdServer == "" {
|
||||||
c.Skip("Skipping etcd backend IAM test as no etcd server is configured.")
|
c.Skip("Skipping etcd backend IAM test as no etcd server is configured.")
|
||||||
}
|
}
|
||||||
@@ -207,6 +206,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
|||||||
suite.TestCannedPolicies(c)
|
suite.TestCannedPolicies(c)
|
||||||
suite.TestGroupAddRemove(c)
|
suite.TestGroupAddRemove(c)
|
||||||
suite.TestServiceAccountOpsByAdmin(c)
|
suite.TestServiceAccountOpsByAdmin(c)
|
||||||
|
suite.TestServiceAccountPrivilegeEscalationBug(c)
|
||||||
suite.TestServiceAccountOpsByUser(c)
|
suite.TestServiceAccountOpsByUser(c)
|
||||||
suite.TestAddServiceAccountPerms(c)
|
suite.TestAddServiceAccountPerms(c)
|
||||||
suite.TearDownSuite(c)
|
suite.TearDownSuite(c)
|
||||||
@@ -825,7 +825,7 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
|
|||||||
if set.CreateStringSet(groups...).Contains(group) {
|
if set.CreateStringSet(groups...).Contains(group) {
|
||||||
c.Fatalf("created group still present!")
|
c.Fatalf("created group still present!")
|
||||||
}
|
}
|
||||||
groupInfo, err = s.adm.GetGroupDescription(ctx, group)
|
_, err = s.adm.GetGroupDescription(ctx, group)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
c.Fatalf("group appears to exist")
|
c.Fatalf("group appears to exist")
|
||||||
}
|
}
|
||||||
@@ -877,7 +877,7 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
|
|||||||
|
|
||||||
// Create an madmin client with user creds
|
// Create an madmin client with user creds
|
||||||
userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||||
Creds: cr.NewStaticV4(accessKey, secretKey, ""),
|
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||||
Secure: s.secure,
|
Secure: s.secure,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -897,14 +897,6 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
|
|||||||
// 3. Check S3 access
|
// 3. Check S3 access
|
||||||
c.assertSvcAccS3Access(ctx, s, cr, bucket)
|
c.assertSvcAccS3Access(ctx, s, cr, bucket)
|
||||||
|
|
||||||
// 4. Check that svc account can restrict the policy, and that the
|
|
||||||
// session policy can be updated.
|
|
||||||
c.assertSvcAccSessionPolicyUpdate(ctx, s, userAdmClient, accessKey, bucket)
|
|
||||||
|
|
||||||
// 4. Check that service account's secret key and account status can be
|
|
||||||
// updated.
|
|
||||||
c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, accessKey, bucket)
|
|
||||||
|
|
||||||
// 5. Check that service account can be deleted.
|
// 5. Check that service account can be deleted.
|
||||||
c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
|
c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
|
||||||
|
|
||||||
@@ -980,13 +972,102 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
|
|||||||
c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
|
c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *TestSuiteIAM) TestServiceAccountPrivilegeEscalationBug(c *check) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := s.client.MakeBucket(ctx, "public", minio.MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("bucket creat error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.client.MakeBucket(ctx, "private", minio.MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("bucket creat error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
pubPolicyBytes := []byte(`{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"s3:*"
|
||||||
|
],
|
||||||
|
"Resource": [
|
||||||
|
"arn:aws:s3:::public",
|
||||||
|
"arn:aws:s3:::public/*"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
fullS3PolicyBytes := []byte(`{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": [
|
||||||
|
"s3:*"
|
||||||
|
],
|
||||||
|
"Resource": [
|
||||||
|
"arn:aws:s3:::*"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
`)
|
||||||
|
|
||||||
|
// Create a service account for the root user.
|
||||||
|
cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||||
|
TargetUser: globalActiveCred.AccessKey,
|
||||||
|
Policy: pubPolicyBytes,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("admin should be able to create service account for themselves %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
|
||||||
|
|
||||||
|
// Check that the service account can access the public bucket.
|
||||||
|
buckets, err := svcClient.ListBuckets(ctx)
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("err fetching buckets %s", err)
|
||||||
|
}
|
||||||
|
if len(buckets) != 1 || buckets[0].Name != "public" {
|
||||||
|
c.Fatalf("service account should only have access to public bucket")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create an madmin client with the service account creds.
|
||||||
|
svcAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||||
|
Creds: credentials.NewStaticV4(cr.AccessKey, cr.SecretKey, ""),
|
||||||
|
Secure: s.secure,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("Err creating svcacct admin client: %v", err)
|
||||||
|
}
|
||||||
|
svcAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||||
|
|
||||||
|
// Attempt to update the policy on the service account.
|
||||||
|
err = svcAdmClient.UpdateServiceAccount(ctx, cr.AccessKey,
|
||||||
|
madmin.UpdateServiceAccountReq{
|
||||||
|
NewPolicy: fullS3PolicyBytes,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
c.Fatalf("service account should not be able to update policy on itself")
|
||||||
|
} else if !strings.Contains(err.Error(), "Access Denied") {
|
||||||
|
c.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
|
func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
pluginEndpoint := os.Getenv("POLICY_PLUGIN_ENDPOINT")
|
pluginEndpoint := env.Get("_MINIO_POLICY_PLUGIN_ENDPOINT", "")
|
||||||
if pluginEndpoint == "" {
|
if pluginEndpoint == "" {
|
||||||
c.Skip("POLICY_PLUGIN_ENDPOINT not given - skipping.")
|
c.Skip("_MINIO_POLICY_PLUGIN_ENDPOINT not given - skipping.")
|
||||||
}
|
}
|
||||||
|
|
||||||
configCmds := []string{
|
configCmds := []string{
|
||||||
@@ -1072,7 +1153,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
|
|||||||
|
|
||||||
// Create an madmin client with user creds
|
// Create an madmin client with user creds
|
||||||
userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||||
Creds: cr.NewStaticV4(accessKey, secretKey, ""),
|
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||||
Secure: s.secure,
|
Secure: s.secure,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1261,6 +1342,52 @@ func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *check) mustNotDelete(ctx context.Context, client *minio.Client, bucket string, vid string) {
|
||||||
|
c.Helper()
|
||||||
|
|
||||||
|
err := client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{VersionID: vid})
|
||||||
|
if err == nil {
|
||||||
|
c.Fatalf("user must not be allowed to delete")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatal("user must be able to create delete marker")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *check) mustDownload(ctx context.Context, client *minio.Client, bucket string) {
|
||||||
|
c.Helper()
|
||||||
|
rd, err := client.GetObject(ctx, bucket, "some-object", minio.GetObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("download did not succeed got %#v", err)
|
||||||
|
}
|
||||||
|
if _, err = io.Copy(io.Discard, rd); err != nil {
|
||||||
|
c.Fatalf("download did not succeed got %#v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *check) mustUploadReturnVersions(ctx context.Context, client *minio.Client, bucket string) []string {
|
||||||
|
c.Helper()
|
||||||
|
versions := []string{}
|
||||||
|
for i := 0; i < 5; i++ {
|
||||||
|
ui, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("upload did not succeed got %#v", err)
|
||||||
|
}
|
||||||
|
versions = append(versions, ui.VersionID)
|
||||||
|
}
|
||||||
|
return versions
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *check) mustUpload(ctx context.Context, client *minio.Client, bucket string) {
|
||||||
|
c.Helper()
|
||||||
|
_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("upload did not succeed got %#v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
|
func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
|
||||||
c.Helper()
|
c.Helper()
|
||||||
_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
|
_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
|
||||||
@@ -1283,7 +1410,11 @@ func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *ma
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
c.Fatalf("unable to list svc accounts: %v", err)
|
c.Fatalf("unable to list svc accounts: %v", err)
|
||||||
}
|
}
|
||||||
if !set.CreateStringSet(listResp.Accounts...).Contains(svcAK) {
|
var accessKeys []string
|
||||||
|
for _, item := range listResp.Accounts {
|
||||||
|
accessKeys = append(accessKeys, item.AccessKey)
|
||||||
|
}
|
||||||
|
if !set.CreateStringSet(accessKeys...).Contains(svcAK) {
|
||||||
c.Fatalf("service account did not appear in listing!")
|
c.Fatalf("service account did not appear in listing!")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+776
-192
File diff suppressed because it is too large
Load Diff
+156
-4
@@ -21,15 +21,17 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
)
|
)
|
||||||
@@ -71,7 +73,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
|
|||||||
// Initialize boot time
|
// Initialize boot time
|
||||||
globalBootTime = UTCNow()
|
globalBootTime = UTCNow()
|
||||||
|
|
||||||
globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
|
globalEndpoints = mustGetPoolEndpoints(0, erasureDirs...)
|
||||||
|
|
||||||
initAllSubsystems(ctx)
|
initAllSubsystems(ctx)
|
||||||
|
|
||||||
@@ -106,7 +108,7 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
endpoints := mustGetPoolEndpoints(erasureDirs...)
|
endpoints := mustGetPoolEndpoints(0, erasureDirs...)
|
||||||
globalPolicySys = NewPolicySys()
|
globalPolicySys = NewPolicySys()
|
||||||
objLayer, err := newErasureServerPools(ctx, endpoints)
|
objLayer, err := newErasureServerPools(ctx, endpoints)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -166,6 +168,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
|
|||||||
func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
|
func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
|
||||||
queryVal := url.Values{}
|
queryVal := url.Values{}
|
||||||
queryVal.Set("action", string(cmd.toServiceAction()))
|
queryVal.Set("action", string(cmd.toServiceAction()))
|
||||||
|
queryVal.Set("type", "2")
|
||||||
resource := adminPathPrefix + adminAPIVersionPrefix + "/service?" + queryVal.Encode()
|
resource := adminPathPrefix + adminAPIVersionPrefix + "/service?" + queryVal.Encode()
|
||||||
req, err := newTestRequest(http.MethodPost, resource, 0, nil)
|
req, err := newTestRequest(http.MethodPost, resource, 0, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -218,12 +221,18 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
|
|||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
adminTestBed.router.ServeHTTP(rec, req)
|
adminTestBed.router.ServeHTTP(rec, req)
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
|
||||||
resp, _ := io.ReadAll(rec.Body)
|
resp, _ := io.ReadAll(rec.Body)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
|
t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
|
||||||
http.StatusOK, rec.Code, string(resp))
|
http.StatusOK, rec.Code, string(resp))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
result := &serviceResult{}
|
||||||
|
if err := json.Unmarshal(resp, result); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
_ = result
|
||||||
|
|
||||||
// Wait until testServiceSignalReceiver() called in a goroutine quits.
|
// Wait until testServiceSignalReceiver() called in a goroutine quits.
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
@@ -380,3 +389,146 @@ func TestExtractHealInitParams(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type byResourceUID struct{ madmin.LockEntries }
|
||||||
|
|
||||||
|
func (b byResourceUID) Less(i, j int) bool {
|
||||||
|
toUniqLock := func(entry madmin.LockEntry) string {
|
||||||
|
return fmt.Sprintf("%s/%s", entry.Resource, entry.ID)
|
||||||
|
}
|
||||||
|
return toUniqLock(b.LockEntries[i]) < toUniqLock(b.LockEntries[j])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTopLockEntries(t *testing.T) {
|
||||||
|
locksHeld := make(map[string][]lockRequesterInfo)
|
||||||
|
var owners []string
|
||||||
|
for i := 0; i < 4; i++ {
|
||||||
|
owners = append(owners, fmt.Sprintf("node-%d", i))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simulate DeleteObjects of 10 objects in a single request. i.e same lock
|
||||||
|
// request UID, but 10 different resource names associated with it.
|
||||||
|
var lris []lockRequesterInfo
|
||||||
|
uuid := mustGetUUID()
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
resource := fmt.Sprintf("bucket/delete-object-%d", i)
|
||||||
|
lri := lockRequesterInfo{
|
||||||
|
Name: resource,
|
||||||
|
Writer: true,
|
||||||
|
UID: uuid,
|
||||||
|
Owner: owners[i%len(owners)],
|
||||||
|
Group: true,
|
||||||
|
Quorum: 3,
|
||||||
|
}
|
||||||
|
lris = append(lris, lri)
|
||||||
|
locksHeld[resource] = []lockRequesterInfo{lri}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add a few concurrent read locks to the mix
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
resource := fmt.Sprintf("bucket/get-object-%d", i)
|
||||||
|
lri := lockRequesterInfo{
|
||||||
|
Name: resource,
|
||||||
|
UID: mustGetUUID(),
|
||||||
|
Owner: owners[i%len(owners)],
|
||||||
|
Quorum: 2,
|
||||||
|
}
|
||||||
|
lris = append(lris, lri)
|
||||||
|
locksHeld[resource] = append(locksHeld[resource], lri)
|
||||||
|
// concurrent read lock, same resource different uid
|
||||||
|
lri.UID = mustGetUUID()
|
||||||
|
lris = append(lris, lri)
|
||||||
|
locksHeld[resource] = append(locksHeld[resource], lri)
|
||||||
|
}
|
||||||
|
|
||||||
|
var peerLocks []*PeerLocks
|
||||||
|
for _, owner := range owners {
|
||||||
|
peerLocks = append(peerLocks, &PeerLocks{
|
||||||
|
Addr: owner,
|
||||||
|
Locks: locksHeld,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
var exp madmin.LockEntries
|
||||||
|
for _, lri := range lris {
|
||||||
|
lockType := func(lri lockRequesterInfo) string {
|
||||||
|
if lri.Writer {
|
||||||
|
return "WRITE"
|
||||||
|
}
|
||||||
|
return "READ"
|
||||||
|
}
|
||||||
|
exp = append(exp, madmin.LockEntry{
|
||||||
|
Resource: lri.Name,
|
||||||
|
Type: lockType(lri),
|
||||||
|
ServerList: owners,
|
||||||
|
Owner: lri.Owner,
|
||||||
|
ID: lri.UID,
|
||||||
|
Quorum: lri.Quorum,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
peerLocks []*PeerLocks
|
||||||
|
expected madmin.LockEntries
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
peerLocks: peerLocks,
|
||||||
|
expected: exp,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// printEntries := func(entries madmin.LockEntries) {
|
||||||
|
// for i, entry := range entries {
|
||||||
|
// fmt.Printf("%d: %s %s %s %s %v %d\n", i, entry.Resource, entry.ID, entry.Owner, entry.Type, entry.ServerList, entry.Elapsed)
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
check := func(exp, got madmin.LockEntries) (int, bool) {
|
||||||
|
if len(exp) != len(got) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
sort.Slice(exp, byResourceUID{exp}.Less)
|
||||||
|
sort.Slice(got, byResourceUID{got}.Less)
|
||||||
|
// printEntries(exp)
|
||||||
|
// printEntries(got)
|
||||||
|
for i, e := range exp {
|
||||||
|
if !e.Timestamp.Equal(got[i].Timestamp) {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
// Skip checking elapsed since it's time sensitive.
|
||||||
|
// if e.Elapsed != got[i].Elapsed {
|
||||||
|
// return false
|
||||||
|
// }
|
||||||
|
if e.Resource != got[i].Resource {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
if e.Type != got[i].Type {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
if e.Source != got[i].Source {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
if e.Owner != got[i].Owner {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
if e.ID != got[i].ID {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
if len(e.ServerList) != len(got[i].ServerList) {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
for j := range e.ServerList {
|
||||||
|
if e.ServerList[j] != got[i].ServerList[j] {
|
||||||
|
return i, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, true
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, tc := range testCases {
|
||||||
|
got := topLockEntries(tc.peerLocks, false)
|
||||||
|
if idx, ok := check(tc.expected, got); !ok {
|
||||||
|
t.Fatalf("%d: mismatch at %d \n expected %#v but got %#v", i, idx, tc.expected[idx], got[idx])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+22
-30
@@ -27,7 +27,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -253,7 +254,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
|
|||||||
} else {
|
} else {
|
||||||
clientToken := he.clientToken
|
clientToken := he.clientToken
|
||||||
if globalIsDistErasure {
|
if globalIsDistErasure {
|
||||||
clientToken = fmt.Sprintf("%s@%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
clientToken = fmt.Sprintf("%s:%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||||
}
|
}
|
||||||
|
|
||||||
hsp = madmin.HealStopSuccess{
|
hsp = madmin.HealStopSuccess{
|
||||||
@@ -327,7 +328,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
|
|||||||
|
|
||||||
clientToken := h.clientToken
|
clientToken := h.clientToken
|
||||||
if globalIsDistErasure {
|
if globalIsDistErasure {
|
||||||
clientToken = fmt.Sprintf("%s@%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||||
}
|
}
|
||||||
|
|
||||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||||
@@ -407,9 +408,6 @@ type healSequence struct {
|
|||||||
// bucket, and object on which heal seq. was initiated
|
// bucket, and object on which heal seq. was initiated
|
||||||
bucket, object string
|
bucket, object string
|
||||||
|
|
||||||
// A channel of entities with heal result
|
|
||||||
respCh chan healResult
|
|
||||||
|
|
||||||
// Report healing progress
|
// Report healing progress
|
||||||
reportProgress bool
|
reportProgress bool
|
||||||
|
|
||||||
@@ -472,7 +470,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
|
|||||||
clientToken := mustGetUUID()
|
clientToken := mustGetUUID()
|
||||||
|
|
||||||
return &healSequence{
|
return &healSequence{
|
||||||
respCh: make(chan healResult),
|
|
||||||
bucket: bucket,
|
bucket: bucket,
|
||||||
object: objPrefix,
|
object: objPrefix,
|
||||||
reportProgress: true,
|
reportProgress: true,
|
||||||
@@ -687,13 +684,6 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *healSequence) logHeal(healType madmin.HealItemType) {
|
|
||||||
h.mutex.Lock()
|
|
||||||
h.scannedItemsMap[healType]++
|
|
||||||
h.lastHealActivity = UTCNow()
|
|
||||||
h.mutex.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
|
func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
|
||||||
// Send heal request
|
// Send heal request
|
||||||
task := healTask{
|
task := healTask{
|
||||||
@@ -719,15 +709,18 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
|
|||||||
if serverDebugLog {
|
if serverDebugLog {
|
||||||
logger.Info("Task in the queue: %#v", task)
|
logger.Info("Task in the queue: %#v", task)
|
||||||
}
|
}
|
||||||
case <-h.ctx.Done():
|
|
||||||
return nil
|
|
||||||
default:
|
default:
|
||||||
// task queue is full, no more workers, we shall move on and heal later.
|
// task queue is full, no more workers, we shall move on and heal later.
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
} else {
|
// Don't wait for result
|
||||||
// respCh must be set for guaranteed result
|
return nil
|
||||||
task.respCh = h.respCh
|
}
|
||||||
|
|
||||||
|
// respCh must be set to wait for result.
|
||||||
|
// We make it size 1, so a result can always be written
|
||||||
|
// even if we aren't listening.
|
||||||
|
task.respCh = make(chan healResult, 1)
|
||||||
select {
|
select {
|
||||||
case globalBackgroundHealRoutine.tasks <- task:
|
case globalBackgroundHealRoutine.tasks <- task:
|
||||||
if serverDebugLog {
|
if serverDebugLog {
|
||||||
@@ -736,11 +729,10 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
|
|||||||
case <-h.ctx.Done():
|
case <-h.ctx.Done():
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// task queued, now wait for the response.
|
// task queued, now wait for the response.
|
||||||
select {
|
select {
|
||||||
case res := <-h.respCh:
|
case res := <-task.respCh:
|
||||||
if !h.reportProgress {
|
if !h.reportProgress {
|
||||||
if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
|
if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
|
||||||
return nil
|
return nil
|
||||||
@@ -786,6 +778,11 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
|
func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
|
||||||
|
if h.clientToken == bgHealingUUID {
|
||||||
|
// For background heal do nothing.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if err := h.healDiskMeta(objAPI); err != nil {
|
if err := h.healDiskMeta(objAPI); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -804,7 +801,7 @@ func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
|
|||||||
func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
|
func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
|
||||||
bucketsOnly := false // Heals buckets and objects also.
|
bucketsOnly := false // Heals buckets and objects also.
|
||||||
h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
|
h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
|
||||||
close(h.traverseAndHealDoneCh)
|
xioutil.SafeClose(h.traverseAndHealDoneCh)
|
||||||
}
|
}
|
||||||
|
|
||||||
// healMinioSysMeta - heals all files under a given meta prefix, returns a function
|
// healMinioSysMeta - heals all files under a given meta prefix, returns a function
|
||||||
@@ -814,7 +811,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
|
|||||||
// NOTE: Healing on meta is run regardless
|
// NOTE: Healing on meta is run regardless
|
||||||
// of any bucket being selected, this is to ensure that
|
// of any bucket being selected, this is to ensure that
|
||||||
// meta are always upto date and correct.
|
// meta are always upto date and correct.
|
||||||
return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
|
return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
|
||||||
if h.isQuitting() {
|
if h.isQuitting() {
|
||||||
return errHealStopSignalled
|
return errHealStopSignalled
|
||||||
}
|
}
|
||||||
@@ -871,15 +868,10 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
|
|||||||
|
|
||||||
if !h.settings.Recursive {
|
if !h.settings.Recursive {
|
||||||
if h.object != "" {
|
if h.object != "" {
|
||||||
// Check if an object named as the objPrefix exists,
|
if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
|
||||||
// and if so heal it.
|
|
||||||
oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
|
|
||||||
if err == nil {
|
|
||||||
if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -891,7 +883,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
|
|||||||
}
|
}
|
||||||
|
|
||||||
// healObject - heal the given object and record result
|
// healObject - heal the given object and record result
|
||||||
func (h *healSequence) healObject(bucket, object, versionID string) error {
|
func (h *healSequence) healObject(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
|
||||||
if h.isQuitting() {
|
if h.isQuitting() {
|
||||||
return errHealStopSignalled
|
return errHealStopSignalled
|
||||||
}
|
}
|
||||||
|
|||||||
+243
-121
@@ -19,10 +19,13 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/klauspost/compress/gzhttp"
|
"github.com/klauspost/compress/gzhttp"
|
||||||
"github.com/klauspost/compress/gzip"
|
"github.com/klauspost/compress/gzip"
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
)
|
)
|
||||||
@@ -31,8 +34,112 @@ const (
|
|||||||
adminPathPrefix = minioReservedBucketPath + "/admin"
|
adminPathPrefix = minioReservedBucketPath + "/admin"
|
||||||
adminAPIVersion = madmin.AdminAPIVersion
|
adminAPIVersion = madmin.AdminAPIVersion
|
||||||
adminAPIVersionPrefix = SlashSeparator + adminAPIVersion
|
adminAPIVersionPrefix = SlashSeparator + adminAPIVersion
|
||||||
|
adminAPISiteReplicationDevNull = "/site-replication/devnull"
|
||||||
|
adminAPISiteReplicationNetPerf = "/site-replication/netperf"
|
||||||
|
adminAPIClientDevNull = "/speedtest/client/devnull"
|
||||||
|
adminAPIClientDevExtraTime = "/speedtest/client/devnull/extratime"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var gzipHandler = func() func(http.Handler) http.HandlerFunc {
|
||||||
|
gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
|
||||||
|
if err != nil {
|
||||||
|
// Static params, so this is very unlikely.
|
||||||
|
logger.Fatal(err, "Unable to initialize server")
|
||||||
|
}
|
||||||
|
return gz
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Set of handler options as bit flags
|
||||||
|
type hFlag uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
// this flag disables gzip compression of responses
|
||||||
|
noGZFlag = 1 << iota
|
||||||
|
|
||||||
|
// this flag enables tracing body and headers instead of just headers
|
||||||
|
traceAllFlag
|
||||||
|
|
||||||
|
// pass this flag to skip checking if object layer is available
|
||||||
|
noObjLayerFlag
|
||||||
|
)
|
||||||
|
|
||||||
|
// Has checks if the the given flag is enabled in `h`.
|
||||||
|
func (h hFlag) Has(flag hFlag) bool {
|
||||||
|
// Use bitwise-AND and check if the result is non-zero.
|
||||||
|
return h&flag != 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func getHandlerName(f http.HandlerFunc) string {
|
||||||
|
name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
|
||||||
|
name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.adminAPIHandlers.")
|
||||||
|
name = strings.TrimSuffix(name, "Handler-fm")
|
||||||
|
name = strings.TrimSuffix(name, "-fm")
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
// adminMiddleware performs some common admin handler functionality for all
|
||||||
|
// handlers:
|
||||||
|
//
|
||||||
|
// - updates request context with `logger.ReqInfo` and api name based on the
|
||||||
|
// name of the function handler passed (this handler must be a method of
|
||||||
|
// `adminAPIHandlers`).
|
||||||
|
//
|
||||||
|
// - sets up call to send AuditLog
|
||||||
|
//
|
||||||
|
// Note that, while this is a middleware function (i.e. it takes a handler
|
||||||
|
// function and returns one), due to flags being passed based on required
|
||||||
|
// conditions, it is done per-"handler function registration" in the router.
|
||||||
|
//
|
||||||
|
// When no flags are passed, gzip compression, http tracing of headers and
|
||||||
|
// checking of object layer availability are all enabled. Use flags to modify
|
||||||
|
// this behavior.
|
||||||
|
func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc {
|
||||||
|
// Collect all flags with bitwise-OR and assign operator
|
||||||
|
var handlerFlags hFlag
|
||||||
|
for _, flag := range flags {
|
||||||
|
handlerFlags |= flag
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get name of the handler using reflection. NOTE: The passed in handler
|
||||||
|
// function must be a method of `adminAPIHandlers` for this extraction to
|
||||||
|
// work as expected.
|
||||||
|
handlerName := getHandlerName(f)
|
||||||
|
|
||||||
|
var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Update request context with `logger.ReqInfo`.
|
||||||
|
r = r.WithContext(newContext(r, w, handlerName))
|
||||||
|
|
||||||
|
defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
|
||||||
|
|
||||||
|
// Check if object layer is available, if not return error early.
|
||||||
|
if !handlerFlags.Has(noObjLayerFlag) {
|
||||||
|
objectAPI := newObjectLayerFn()
|
||||||
|
if objectAPI == nil || globalNotificationSys == nil {
|
||||||
|
writeErrorResponseJSON(r.Context(), w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply http tracing "middleware" based on presence of flag.
|
||||||
|
var f2 http.HandlerFunc
|
||||||
|
if handlerFlags.Has(traceAllFlag) {
|
||||||
|
f2 = httpTraceAll(f)
|
||||||
|
} else {
|
||||||
|
f2 = httpTraceHdrs(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// call the final handler
|
||||||
|
f2(w, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable compression of responses based on presence of flag.
|
||||||
|
if !handlerFlags.Has(noGZFlag) {
|
||||||
|
handler = gzipHandler(handler)
|
||||||
|
}
|
||||||
|
|
||||||
|
return handler
|
||||||
|
}
|
||||||
|
|
||||||
// adminAPIHandlers provides HTTP handlers for MinIO admin API.
|
// adminAPIHandlers provides HTTP handlers for MinIO admin API.
|
||||||
type adminAPIHandlers struct{}
|
type adminAPIHandlers struct{}
|
||||||
|
|
||||||
@@ -46,263 +153,278 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
|||||||
adminAPIVersionPrefix,
|
adminAPIVersionPrefix,
|
||||||
}
|
}
|
||||||
|
|
||||||
gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
|
|
||||||
if err != nil {
|
|
||||||
// Static params, so this is very unlikely.
|
|
||||||
logger.Fatal(err, "Unable to initialize server")
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, adminVersion := range adminVersions {
|
for _, adminVersion := range adminVersions {
|
||||||
// Restart and stop MinIO service.
|
// Restart and stop MinIO service type=2
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(gz(httpTraceAll(adminAPI.ServiceHandler))).Queries("action", "{action:.*}")
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceV2Handler, traceAllFlag)).Queries("action", "{action:.*}", "type", "2")
|
||||||
// Update MinIO servers.
|
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(gz(httpTraceAll(adminAPI.ServerUpdateHandler))).Queries("updateURL", "{updateURL:.*}")
|
// Deprecated: Restart and stop MinIO service.
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceHandler, traceAllFlag)).Queries("action", "{action:.*}")
|
||||||
|
|
||||||
|
// Update all MinIO servers type=2
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateV2Handler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}", "type", "2")
|
||||||
|
|
||||||
|
// Deprecated: Update MinIO servers.
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateHandler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}")
|
||||||
|
|
||||||
// Info operations
|
// Info operations
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(gz(httpTraceAll(adminAPI.ServerInfoHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(adminMiddleware(adminAPI.ServerInfoHandler, traceAllFlag, noObjLayerFlag))
|
||||||
adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(httpTraceAll(adminAPI.InspectDataHandler))
|
adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(adminMiddleware(adminAPI.InspectDataHandler, noGZFlag, traceAllFlag))
|
||||||
|
|
||||||
// StorageInfo operations
|
// StorageInfo operations
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
|
||||||
// DataUsageInfo operations
|
// DataUsageInfo operations
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
|
||||||
// Metrics operation
|
// Metrics operation
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(gz(httpTraceAll(adminAPI.MetricsHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(adminMiddleware(adminAPI.MetricsHandler, traceAllFlag))
|
||||||
|
|
||||||
if globalIsDistErasure || globalIsErasure {
|
if globalIsDistErasure || globalIsErasure {
|
||||||
// Heal operations
|
// Heal operations
|
||||||
|
|
||||||
// Heal processing endpoint.
|
// Heal processing endpoint.
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(gz(httpTraceAll(adminAPI.BackgroundHealStatusHandler)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(adminMiddleware(adminAPI.BackgroundHealStatusHandler, traceAllFlag))
|
||||||
|
|
||||||
// Pool operations
|
// Pool operations
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/pools/list").HandlerFunc(gz(httpTraceAll(adminAPI.ListPools)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/pools/list").HandlerFunc(adminMiddleware(adminAPI.ListPools, traceAllFlag))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/pools/status").HandlerFunc(gz(httpTraceAll(adminAPI.StatusPool))).Queries("pool", "{pool:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/pools/status").HandlerFunc(adminMiddleware(adminAPI.StatusPool, traceAllFlag)).Queries("pool", "{pool:.*}")
|
||||||
|
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(adminMiddleware(adminAPI.StartDecommission, traceAllFlag)).Queries("pool", "{pool:.*}")
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(adminMiddleware(adminAPI.CancelDecommission, traceAllFlag)).Queries("pool", "{pool:.*}")
|
||||||
|
|
||||||
// Rebalance operations
|
// Rebalance operations
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStart)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(adminMiddleware(adminAPI.RebalanceStart, traceAllFlag))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStatus)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(adminMiddleware(adminAPI.RebalanceStatus, traceAllFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStop)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(adminMiddleware(adminAPI.RebalanceStop, traceAllFlag))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Profiling operations - deprecated API
|
// Profiling operations - deprecated API
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(gz(httpTraceAll(adminAPI.StartProfilingHandler))).
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(adminMiddleware(adminAPI.StartProfilingHandler, traceAllFlag, noObjLayerFlag)).
|
||||||
Queries("profilerType", "{profilerType:.*}")
|
Queries("profilerType", "{profilerType:.*}")
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(gz(httpTraceAll(adminAPI.DownloadProfilingHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(adminMiddleware(adminAPI.DownloadProfilingHandler, traceAllFlag, noObjLayerFlag))
|
||||||
// Profiling operations
|
// Profiling operations
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(gz(httpTraceAll(adminAPI.ProfileHandler)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(adminMiddleware(adminAPI.ProfileHandler, traceAllFlag, noObjLayerFlag))
|
||||||
|
|
||||||
// Config KV operations.
|
// Config KV operations.
|
||||||
if enableConfigOps {
|
if enableConfigOps {
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetConfigKVHandler))).Queries("key", "{key:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(adminMiddleware(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetConfigKVHandler)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(adminMiddleware(adminAPI.SetConfigKVHandler))
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.DelConfigKVHandler)))
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(adminMiddleware(adminAPI.DelConfigKVHandler))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Enable config help in all modes.
|
// Enable config help in all modes.
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(gz(httpTraceAll(adminAPI.HelpConfigKVHandler))).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(adminMiddleware(adminAPI.HelpConfigKVHandler, traceAllFlag)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
|
||||||
|
|
||||||
// Config KV history operations.
|
// Config KV history operations.
|
||||||
if enableConfigOps {
|
if enableConfigOps {
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(gz(httpTraceAll(adminAPI.ListConfigHistoryKVHandler))).Queries("count", "{count:[0-9]+}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.ListConfigHistoryKVHandler, traceAllFlag)).Queries("count", "{count:[0-9]+}")
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler))).Queries("restoreId", "{restoreId:.*}")
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler))).Queries("restoreId", "{restoreId:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config import/export bulk operations
|
// Config import/export bulk operations
|
||||||
if enableConfigOps {
|
if enableConfigOps {
|
||||||
// Get config
|
// Get config
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetConfigHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(adminMiddleware(adminAPI.GetConfigHandler))
|
||||||
// Set config
|
// Set config
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetConfigHandler)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(adminMiddleware(adminAPI.SetConfigHandler))
|
||||||
}
|
}
|
||||||
|
|
||||||
// -- IAM APIs --
|
// -- IAM APIs --
|
||||||
|
|
||||||
// Add policy IAM
|
// Add policy IAM
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(gz(httpTraceAll(adminAPI.AddCannedPolicy))).Queries("name", "{name:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(adminMiddleware(adminAPI.AddCannedPolicy, traceAllFlag)).Queries("name", "{name:.*}")
|
||||||
|
|
||||||
// Add user IAM
|
// Add user IAM
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(gz(httpTraceAll(adminAPI.AccountInfoHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(adminMiddleware(adminAPI.AccountInfoHandler, traceAllFlag))
|
||||||
|
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddUser))).Queries("accessKey", "{accessKey:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(adminMiddleware(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
|
||||||
|
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetUserStatus))).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(adminMiddleware(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
|
||||||
|
|
||||||
// Service accounts ops
|
// Service accounts ops
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddServiceAccount)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccount))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateServiceAccount))).Queries("accessKey", "{accessKey:.*}")
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(adminMiddleware(adminAPI.UpdateServiceAccount)).Queries("accessKey", "{accessKey:.*}")
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoServiceAccount))).Queries("accessKey", "{accessKey:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(adminMiddleware(adminAPI.InfoServiceAccount)).Queries("accessKey", "{accessKey:.*}")
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListServiceAccounts)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(adminMiddleware(adminAPI.ListServiceAccounts))
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteServiceAccount))).Queries("accessKey", "{accessKey:.*}")
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(adminMiddleware(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
|
||||||
|
|
||||||
// STS accounts ops
|
// STS accounts ops
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(gz(httpTraceHdrs(adminAPI.TemporaryAccountInfo))).Queries("accessKey", "{accessKey:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(adminMiddleware(adminAPI.TemporaryAccountInfo)).Queries("accessKey", "{accessKey:.*}")
|
||||||
|
|
||||||
// Info policy IAM latest
|
// Info policy IAM latest
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoCannedPolicy))).Queries("name", "{name:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(adminMiddleware(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
|
||||||
// List policies latest
|
// List policies latest
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketPolicies))).Queries("bucket", "{bucket:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(adminMiddleware(adminAPI.ListBucketPolicies)).Queries("bucket", "{bucket:.*}")
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListCannedPolicies)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(adminMiddleware(adminAPI.ListCannedPolicies))
|
||||||
|
|
||||||
// Builtin IAM policy associations
|
// Builtin IAM policy associations
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/builtin/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListPolicyMappingEntities)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/builtin/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListPolicyMappingEntities))
|
||||||
|
|
||||||
// Remove policy IAM
|
// Remove policy IAM
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveCannedPolicy))).Queries("name", "{name:.*}")
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(adminMiddleware(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
|
||||||
|
|
||||||
// Set user or group policy
|
// Set user or group policy
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
|
||||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup))).
|
HandlerFunc(adminMiddleware(adminAPI.SetPolicyForUserOrGroup)).
|
||||||
Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
|
Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
|
||||||
|
|
||||||
// Attach policies to user or group
|
// Attach/Detach policies to/from user or group
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/attach").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachPolicyBuiltin)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyBuiltin))
|
||||||
|
|
||||||
// Detach policies from user or group
|
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/detach").HandlerFunc(gz(httpTraceHdrs(adminAPI.DetachPolicyBuiltin)))
|
|
||||||
|
|
||||||
// Remove user IAM
|
// Remove user IAM
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveUser))).Queries("accessKey", "{accessKey:.*}")
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(adminMiddleware(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
|
||||||
|
|
||||||
// List users
|
// List users
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketUsers))).Queries("bucket", "{bucket:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(adminMiddleware(adminAPI.ListBucketUsers)).Queries("bucket", "{bucket:.*}")
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListUsers)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(adminMiddleware(adminAPI.ListUsers))
|
||||||
|
|
||||||
// User info
|
// User info
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetUserInfo))).Queries("accessKey", "{accessKey:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(adminMiddleware(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
|
||||||
// Add/Remove members from group
|
// Add/Remove members from group
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateGroupMembers)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(adminMiddleware(adminAPI.UpdateGroupMembers))
|
||||||
|
|
||||||
// Get Group
|
// Get Group
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetGroup))).Queries("group", "{group:.*}")
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(adminMiddleware(adminAPI.GetGroup)).Queries("group", "{group:.*}")
|
||||||
|
|
||||||
// List Groups
|
// List Groups
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListGroups)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(adminMiddleware(adminAPI.ListGroups))
|
||||||
|
|
||||||
// Set Group Status
|
// Set Group Status
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetGroupStatus))).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(adminMiddleware(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
|
||||||
|
|
||||||
// Export IAM info to zipped file
|
// Export IAM info to zipped file
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-iam").HandlerFunc(httpTraceHdrs(adminAPI.ExportIAM))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-iam").HandlerFunc(adminMiddleware(adminAPI.ExportIAM, noGZFlag))
|
||||||
|
|
||||||
// Import IAM info
|
// Import IAM info
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(adminMiddleware(adminAPI.ImportIAM, noGZFlag))
|
||||||
|
|
||||||
// IDentity Provider configuration APIs
|
// IDentity Provider configuration APIs
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddIdentityProviderCfg)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.AddIdentityProviderCfg))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateIdentityProviderCfg)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.UpdateIdentityProviderCfg))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListIdentityProviderCfg)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(adminMiddleware(adminAPI.ListIdentityProviderCfg))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.GetIdentityProviderCfg))
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg)))
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.DeleteIdentityProviderCfg))
|
||||||
|
|
||||||
|
// LDAP specific service accounts ops
|
||||||
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp/ldap/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccountLDAP))
|
||||||
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys").
|
||||||
|
HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).
|
||||||
|
Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}")
|
||||||
|
|
||||||
// LDAP IAM operations
|
// LDAP IAM operations
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListLDAPPolicyMappingEntities)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachDetachPolicyLDAP)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP))
|
||||||
// -- END IAM APIs --
|
// -- END IAM APIs --
|
||||||
|
|
||||||
// GetBucketQuotaConfig
|
// GetBucketQuotaConfig
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
|
adminMiddleware(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
|
||||||
// PutBucketQuotaConfig
|
// PutBucketQuotaConfig
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
|
adminMiddleware(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
|
||||||
|
|
||||||
// Bucket replication operations
|
// Bucket replication operations
|
||||||
// GetBucketTargetHandler
|
// GetBucketTargetHandler
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.ListRemoteTargetsHandler))).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
|
adminMiddleware(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
|
||||||
// SetRemoteTargetHandler
|
// SetRemoteTargetHandler
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.SetRemoteTargetHandler))).Queries("bucket", "{bucket:.*}")
|
adminMiddleware(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
|
||||||
// RemoveRemoteTargetHandler
|
// RemoveRemoteTargetHandler
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
|
adminMiddleware(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
|
||||||
// ReplicationDiff - MinIO extension API
|
// ReplicationDiff - MinIO extension API
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
|
adminMiddleware(adminAPI.ReplicationDiffHandler)).Queries("bucket", "{bucket:.*}")
|
||||||
|
// ReplicationMRFHandler - MinIO extension API
|
||||||
|
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/replication/mrf").HandlerFunc(
|
||||||
|
adminMiddleware(adminAPI.ReplicationMRFHandler)).Queries("bucket", "{bucket:.*}")
|
||||||
|
|
||||||
// Batch job operations
|
// Batch job operations
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc(
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.StartBatchJob)))
|
adminMiddleware(adminAPI.StartBatchJob))
|
||||||
|
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.ListBatchJobs)))
|
adminMiddleware(adminAPI.ListBatchJobs))
|
||||||
|
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
|
adminMiddleware(adminAPI.DescribeBatchJob))
|
||||||
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(
|
||||||
|
adminMiddleware(adminAPI.CancelBatchJob))
|
||||||
|
|
||||||
// Bucket migration operations
|
// Bucket migration operations
|
||||||
// ExportBucketMetaHandler
|
// ExportBucketMetaHandler
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.ExportBucketMetadataHandler)))
|
adminMiddleware(adminAPI.ExportBucketMetadataHandler))
|
||||||
// ImportBucketMetaHandler
|
// ImportBucketMetaHandler
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-bucket-metadata").HandlerFunc(
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-bucket-metadata").HandlerFunc(
|
||||||
gz(httpTraceHdrs(adminAPI.ImportBucketMetadataHandler)))
|
adminMiddleware(adminAPI.ImportBucketMetadataHandler))
|
||||||
|
|
||||||
// Remote Tier management operations
|
// Remote Tier management operations
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(adminMiddleware(adminAPI.AddTierHandler))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.EditTierHandler))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(adminMiddleware(adminAPI.ListTierHandler))
|
||||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveTierHandler)))
|
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.RemoveTierHandler))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.VerifyTierHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.VerifyTierHandler))
|
||||||
// Tier stats
|
// Tier stats
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(adminMiddleware(adminAPI.TierStatsHandler))
|
||||||
|
|
||||||
// Cluster Replication APIs
|
// Cluster Replication APIs
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationAdd))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationRemove)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationRemove))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationInfo))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationMetaInfo))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationStatus))
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPISiteReplicationDevNull).HandlerFunc(adminMiddleware(adminAPI.SiteReplicationDevNull, noObjLayerFlag))
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPISiteReplicationNetPerf).HandlerFunc(adminMiddleware(adminAPI.SiteReplicationNetPerf, noObjLayerFlag))
|
||||||
|
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(adminMiddleware(adminAPI.SRPeerEdit))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(adminMiddleware(adminAPI.SRPeerRemove))
|
||||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationResyncOp))).Queries("operation", "{operation:.*}")
|
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationResyncOp)).Queries("operation", "{operation:.*}")
|
||||||
|
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/state/edit").HandlerFunc(adminMiddleware(adminAPI.SRStateEdit))
|
||||||
|
|
||||||
if globalIsDistErasure {
|
if globalIsDistErasure {
|
||||||
// Top locks
|
// Top locks
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(gz(httpTraceHdrs(adminAPI.TopLocksHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(adminMiddleware(adminAPI.TopLocksHandler))
|
||||||
// Force unlocks paths
|
// Force unlocks paths
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock").
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock").
|
||||||
Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
|
Queries("paths", "{paths:.*}").HandlerFunc(adminMiddleware(adminAPI.ForceUnlockHandler))
|
||||||
}
|
}
|
||||||
|
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedTestHandler))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(adminMiddleware(adminAPI.ObjectSpeedTestHandler, noGZFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(adminMiddleware(adminAPI.ObjectSpeedTestHandler, noGZFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(adminMiddleware(adminAPI.DriveSpeedtestHandler, noGZFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(adminMiddleware(adminAPI.NetperfHandler, noGZFlag))
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/site").HandlerFunc(adminMiddleware(adminAPI.SitePerfHandler, noGZFlag))
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevNull).HandlerFunc(adminMiddleware(adminAPI.ClientDevNull, noGZFlag))
|
||||||
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevExtraTime).HandlerFunc(adminMiddleware(adminAPI.ClientDevNullExtraTime, noGZFlag))
|
||||||
|
|
||||||
// HTTP Trace
|
// HTTP Trace
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(http.HandlerFunc(adminAPI.TraceHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminMiddleware(adminAPI.TraceHandler, noObjLayerFlag))
|
||||||
|
|
||||||
// Console Logs
|
// Console Logs
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(gz(httpTraceAll(adminAPI.ConsoleLogHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(adminMiddleware(adminAPI.ConsoleLogHandler, traceAllFlag))
|
||||||
|
|
||||||
// -- KMS APIs --
|
// -- KMS APIs --
|
||||||
//
|
//
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSStatusHandler)))
|
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(adminMiddleware(adminAPI.KMSStatusHandler, traceAllFlag))
|
||||||
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(gz(httpTraceAll(adminAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}")
|
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(adminMiddleware(adminAPI.KMSCreateKeyHandler, traceAllFlag)).Queries("key-id", "{key-id:.*}")
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSKeyStatusHandler)))
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(adminMiddleware(adminAPI.KMSKeyStatusHandler, traceAllFlag))
|
||||||
|
|
||||||
// Keep obdinfo for backward compatibility with mc
|
// Keep obdinfo for backward compatibility with mc
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
|
||||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler))
|
||||||
// -- Health API --
|
// -- Health API --
|
||||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
|
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
|
||||||
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
|
HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler))
|
||||||
}
|
}
|
||||||
|
|
||||||
// If none of the routes match add default error handler routes
|
// If none of the routes match add default error handler routes
|
||||||
|
|||||||
@@ -26,8 +26,9 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -86,7 +87,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
|||||||
}
|
}
|
||||||
|
|
||||||
props := madmin.ServerProperties{
|
props := madmin.ServerProperties{
|
||||||
State: string(madmin.ItemInitializing),
|
|
||||||
Endpoint: addr,
|
Endpoint: addr,
|
||||||
Uptime: UTCNow().Unix() - globalBootTime.Unix(),
|
Uptime: UTCNow().Unix() - globalBootTime.Unix(),
|
||||||
Version: Version,
|
Version: Version,
|
||||||
@@ -118,7 +118,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
|||||||
config.EnvRootUser: {},
|
config.EnvRootUser: {},
|
||||||
config.EnvRootPassword: {},
|
config.EnvRootPassword: {},
|
||||||
config.EnvMinIOSubnetAPIKey: {},
|
config.EnvMinIOSubnetAPIKey: {},
|
||||||
config.EnvKMSSecretKey: {},
|
kms.EnvKMSSecretKey: {},
|
||||||
}
|
}
|
||||||
for _, v := range os.Environ() {
|
for _, v := range os.Environ() {
|
||||||
if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {
|
if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {
|
||||||
@@ -141,11 +141,12 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
|||||||
|
|
||||||
objLayer := newObjectLayerFn()
|
objLayer := newObjectLayerFn()
|
||||||
if objLayer != nil {
|
if objLayer != nil {
|
||||||
storageInfo := objLayer.LocalStorageInfo(GlobalContext)
|
storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
|
||||||
props.State = string(madmin.ItemOnline)
|
props.State = string(madmin.ItemOnline)
|
||||||
props.Disks = storageInfo.Disks
|
props.Disks = storageInfo.Disks
|
||||||
} else {
|
} else {
|
||||||
props.State = string(madmin.ItemOffline)
|
props.State = string(madmin.ItemInitializing)
|
||||||
|
props.Disks = getOfflineDisks("", globalEndpoints)
|
||||||
}
|
}
|
||||||
|
|
||||||
return props
|
return props
|
||||||
|
|||||||
+201
-91
@@ -1,4 +1,4 @@
|
|||||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
//
|
//
|
||||||
// This file is part of MinIO Object Storage stack
|
// This file is part of MinIO Object Storage stack
|
||||||
//
|
//
|
||||||
@@ -28,9 +28,10 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||||
|
"github.com/minio/minio/internal/ioutil"
|
||||||
"google.golang.org/api/googleapi"
|
"google.golang.org/api/googleapi"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/minio-go/v7/pkg/tags"
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
@@ -38,13 +39,15 @@ import (
|
|||||||
"github.com/minio/minio/internal/bucket/replication"
|
"github.com/minio/minio/internal/bucket/replication"
|
||||||
"github.com/minio/minio/internal/config/dns"
|
"github.com/minio/minio/internal/config/dns"
|
||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
|
|
||||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||||
"github.com/minio/minio/internal/bucket/versioning"
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
|
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||||
"github.com/minio/minio/internal/event"
|
"github.com/minio/minio/internal/event"
|
||||||
"github.com/minio/minio/internal/hash"
|
"github.com/minio/minio/internal/hash"
|
||||||
"github.com/minio/pkg/bucket/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
// APIError structure
|
// APIError structure
|
||||||
@@ -84,6 +87,7 @@ const (
|
|||||||
ErrInternalError
|
ErrInternalError
|
||||||
ErrInvalidAccessKeyID
|
ErrInvalidAccessKeyID
|
||||||
ErrAccessKeyDisabled
|
ErrAccessKeyDisabled
|
||||||
|
ErrInvalidArgument
|
||||||
ErrInvalidBucketName
|
ErrInvalidBucketName
|
||||||
ErrInvalidDigest
|
ErrInvalidDigest
|
||||||
ErrInvalidRange
|
ErrInvalidRange
|
||||||
@@ -132,7 +136,10 @@ const (
|
|||||||
ErrReplicationNeedsVersioningError
|
ErrReplicationNeedsVersioningError
|
||||||
ErrReplicationBucketNeedsVersioningError
|
ErrReplicationBucketNeedsVersioningError
|
||||||
ErrReplicationDenyEditError
|
ErrReplicationDenyEditError
|
||||||
|
ErrRemoteTargetDenyAddError
|
||||||
ErrReplicationNoExistingObjects
|
ErrReplicationNoExistingObjects
|
||||||
|
ErrReplicationValidationError
|
||||||
|
ErrReplicationPermissionCheckError
|
||||||
ErrObjectRestoreAlreadyInProgress
|
ErrObjectRestoreAlreadyInProgress
|
||||||
ErrNoSuchKey
|
ErrNoSuchKey
|
||||||
ErrNoSuchUpload
|
ErrNoSuchUpload
|
||||||
@@ -145,6 +152,7 @@ const (
|
|||||||
ErrMethodNotAllowed
|
ErrMethodNotAllowed
|
||||||
ErrInvalidPart
|
ErrInvalidPart
|
||||||
ErrInvalidPartOrder
|
ErrInvalidPartOrder
|
||||||
|
ErrMissingPart
|
||||||
ErrAuthorizationHeaderMalformed
|
ErrAuthorizationHeaderMalformed
|
||||||
ErrMalformedPOSTRequest
|
ErrMalformedPOSTRequest
|
||||||
ErrPOSTFileRequired
|
ErrPOSTFileRequired
|
||||||
@@ -164,7 +172,6 @@ const (
|
|||||||
ErrMalformedDate
|
ErrMalformedDate
|
||||||
ErrMalformedPresignedDate
|
ErrMalformedPresignedDate
|
||||||
ErrMalformedCredentialDate
|
ErrMalformedCredentialDate
|
||||||
ErrMalformedCredentialRegion
|
|
||||||
ErrMalformedExpires
|
ErrMalformedExpires
|
||||||
ErrNegativeExpires
|
ErrNegativeExpires
|
||||||
ErrAuthHeaderEmpty
|
ErrAuthHeaderEmpty
|
||||||
@@ -179,8 +186,11 @@ const (
|
|||||||
ErrBucketAlreadyExists
|
ErrBucketAlreadyExists
|
||||||
ErrMetadataTooLarge
|
ErrMetadataTooLarge
|
||||||
ErrUnsupportedMetadata
|
ErrUnsupportedMetadata
|
||||||
|
ErrUnsupportedHostHeader
|
||||||
ErrMaximumExpires
|
ErrMaximumExpires
|
||||||
ErrSlowDown
|
ErrSlowDownRead
|
||||||
|
ErrSlowDownWrite
|
||||||
|
ErrMaxVersionsExceeded
|
||||||
ErrInvalidPrefixMarker
|
ErrInvalidPrefixMarker
|
||||||
ErrBadRequest
|
ErrBadRequest
|
||||||
ErrKeyTooLongError
|
ErrKeyTooLongError
|
||||||
@@ -197,6 +207,7 @@ const (
|
|||||||
ErrInvalidTagDirective
|
ErrInvalidTagDirective
|
||||||
ErrPolicyAlreadyAttached
|
ErrPolicyAlreadyAttached
|
||||||
ErrPolicyNotAttached
|
ErrPolicyNotAttached
|
||||||
|
ErrExcessData
|
||||||
// Add new error codes here.
|
// Add new error codes here.
|
||||||
|
|
||||||
// SSE-S3/SSE-KMS related API errors
|
// SSE-S3/SSE-KMS related API errors
|
||||||
@@ -219,6 +230,7 @@ const (
|
|||||||
ErrIncompatibleEncryptionMethod
|
ErrIncompatibleEncryptionMethod
|
||||||
ErrKMSNotConfigured
|
ErrKMSNotConfigured
|
||||||
ErrKMSKeyNotFoundException
|
ErrKMSKeyNotFoundException
|
||||||
|
ErrKMSDefaultKeyAlreadyConfigured
|
||||||
|
|
||||||
ErrNoAccessKey
|
ErrNoAccessKey
|
||||||
ErrInvalidToken
|
ErrInvalidToken
|
||||||
@@ -242,18 +254,17 @@ const (
|
|||||||
// Add new extended error codes here.
|
// Add new extended error codes here.
|
||||||
|
|
||||||
// MinIO extended errors.
|
// MinIO extended errors.
|
||||||
ErrReadQuorum
|
|
||||||
ErrWriteQuorum
|
|
||||||
ErrStorageFull
|
ErrStorageFull
|
||||||
ErrRequestBodyParse
|
ErrRequestBodyParse
|
||||||
ErrObjectExistsAsDirectory
|
ErrObjectExistsAsDirectory
|
||||||
ErrInvalidObjectName
|
ErrInvalidObjectName
|
||||||
ErrInvalidObjectNamePrefixSlash
|
ErrInvalidObjectNamePrefixSlash
|
||||||
ErrInvalidResourceName
|
ErrInvalidResourceName
|
||||||
|
ErrInvalidLifecycleQueryParameter
|
||||||
ErrServerNotInitialized
|
ErrServerNotInitialized
|
||||||
ErrOperationTimedOut
|
ErrRequestTimedout
|
||||||
ErrClientDisconnected
|
ErrClientDisconnected
|
||||||
ErrOperationMaxedOut
|
ErrTooManyRequests
|
||||||
ErrInvalidRequest
|
ErrInvalidRequest
|
||||||
ErrTransitionStorageClassNotFoundError
|
ErrTransitionStorageClassNotFoundError
|
||||||
// MinIO storage class error codes
|
// MinIO storage class error codes
|
||||||
@@ -265,8 +276,10 @@ const (
|
|||||||
|
|
||||||
ErrMalformedJSON
|
ErrMalformedJSON
|
||||||
ErrAdminNoSuchUser
|
ErrAdminNoSuchUser
|
||||||
|
ErrAdminNoSuchUserLDAPWarn
|
||||||
ErrAdminNoSuchGroup
|
ErrAdminNoSuchGroup
|
||||||
ErrAdminGroupNotEmpty
|
ErrAdminGroupNotEmpty
|
||||||
|
ErrAdminGroupDisabled
|
||||||
ErrAdminNoSuchJob
|
ErrAdminNoSuchJob
|
||||||
ErrAdminNoSuchPolicy
|
ErrAdminNoSuchPolicy
|
||||||
ErrAdminPolicyChangeAlreadyApplied
|
ErrAdminPolicyChangeAlreadyApplied
|
||||||
@@ -280,6 +293,7 @@ const (
|
|||||||
ErrAdminConfigEnvOverridden
|
ErrAdminConfigEnvOverridden
|
||||||
ErrAdminConfigDuplicateKeys
|
ErrAdminConfigDuplicateKeys
|
||||||
ErrAdminConfigInvalidIDPType
|
ErrAdminConfigInvalidIDPType
|
||||||
|
ErrAdminConfigLDAPNonDefaultConfigName
|
||||||
ErrAdminConfigLDAPValidation
|
ErrAdminConfigLDAPValidation
|
||||||
ErrAdminConfigIDPCfgNameAlreadyExists
|
ErrAdminConfigIDPCfgNameAlreadyExists
|
||||||
ErrAdminConfigIDPCfgNameDoesNotExist
|
ErrAdminConfigIDPCfgNameDoesNotExist
|
||||||
@@ -296,6 +310,7 @@ const (
|
|||||||
ErrSiteReplicationBucketMetaError
|
ErrSiteReplicationBucketMetaError
|
||||||
ErrSiteReplicationIAMError
|
ErrSiteReplicationIAMError
|
||||||
ErrSiteReplicationConfigMissing
|
ErrSiteReplicationConfigMissing
|
||||||
|
ErrSiteReplicationIAMConfigMismatch
|
||||||
|
|
||||||
// Pool rebalance errors
|
// Pool rebalance errors
|
||||||
ErrAdminRebalanceAlreadyStarted
|
ErrAdminRebalanceAlreadyStarted
|
||||||
@@ -410,6 +425,15 @@ const (
|
|||||||
ErrPostPolicyConditionInvalidFormat
|
ErrPostPolicyConditionInvalidFormat
|
||||||
|
|
||||||
ErrInvalidChecksum
|
ErrInvalidChecksum
|
||||||
|
|
||||||
|
// Lambda functions
|
||||||
|
ErrLambdaARNInvalid
|
||||||
|
ErrLambdaARNNotFound
|
||||||
|
|
||||||
|
// New Codes for GetObjectAttributes and GetObjectVersionAttributes
|
||||||
|
ErrInvalidAttributeName
|
||||||
|
|
||||||
|
apiErrCodeEnd // This is used only for the testing code
|
||||||
)
|
)
|
||||||
|
|
||||||
type errorCodeMap map[APIErrorCode]APIError
|
type errorCodeMap map[APIErrorCode]APIError
|
||||||
@@ -423,8 +447,7 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
|
|||||||
apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
|
apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
|
||||||
}
|
}
|
||||||
if globalSite.Region != "" {
|
if globalSite.Region != "" {
|
||||||
switch errCode {
|
if errCode == ErrAuthorizationHeaderMalformed {
|
||||||
case ErrAuthorizationHeaderMalformed:
|
|
||||||
apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
|
apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
|
||||||
return apiErr
|
return apiErr
|
||||||
}
|
}
|
||||||
@@ -519,6 +542,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Your proposed upload exceeds the maximum allowed object size.",
|
Description: "Your proposed upload exceeds the maximum allowed object size.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrExcessData: {
|
||||||
|
Code: "ExcessData",
|
||||||
|
Description: "More data provided than indicated content length",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrPolicyTooLarge: {
|
ErrPolicyTooLarge: {
|
||||||
Code: "PolicyTooLarge",
|
Code: "PolicyTooLarge",
|
||||||
Description: "Policy exceeds the maximum allowed document size.",
|
Description: "Policy exceeds the maximum allowed document size.",
|
||||||
@@ -544,6 +572,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Your account is disabled; please contact your administrator.",
|
Description: "Your account is disabled; please contact your administrator.",
|
||||||
HTTPStatusCode: http.StatusForbidden,
|
HTTPStatusCode: http.StatusForbidden,
|
||||||
},
|
},
|
||||||
|
ErrInvalidArgument: {
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Invalid argument",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrInvalidBucketName: {
|
ErrInvalidBucketName: {
|
||||||
Code: "InvalidBucketName",
|
Code: "InvalidBucketName",
|
||||||
Description: "The specified bucket is not valid.",
|
Description: "The specified bucket is not valid.",
|
||||||
@@ -669,6 +702,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
|
Description: "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrMissingPart: {
|
||||||
|
Code: "InvalidRequest",
|
||||||
|
Description: "You must specify at least one part",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrInvalidPartOrder: {
|
ErrInvalidPartOrder: {
|
||||||
Code: "InvalidPartOrder",
|
Code: "InvalidPartOrder",
|
||||||
Description: "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
|
Description: "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
|
||||||
@@ -814,11 +852,21 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Request is not valid yet",
|
Description: "Request is not valid yet",
|
||||||
HTTPStatusCode: http.StatusForbidden,
|
HTTPStatusCode: http.StatusForbidden,
|
||||||
},
|
},
|
||||||
ErrSlowDown: {
|
ErrSlowDownRead: {
|
||||||
Code: "SlowDown",
|
Code: "SlowDownRead",
|
||||||
Description: "Resource requested is unreadable, please reduce your request rate",
|
Description: "Resource requested is unreadable, please reduce your request rate",
|
||||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||||
},
|
},
|
||||||
|
ErrSlowDownWrite: {
|
||||||
|
Code: "SlowDownWrite",
|
||||||
|
Description: "Resource requested is unwritable, please reduce your request rate",
|
||||||
|
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||||
|
},
|
||||||
|
ErrMaxVersionsExceeded: {
|
||||||
|
Code: "MaxVersionsExceeded",
|
||||||
|
Description: "You've exceeded the limit on the number of versions you can create on this object",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrInvalidPrefixMarker: {
|
ErrInvalidPrefixMarker: {
|
||||||
Code: "InvalidPrefixMarker",
|
Code: "InvalidPrefixMarker",
|
||||||
Description: "Invalid marker prefix combination",
|
Description: "Invalid marker prefix combination",
|
||||||
@@ -911,7 +959,7 @@ var errorCodes = errorCodeMap{
|
|||||||
},
|
},
|
||||||
ErrReplicationBandwidthLimitError: {
|
ErrReplicationBandwidthLimitError: {
|
||||||
Code: "XMinioAdminReplicationBandwidthLimitError",
|
Code: "XMinioAdminReplicationBandwidthLimitError",
|
||||||
Description: "Bandwidth limit for remote target must be atleast 100MBps",
|
Description: "Bandwidth limit for remote target must be at least 100MBps",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrReplicationNoExistingObjects: {
|
ErrReplicationNoExistingObjects: {
|
||||||
@@ -919,6 +967,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "No matching ExistingsObjects rule enabled",
|
Description: "No matching ExistingsObjects rule enabled",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrRemoteTargetDenyAddError: {
|
||||||
|
Code: "XMinioAdminRemoteTargetDenyAdd",
|
||||||
|
Description: "Cannot add remote target endpoint since this server is in a cluster replication setup",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrReplicationDenyEditError: {
|
ErrReplicationDenyEditError: {
|
||||||
Code: "XMinioReplicationDenyEdit",
|
Code: "XMinioReplicationDenyEdit",
|
||||||
Description: "Cannot alter local replication config since this server is in a cluster replication setup",
|
Description: "Cannot alter local replication config since this server is in a cluster replication setup",
|
||||||
@@ -974,6 +1027,16 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Versioning must be 'Enabled' on the bucket to add a replication target",
|
Description: "Versioning must be 'Enabled' on the bucket to add a replication target",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrReplicationValidationError: {
|
||||||
|
Code: "InvalidRequest",
|
||||||
|
Description: "Replication validation failed on target",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
|
ErrReplicationPermissionCheckError: {
|
||||||
|
Code: "ReplicationPermissionCheck",
|
||||||
|
Description: "X-Minio-Source-Replication-Check cannot be specified in request. Request cannot be completed",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrNoSuchObjectLockConfiguration: {
|
ErrNoSuchObjectLockConfiguration: {
|
||||||
Code: "NoSuchObjectLockConfiguration",
|
Code: "NoSuchObjectLockConfiguration",
|
||||||
Description: "The specified object does not have a ObjectLock configuration",
|
Description: "The specified object does not have a ObjectLock configuration",
|
||||||
@@ -1087,8 +1150,13 @@ var errorCodes = errorCodeMap{
|
|||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrInvalidEncryptionMethod: {
|
ErrInvalidEncryptionMethod: {
|
||||||
Code: "InvalidRequest",
|
Code: "InvalidArgument",
|
||||||
Description: "The encryption method specified is not supported",
|
Description: "Server Side Encryption with AWS KMS managed key requires HTTP header x-amz-server-side-encryption : aws:kms",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
|
ErrIncompatibleEncryptionMethod: {
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Server Side Encryption with Customer provided key is incompatible with the encryption method specified",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrInvalidEncryptionKeyID: {
|
ErrInvalidEncryptionKeyID: {
|
||||||
@@ -1151,11 +1219,6 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "The provided encryption parameters did not match the ones used originally.",
|
Description: "The provided encryption parameters did not match the ones used originally.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrIncompatibleEncryptionMethod: {
|
|
||||||
Code: "InvalidArgument",
|
|
||||||
Description: "Server side encryption specified with both SSE-C and SSE-S3 headers",
|
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
|
||||||
},
|
|
||||||
ErrKMSNotConfigured: {
|
ErrKMSNotConfigured: {
|
||||||
Code: "NotImplemented",
|
Code: "NotImplemented",
|
||||||
Description: "Server side encryption specified but KMS is not configured",
|
Description: "Server side encryption specified but KMS is not configured",
|
||||||
@@ -1166,6 +1229,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Invalid keyId",
|
Description: "Invalid keyId",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrKMSDefaultKeyAlreadyConfigured: {
|
||||||
|
Code: "KMS.DefaultKeyAlreadyConfiguredException",
|
||||||
|
Description: "A default encryption already exists and cannot be changed on KMS",
|
||||||
|
HTTPStatusCode: http.StatusConflict,
|
||||||
|
},
|
||||||
ErrNoAccessKey: {
|
ErrNoAccessKey: {
|
||||||
Code: "AccessDenied",
|
Code: "AccessDenied",
|
||||||
Description: "No AWSAccessKey was presented",
|
Description: "No AWSAccessKey was presented",
|
||||||
@@ -1230,11 +1298,21 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "The JSON you provided was not well-formed or did not validate against our published format.",
|
Description: "The JSON you provided was not well-formed or did not validate against our published format.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrInvalidLifecycleQueryParameter: {
|
||||||
|
Code: "XMinioInvalidLifecycleParameter",
|
||||||
|
Description: "The boolean value provided for withUpdatedAt query parameter was invalid.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrAdminNoSuchUser: {
|
ErrAdminNoSuchUser: {
|
||||||
Code: "XMinioAdminNoSuchUser",
|
Code: "XMinioAdminNoSuchUser",
|
||||||
Description: "The specified user does not exist.",
|
Description: "The specified user does not exist.",
|
||||||
HTTPStatusCode: http.StatusNotFound,
|
HTTPStatusCode: http.StatusNotFound,
|
||||||
},
|
},
|
||||||
|
ErrAdminNoSuchUserLDAPWarn: {
|
||||||
|
Code: "XMinioAdminNoSuchUser",
|
||||||
|
Description: "The specified user does not exist. If you meant a user in LDAP, use `mc idp ldap`",
|
||||||
|
HTTPStatusCode: http.StatusNotFound,
|
||||||
|
},
|
||||||
ErrAdminNoSuchGroup: {
|
ErrAdminNoSuchGroup: {
|
||||||
Code: "XMinioAdminNoSuchGroup",
|
Code: "XMinioAdminNoSuchGroup",
|
||||||
Description: "The specified group does not exist.",
|
Description: "The specified group does not exist.",
|
||||||
@@ -1250,6 +1328,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "The specified group is not empty - cannot remove it.",
|
Description: "The specified group is not empty - cannot remove it.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrAdminGroupDisabled: {
|
||||||
|
Code: "XMinioAdminGroupDisabled",
|
||||||
|
Description: "The specified group is disabled.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrAdminNoSuchPolicy: {
|
ErrAdminNoSuchPolicy: {
|
||||||
Code: "XMinioAdminNoSuchPolicy",
|
Code: "XMinioAdminNoSuchPolicy",
|
||||||
Description: "The canned policy does not exist.",
|
Description: "The canned policy does not exist.",
|
||||||
@@ -1312,6 +1395,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
|
Description: fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrAdminConfigLDAPNonDefaultConfigName: {
|
||||||
|
Code: "XMinioAdminConfigLDAPNonDefaultConfigName",
|
||||||
|
Description: "Only a single LDAP configuration is supported - config name must be empty or `_`",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrAdminConfigLDAPValidation: {
|
ErrAdminConfigLDAPValidation: {
|
||||||
Code: "XMinioAdminConfigLDAPValidation",
|
Code: "XMinioAdminConfigLDAPValidation",
|
||||||
Description: "LDAP Configuration validation failed",
|
Description: "LDAP Configuration validation failed",
|
||||||
@@ -1319,7 +1407,7 @@ var errorCodes = errorCodeMap{
|
|||||||
},
|
},
|
||||||
ErrAdminConfigIDPCfgNameAlreadyExists: {
|
ErrAdminConfigIDPCfgNameAlreadyExists: {
|
||||||
Code: "XMinioAdminConfigIDPCfgNameAlreadyExists",
|
Code: "XMinioAdminConfigIDPCfgNameAlreadyExists",
|
||||||
Description: "An IDP configuration with the given name aleady exists",
|
Description: "An IDP configuration with the given name already exists",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrAdminConfigIDPCfgNameDoesNotExist: {
|
ErrAdminConfigIDPCfgNameDoesNotExist: {
|
||||||
@@ -1357,7 +1445,7 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Cannot respond to plain-text request from TLS-encrypted server",
|
Description: "Cannot respond to plain-text request from TLS-encrypted server",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrOperationTimedOut: {
|
ErrRequestTimedout: {
|
||||||
Code: "RequestTimeout",
|
Code: "RequestTimeout",
|
||||||
Description: "A timeout occurred while trying to lock a resource, please reduce your request rate",
|
Description: "A timeout occurred while trying to lock a resource, please reduce your request rate",
|
||||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||||
@@ -1367,9 +1455,9 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Client disconnected before response was ready",
|
Description: "Client disconnected before response was ready",
|
||||||
HTTPStatusCode: 499, // No official code, use nginx value.
|
HTTPStatusCode: 499, // No official code, use nginx value.
|
||||||
},
|
},
|
||||||
ErrOperationMaxedOut: {
|
ErrTooManyRequests: {
|
||||||
Code: "SlowDown",
|
Code: "TooManyRequests",
|
||||||
Description: "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
|
Description: "Deadline exceeded while waiting in incoming queue, please reduce your request rate",
|
||||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||||
},
|
},
|
||||||
ErrUnsupportedMetadata: {
|
ErrUnsupportedMetadata: {
|
||||||
@@ -1377,6 +1465,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Your metadata headers are not supported.",
|
Description: "Your metadata headers are not supported.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrUnsupportedHostHeader: {
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Your Host header is malformed.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrObjectTampered: {
|
ErrObjectTampered: {
|
||||||
Code: "XMinioObjectTampered",
|
Code: "XMinioObjectTampered",
|
||||||
Description: errObjectTampered.Error(),
|
Description: errObjectTampered.Error(),
|
||||||
@@ -1423,6 +1516,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Site not found in site replication configuration",
|
Description: "Site not found in site replication configuration",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrSiteReplicationIAMConfigMismatch: {
|
||||||
|
Code: "XMinioSiteReplicationIAMConfigMismatch",
|
||||||
|
Description: "IAM configuration mismatch between sites",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
ErrAdminRebalanceAlreadyStarted: {
|
ErrAdminRebalanceAlreadyStarted: {
|
||||||
Code: "XMinioAdminRebalanceAlreadyStarted",
|
Code: "XMinioAdminRebalanceAlreadyStarted",
|
||||||
Description: "Pool rebalance is already started",
|
Description: "Pool rebalance is already started",
|
||||||
@@ -1509,7 +1607,7 @@ var errorCodes = errorCodeMap{
|
|||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
ErrBusy: {
|
ErrBusy: {
|
||||||
Code: "Busy",
|
Code: "ServerBusy",
|
||||||
Description: "The service is unavailable. Please retry.",
|
Description: "The service is unavailable. Please retry.",
|
||||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||||
},
|
},
|
||||||
@@ -1948,6 +2046,16 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "Invalid checksum provided.",
|
Description: "Invalid checksum provided.",
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
},
|
},
|
||||||
|
ErrLambdaARNInvalid: {
|
||||||
|
Code: "LambdaARNInvalid",
|
||||||
|
Description: "The specified lambda ARN is invalid",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
|
ErrLambdaARNNotFound: {
|
||||||
|
Code: "LambdaARNNotFound",
|
||||||
|
Description: "The specified lambda ARN does not exist",
|
||||||
|
HTTPStatusCode: http.StatusNotFound,
|
||||||
|
},
|
||||||
ErrPolicyAlreadyAttached: {
|
ErrPolicyAlreadyAttached: {
|
||||||
Code: "XMinioPolicyAlreadyAttached",
|
Code: "XMinioPolicyAlreadyAttached",
|
||||||
Description: "The specified policy is already attached.",
|
Description: "The specified policy is already attached.",
|
||||||
@@ -1958,6 +2066,11 @@ var errorCodes = errorCodeMap{
|
|||||||
Description: "The specified policy is not found.",
|
Description: "The specified policy is not found.",
|
||||||
HTTPStatusCode: http.StatusNotFound,
|
HTTPStatusCode: http.StatusNotFound,
|
||||||
},
|
},
|
||||||
|
ErrInvalidAttributeName: {
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Invalid attribute name specified.",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
},
|
||||||
// Add your error structure here.
|
// Add your error structure here.
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1970,18 +2083,23 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Only return ErrClientDisconnected if the provided context is actually canceled.
|
// Only return ErrClientDisconnected if the provided context is actually canceled.
|
||||||
// This way downstream context.Canceled will still report ErrOperationTimedOut
|
// This way downstream context.Canceled will still report ErrRequestTimedout
|
||||||
if contextCanceled(ctx) {
|
if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
|
||||||
if ctx.Err() == context.Canceled {
|
|
||||||
return ErrClientDisconnected
|
return ErrClientDisconnected
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
// Unwrap the error first
|
||||||
|
err = unwrapAll(err)
|
||||||
|
|
||||||
switch err {
|
switch err {
|
||||||
case errInvalidArgument:
|
case errInvalidArgument:
|
||||||
apiErr = ErrAdminInvalidArgument
|
apiErr = ErrAdminInvalidArgument
|
||||||
|
case errNoSuchPolicy:
|
||||||
|
apiErr = ErrAdminNoSuchPolicy
|
||||||
case errNoSuchUser:
|
case errNoSuchUser:
|
||||||
apiErr = ErrAdminNoSuchUser
|
apiErr = ErrAdminNoSuchUser
|
||||||
|
case errNoSuchUserLDAPWarn:
|
||||||
|
apiErr = ErrAdminNoSuchUserLDAPWarn
|
||||||
case errNoSuchServiceAccount:
|
case errNoSuchServiceAccount:
|
||||||
apiErr = ErrAdminServiceAccountNotFound
|
apiErr = ErrAdminServiceAccountNotFound
|
||||||
case errNoSuchGroup:
|
case errNoSuchGroup:
|
||||||
@@ -1990,8 +2108,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
apiErr = ErrAdminGroupNotEmpty
|
apiErr = ErrAdminGroupNotEmpty
|
||||||
case errNoSuchJob:
|
case errNoSuchJob:
|
||||||
apiErr = ErrAdminNoSuchJob
|
apiErr = ErrAdminNoSuchJob
|
||||||
case errNoSuchPolicy:
|
|
||||||
apiErr = ErrAdminNoSuchPolicy
|
|
||||||
case errNoPolicyToAttachOrDetach:
|
case errNoPolicyToAttachOrDetach:
|
||||||
apiErr = ErrAdminPolicyChangeAlreadyApplied
|
apiErr = ErrAdminPolicyChangeAlreadyApplied
|
||||||
case errSignatureMismatch:
|
case errSignatureMismatch:
|
||||||
@@ -2010,6 +2126,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
apiErr = ErrAdminInvalidSecretKey
|
apiErr = ErrAdminInvalidSecretKey
|
||||||
case errInvalidStorageClass:
|
case errInvalidStorageClass:
|
||||||
apiErr = ErrInvalidStorageClass
|
apiErr = ErrInvalidStorageClass
|
||||||
|
case errErasureReadQuorum:
|
||||||
|
apiErr = ErrSlowDownRead
|
||||||
|
case errErasureWriteQuorum:
|
||||||
|
apiErr = ErrSlowDownWrite
|
||||||
|
case errMaxVersionsExceeded:
|
||||||
|
apiErr = ErrMaxVersionsExceeded
|
||||||
// SSE errors
|
// SSE errors
|
||||||
case errInvalidEncryptionParameters:
|
case errInvalidEncryptionParameters:
|
||||||
apiErr = ErrInvalidEncryptionParameters
|
apiErr = ErrInvalidEncryptionParameters
|
||||||
@@ -2041,11 +2163,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
apiErr = ErrKMSNotConfigured
|
apiErr = ErrKMSNotConfigured
|
||||||
case errKMSKeyNotFound:
|
case errKMSKeyNotFound:
|
||||||
apiErr = ErrKMSKeyNotFoundException
|
apiErr = ErrKMSKeyNotFoundException
|
||||||
|
case errKMSDefaultKeyAlreadyConfigured:
|
||||||
case context.Canceled, context.DeadlineExceeded:
|
apiErr = ErrKMSDefaultKeyAlreadyConfigured
|
||||||
apiErr = ErrOperationTimedOut
|
case context.Canceled:
|
||||||
case errDiskNotFound:
|
apiErr = ErrClientDisconnected
|
||||||
apiErr = ErrSlowDown
|
case context.DeadlineExceeded:
|
||||||
|
apiErr = ErrRequestTimedout
|
||||||
case objectlock.ErrInvalidRetentionDate:
|
case objectlock.ErrInvalidRetentionDate:
|
||||||
apiErr = ErrInvalidRetentionDate
|
apiErr = ErrInvalidRetentionDate
|
||||||
case objectlock.ErrPastObjectLockRetainDate:
|
case objectlock.ErrPastObjectLockRetainDate:
|
||||||
@@ -2056,11 +2179,14 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
apiErr = ErrObjectLockInvalidHeaders
|
apiErr = ErrObjectLockInvalidHeaders
|
||||||
case objectlock.ErrMalformedXML:
|
case objectlock.ErrMalformedXML:
|
||||||
apiErr = ErrMalformedXML
|
apiErr = ErrMalformedXML
|
||||||
|
case errInvalidMaxParts:
|
||||||
|
apiErr = ErrInvalidMaxParts
|
||||||
|
case ioutil.ErrOverread:
|
||||||
|
apiErr = ErrExcessData
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compression errors
|
// Compression errors
|
||||||
switch err {
|
if err == errInvalidDecompressedSize {
|
||||||
case errInvalidDecompressedSize:
|
|
||||||
apiErr = ErrInvalidDecompressedSize
|
apiErr = ErrInvalidDecompressedSize
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2071,7 +2197,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
|
|
||||||
// etcd specific errors, a key is always a bucket for us return
|
// etcd specific errors, a key is always a bucket for us return
|
||||||
// ErrNoSuchBucket in such a case.
|
// ErrNoSuchBucket in such a case.
|
||||||
if err == dns.ErrNoEntriesFound {
|
if errors.Is(err, dns.ErrNoEntriesFound) {
|
||||||
return ErrNoSuchBucket
|
return ErrNoSuchBucket
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2121,11 +2247,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
case InvalidPart:
|
case InvalidPart:
|
||||||
apiErr = ErrInvalidPart
|
apiErr = ErrInvalidPart
|
||||||
case InsufficientWriteQuorum:
|
case InsufficientWriteQuorum:
|
||||||
apiErr = ErrSlowDown
|
apiErr = ErrSlowDownWrite
|
||||||
case InsufficientReadQuorum:
|
case InsufficientReadQuorum:
|
||||||
apiErr = ErrSlowDown
|
apiErr = ErrSlowDownRead
|
||||||
case InvalidMarkerPrefixCombination:
|
|
||||||
apiErr = ErrNotImplemented
|
|
||||||
case InvalidUploadIDKeyCombination:
|
case InvalidUploadIDKeyCombination:
|
||||||
apiErr = ErrNotImplemented
|
apiErr = ErrNotImplemented
|
||||||
case MalformedUploadID:
|
case MalformedUploadID:
|
||||||
@@ -2138,10 +2262,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
apiErr = ErrContentSHA256Mismatch
|
apiErr = ErrContentSHA256Mismatch
|
||||||
case hash.ChecksumMismatch:
|
case hash.ChecksumMismatch:
|
||||||
apiErr = ErrContentChecksumMismatch
|
apiErr = ErrContentChecksumMismatch
|
||||||
case ObjectTooLarge:
|
case hash.SizeTooSmall:
|
||||||
apiErr = ErrEntityTooLarge
|
|
||||||
case ObjectTooSmall:
|
|
||||||
apiErr = ErrEntityTooSmall
|
apiErr = ErrEntityTooSmall
|
||||||
|
case hash.SizeTooLarge:
|
||||||
|
apiErr = ErrEntityTooLarge
|
||||||
case NotImplemented:
|
case NotImplemented:
|
||||||
apiErr = ErrNotImplemented
|
apiErr = ErrNotImplemented
|
||||||
case PartTooBig:
|
case PartTooBig:
|
||||||
@@ -2186,7 +2310,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
apiErr = ErrTransitionStorageClassNotFoundError
|
apiErr = ErrTransitionStorageClassNotFoundError
|
||||||
case InvalidObjectState:
|
case InvalidObjectState:
|
||||||
apiErr = ErrInvalidObjectState
|
apiErr = ErrInvalidObjectState
|
||||||
|
case PreConditionFailed:
|
||||||
|
apiErr = ErrPreconditionFailed
|
||||||
case BucketQuotaExceeded:
|
case BucketQuotaExceeded:
|
||||||
apiErr = ErrAdminBucketQuotaExceeded
|
apiErr = ErrAdminBucketQuotaExceeded
|
||||||
case *event.ErrInvalidEventName:
|
case *event.ErrInvalidEventName:
|
||||||
@@ -2195,6 +2320,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
apiErr = ErrARNNotification
|
apiErr = ErrARNNotification
|
||||||
case *event.ErrARNNotFound:
|
case *event.ErrARNNotFound:
|
||||||
apiErr = ErrARNNotification
|
apiErr = ErrARNNotification
|
||||||
|
case *levent.ErrInvalidARN:
|
||||||
|
apiErr = ErrLambdaARNInvalid
|
||||||
|
case *levent.ErrARNNotFound:
|
||||||
|
apiErr = ErrLambdaARNNotFound
|
||||||
case *event.ErrUnknownRegion:
|
case *event.ErrUnknownRegion:
|
||||||
apiErr = ErrRegionNotification
|
apiErr = ErrRegionNotification
|
||||||
case *event.ErrInvalidFilterName:
|
case *event.ErrInvalidFilterName:
|
||||||
@@ -2212,7 +2341,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
case *event.ErrUnsupportedConfiguration:
|
case *event.ErrUnsupportedConfiguration:
|
||||||
apiErr = ErrUnsupportedNotification
|
apiErr = ErrUnsupportedNotification
|
||||||
case OperationTimedOut:
|
case OperationTimedOut:
|
||||||
apiErr = ErrOperationTimedOut
|
apiErr = ErrRequestTimedout
|
||||||
case BackendDown:
|
case BackendDown:
|
||||||
apiErr = ErrBackendDown
|
apiErr = ErrBackendDown
|
||||||
case ObjectNameTooLong:
|
case ObjectNameTooLong:
|
||||||
@@ -2222,6 +2351,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
case dns.ErrBucketConflict:
|
case dns.ErrBucketConflict:
|
||||||
apiErr = ErrBucketAlreadyExists
|
apiErr = ErrBucketAlreadyExists
|
||||||
default:
|
default:
|
||||||
|
if _, ok := err.(tags.Error); ok {
|
||||||
|
// tag errors are not exported, so we check their custom interface to avoid logging.
|
||||||
|
// The correct type is inserted by toAPIError.
|
||||||
|
apiErr = ErrInternalError
|
||||||
|
break
|
||||||
|
}
|
||||||
var ie, iw int
|
var ie, iw int
|
||||||
// This work-around is to handle the issue golang/go#30648
|
// This work-around is to handle the issue golang/go#30648
|
||||||
//nolint:gocritic
|
//nolint:gocritic
|
||||||
@@ -2258,38 +2393,27 @@ func toAPIError(ctx context.Context, err error) APIError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
|
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
|
||||||
e, ok := err.(dns.ErrInvalidBucketName)
|
switch apiErr.Code {
|
||||||
if ok {
|
case "NotImplemented":
|
||||||
code := toAPIErrorCode(ctx, e)
|
desc := fmt.Sprintf("%s (%v)", apiErr.Description, err)
|
||||||
apiErr = errorCodes.ToAPIErrWithErr(code, e)
|
|
||||||
}
|
|
||||||
|
|
||||||
if apiErr.Code == "NotImplemented" {
|
|
||||||
switch e := err.(type) {
|
|
||||||
case NotImplemented:
|
|
||||||
desc := e.Error()
|
|
||||||
if desc == "" {
|
|
||||||
desc = apiErr.Description
|
|
||||||
}
|
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: apiErr.Code,
|
Code: apiErr.Code,
|
||||||
Description: desc,
|
Description: desc,
|
||||||
HTTPStatusCode: apiErr.HTTPStatusCode,
|
HTTPStatusCode: apiErr.HTTPStatusCode,
|
||||||
}
|
}
|
||||||
return apiErr
|
case "XMinioBackendDown":
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if apiErr.Code == "XMinioBackendDown" {
|
|
||||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
|
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
|
||||||
return apiErr
|
case "InternalError":
|
||||||
}
|
|
||||||
|
|
||||||
if apiErr.Code == "InternalError" {
|
|
||||||
// If we see an internal error try to interpret
|
// If we see an internal error try to interpret
|
||||||
// any underlying errors if possible depending on
|
// any underlying errors if possible depending on
|
||||||
// their internal error types.
|
// their internal error types.
|
||||||
switch e := err.(type) {
|
switch e := err.(type) {
|
||||||
|
case kms.Error:
|
||||||
|
apiErr = APIError{
|
||||||
|
Description: e.Err.Error(),
|
||||||
|
Code: e.APICode,
|
||||||
|
HTTPStatusCode: e.HTTPStatusCode,
|
||||||
|
}
|
||||||
case batchReplicationJobError:
|
case batchReplicationJobError:
|
||||||
apiErr = APIError(e)
|
apiErr = APIError(e)
|
||||||
case InvalidArgument:
|
case InvalidArgument:
|
||||||
@@ -2301,26 +2425,24 @@ func toAPIError(ctx context.Context, err error) APIError {
|
|||||||
case *xml.SyntaxError:
|
case *xml.SyntaxError:
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "MalformedXML",
|
Code: "MalformedXML",
|
||||||
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
|
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, e),
|
||||||
e.Error()),
|
|
||||||
HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
|
HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
|
||||||
}
|
}
|
||||||
case url.EscapeError:
|
case url.EscapeError:
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "XMinioInvalidObjectName",
|
Code: "XMinioInvalidObjectName",
|
||||||
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
|
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description, e),
|
||||||
e.Error()),
|
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
}
|
}
|
||||||
case versioning.Error:
|
case versioning.Error:
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "IllegalVersioningConfigurationException",
|
Code: "IllegalVersioningConfigurationException",
|
||||||
Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e.Error()),
|
Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e),
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
}
|
}
|
||||||
case lifecycle.Error:
|
case lifecycle.Error:
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "InvalidRequest",
|
Code: "InvalidArgument",
|
||||||
Description: e.Error(),
|
Description: e.Error(),
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
}
|
}
|
||||||
@@ -2381,19 +2503,7 @@ func toAPIError(ctx context.Context, err error) APIError {
|
|||||||
// Add more other SDK related errors here if any in future.
|
// Add more other SDK related errors here if any in future.
|
||||||
default:
|
default:
|
||||||
//nolint:gocritic
|
//nolint:gocritic
|
||||||
if errors.Is(err, errMalformedEncoding) {
|
if errors.Is(err, errMalformedEncoding) || errors.Is(err, errChunkTooBig) || errors.Is(err, strconv.ErrRange) {
|
||||||
apiErr = APIError{
|
|
||||||
Code: "BadRequest",
|
|
||||||
Description: err.Error(),
|
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
|
||||||
}
|
|
||||||
} else if errors.Is(err, errChunkTooBig) {
|
|
||||||
apiErr = APIError{
|
|
||||||
Code: "BadRequest",
|
|
||||||
Description: err.Error(),
|
|
||||||
HTTPStatusCode: http.StatusBadRequest,
|
|
||||||
}
|
|
||||||
} else if errors.Is(err, strconv.ErrRange) {
|
|
||||||
apiErr = APIError{
|
apiErr = APIError{
|
||||||
Code: "BadRequest",
|
Code: "BadRequest",
|
||||||
Description: err.Error(),
|
Description: err.Error(),
|
||||||
|
|||||||
+18
-10
@@ -20,8 +20,6 @@ package cmd
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
@@ -42,9 +40,8 @@ var toAPIErrorTests = []struct {
|
|||||||
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
||||||
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
||||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||||
{err: InsufficientReadQuorum{}, errCode: ErrSlowDown},
|
{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
|
||||||
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDown},
|
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
|
||||||
{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
|
|
||||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||||
{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
|
{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
|
||||||
{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
|
{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
|
||||||
@@ -67,11 +64,6 @@ var toAPIErrorTests = []struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestAPIErrCode(t *testing.T) {
|
func TestAPIErrCode(t *testing.T) {
|
||||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
|
||||||
defer os.RemoveAll(disk)
|
|
||||||
|
|
||||||
initFSObjects(disk, t)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
for i, testCase := range toAPIErrorTests {
|
for i, testCase := range toAPIErrorTests {
|
||||||
errCode := toAPIErrorCode(ctx, testCase.err)
|
errCode := toAPIErrorCode(ctx, testCase.err)
|
||||||
@@ -80,3 +72,19 @@ func TestAPIErrCode(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if an API error is properly defined
|
||||||
|
func TestAPIErrCodeDefinition(t *testing.T) {
|
||||||
|
for errAPI := ErrNone + 1; errAPI < apiErrCodeEnd; errAPI++ {
|
||||||
|
errCode, ok := errorCodes[errAPI]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal(errAPI, "error code is not defined in the API error code table")
|
||||||
|
}
|
||||||
|
if errCode.Code == "" {
|
||||||
|
t.Fatal(errAPI, "error code has an empty XML code")
|
||||||
|
}
|
||||||
|
if errCode.HTTPStatusCode == 0 {
|
||||||
|
t.Fatal(errAPI, "error code has a zero HTTP status code")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+11
-12
@@ -23,11 +23,11 @@ import (
|
|||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
@@ -133,16 +133,15 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
|||||||
w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
|
w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
|
||||||
}
|
}
|
||||||
|
|
||||||
if globalCacheConfig.Enabled {
|
|
||||||
w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
|
|
||||||
w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set tag count if object has tags
|
// Set tag count if object has tags
|
||||||
if len(objInfo.UserTags) > 0 {
|
if len(objInfo.UserTags) > 0 {
|
||||||
tags, _ := url.ParseQuery(objInfo.UserTags)
|
tags, _ := tags.ParseObjectTags(objInfo.UserTags)
|
||||||
if len(tags) > 0 {
|
if tags.Count() > 0 {
|
||||||
w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
|
w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tags.Count())}
|
||||||
|
if opts.Tagging {
|
||||||
|
// This is MinIO only extension to return back tags along with the count.
|
||||||
|
w.Header()[xhttp.AmzObjectTagging] = []string{objInfo.UserTags}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,7 +152,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
|
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||||
// Do not need to send any internal metadata
|
// Do not need to send any internal metadata
|
||||||
// values to client.
|
// values to client.
|
||||||
continue
|
continue
|
||||||
@@ -166,7 +165,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
|||||||
|
|
||||||
var isSet bool
|
var isSet bool
|
||||||
for _, userMetadataPrefix := range userMetadataKeyPrefixes {
|
for _, userMetadataPrefix := range userMetadataKeyPrefixes {
|
||||||
if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(userMetadataPrefix)) {
|
if !stringsHasPrefixFold(k, userMetadataPrefix) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
w.Header()[strings.ToLower(k)] = []string{v}
|
w.Header()[strings.ToLower(k)] = []string{v}
|
||||||
@@ -203,7 +202,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Set the relevant version ID as part of the response header.
|
// Set the relevant version ID as part of the response header.
|
||||||
if objInfo.VersionID != "" {
|
if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
|
||||||
w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
|
w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
|
|||||||
maxkeys = maxObjectList
|
maxkeys = maxObjectList
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix = trimLeadingSlash(values.Get("prefix"))
|
prefix = values.Get("prefix")
|
||||||
marker = trimLeadingSlash(values.Get("marker"))
|
marker = values.Get("marker")
|
||||||
delimiter = values.Get("delimiter")
|
delimiter = values.Get("delimiter")
|
||||||
encodingType = values.Get("encoding-type")
|
encodingType = values.Get("encoding-type")
|
||||||
return
|
return
|
||||||
@@ -57,8 +57,8 @@ func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimit
|
|||||||
maxkeys = maxObjectList
|
maxkeys = maxObjectList
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix = trimLeadingSlash(values.Get("prefix"))
|
prefix = values.Get("prefix")
|
||||||
marker = trimLeadingSlash(values.Get("key-marker"))
|
marker = values.Get("key-marker")
|
||||||
delimiter = values.Get("delimiter")
|
delimiter = values.Get("delimiter")
|
||||||
encodingType = values.Get("encoding-type")
|
encodingType = values.Get("encoding-type")
|
||||||
versionIDMarker = values.Get("version-id-marker")
|
versionIDMarker = values.Get("version-id-marker")
|
||||||
@@ -87,8 +87,8 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
|
|||||||
maxkeys = maxObjectList
|
maxkeys = maxObjectList
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix = trimLeadingSlash(values.Get("prefix"))
|
prefix = values.Get("prefix")
|
||||||
startAfter = trimLeadingSlash(values.Get("start-after"))
|
startAfter = values.Get("start-after")
|
||||||
delimiter = values.Get("delimiter")
|
delimiter = values.Get("delimiter")
|
||||||
fetchOwner = values.Get("fetch-owner") == "true"
|
fetchOwner = values.Get("fetch-owner") == "true"
|
||||||
encodingType = values.Get("encoding-type")
|
encodingType = values.Get("encoding-type")
|
||||||
@@ -118,8 +118,8 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
|
|||||||
maxUploads = maxUploadsList
|
maxUploads = maxUploadsList
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix = trimLeadingSlash(values.Get("prefix"))
|
prefix = values.Get("prefix")
|
||||||
keyMarker = trimLeadingSlash(values.Get("key-marker"))
|
keyMarker = values.Get("key-marker")
|
||||||
uploadIDMarker = values.Get("upload-id-marker")
|
uploadIDMarker = values.Get("upload-id-marker")
|
||||||
delimiter = values.Get("delimiter")
|
delimiter = values.Get("delimiter")
|
||||||
encodingType = values.Get("encoding-type")
|
encodingType = values.Get("encoding-type")
|
||||||
|
|||||||
+117
-25
@@ -35,6 +35,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/hash"
|
"github.com/minio/minio/internal/hash"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
|
"github.com/minio/pkg/v2/policy"
|
||||||
xxml "github.com/minio/xxml"
|
xxml "github.com/minio/xxml"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -84,7 +85,7 @@ type ListVersionsResponse struct {
|
|||||||
VersionIDMarker string `xml:"VersionIdMarker"`
|
VersionIDMarker string `xml:"VersionIdMarker"`
|
||||||
|
|
||||||
MaxKeys int
|
MaxKeys int
|
||||||
Delimiter string
|
Delimiter string `xml:"Delimiter,omitempty"`
|
||||||
// A flag that indicates whether or not ListObjects returned all of the results
|
// A flag that indicates whether or not ListObjects returned all of the results
|
||||||
// that satisfied the search criteria.
|
// that satisfied the search criteria.
|
||||||
IsTruncated bool
|
IsTruncated bool
|
||||||
@@ -114,7 +115,7 @@ type ListObjectsResponse struct {
|
|||||||
NextMarker string `xml:"NextMarker,omitempty"`
|
NextMarker string `xml:"NextMarker,omitempty"`
|
||||||
|
|
||||||
MaxKeys int
|
MaxKeys int
|
||||||
Delimiter string
|
Delimiter string `xml:"Delimiter,omitempty"`
|
||||||
// A flag that indicates whether or not ListObjects returned all of the results
|
// A flag that indicates whether or not ListObjects returned all of the results
|
||||||
// that satisfied the search criteria.
|
// that satisfied the search criteria.
|
||||||
IsTruncated bool
|
IsTruncated bool
|
||||||
@@ -145,7 +146,7 @@ type ListObjectsV2Response struct {
|
|||||||
|
|
||||||
KeyCount int
|
KeyCount int
|
||||||
MaxKeys int
|
MaxKeys int
|
||||||
Delimiter string
|
Delimiter string `xml:"Delimiter,omitempty"`
|
||||||
// A flag that indicates whether or not ListObjects returned all of the results
|
// A flag that indicates whether or not ListObjects returned all of the results
|
||||||
// that satisfied the search criteria.
|
// that satisfied the search criteria.
|
||||||
IsTruncated bool
|
IsTruncated bool
|
||||||
@@ -204,7 +205,7 @@ type ListMultipartUploadsResponse struct {
|
|||||||
UploadIDMarker string `xml:"UploadIdMarker"`
|
UploadIDMarker string `xml:"UploadIdMarker"`
|
||||||
NextKeyMarker string
|
NextKeyMarker string
|
||||||
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
|
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
|
||||||
Delimiter string
|
Delimiter string `xml:"Delimiter,omitempty"`
|
||||||
Prefix string
|
Prefix string
|
||||||
EncodingType string `xml:"EncodingType,omitempty"`
|
EncodingType string `xml:"EncodingType,omitempty"`
|
||||||
MaxUploads int
|
MaxUploads int
|
||||||
@@ -344,6 +345,13 @@ func (s *Metadata) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
|
|||||||
return e.EncodeToken(start.End())
|
return e.EncodeToken(start.End())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ObjectInternalInfo contains some internal information about a given
|
||||||
|
// object, it will printed in listing calls with enabled metadata.
|
||||||
|
type ObjectInternalInfo struct {
|
||||||
|
K int // Data blocks
|
||||||
|
M int // Parity blocks
|
||||||
|
}
|
||||||
|
|
||||||
// Object container for object metadata
|
// Object container for object metadata
|
||||||
type Object struct {
|
type Object struct {
|
||||||
Key string
|
Key string
|
||||||
@@ -352,13 +360,16 @@ type Object struct {
|
|||||||
Size int64
|
Size int64
|
||||||
|
|
||||||
// Owner of the object.
|
// Owner of the object.
|
||||||
Owner Owner
|
Owner *Owner `xml:"Owner,omitempty"`
|
||||||
|
|
||||||
// The class of storage used to store the object.
|
// The class of storage used to store the object.
|
||||||
StorageClass string
|
StorageClass string
|
||||||
|
|
||||||
// UserMetadata user-defined metadata
|
// UserMetadata user-defined metadata
|
||||||
UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
|
UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
|
||||||
|
UserTags string `xml:"UserTags,omitempty"`
|
||||||
|
|
||||||
|
Internal *ObjectInternalInfo `xml:"Internal,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
|
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
|
||||||
@@ -491,20 +502,69 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
|||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cleanReservedKeys(metadata map[string]string) map[string]string {
|
||||||
|
m := cloneMSS(metadata)
|
||||||
|
|
||||||
|
switch kind, _ := crypto.IsEncrypted(metadata); kind {
|
||||||
|
case crypto.S3:
|
||||||
|
m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||||
|
case crypto.S3KMS:
|
||||||
|
m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||||
|
m[xhttp.AmzServerSideEncryptionKmsID] = kmsKeyIDFromMetadata(metadata)
|
||||||
|
if kmsCtx, ok := metadata[crypto.MetaContext]; ok {
|
||||||
|
m[xhttp.AmzServerSideEncryptionKmsContext] = kmsCtx
|
||||||
|
}
|
||||||
|
case crypto.SSEC:
|
||||||
|
m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
var toRemove []string
|
||||||
|
for k := range cleanMinioInternalMetadataKeys(m) {
|
||||||
|
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||||
|
// Do not need to send any internal metadata
|
||||||
|
// values to client.
|
||||||
|
toRemove = append(toRemove, k)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
||||||
|
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
||||||
|
toRemove = append(toRemove, k)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, k := range toRemove {
|
||||||
|
delete(m, k)
|
||||||
|
delete(m, strings.ToLower(k))
|
||||||
|
}
|
||||||
|
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
// generates an ListBucketVersions response for the said bucket with other enumerated options.
|
// generates an ListBucketVersions response for the said bucket with other enumerated options.
|
||||||
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
|
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
|
||||||
versions := make([]ObjectVersion, 0, len(resp.Objects))
|
versions := make([]ObjectVersion, 0, len(resp.Objects))
|
||||||
|
|
||||||
owner := Owner{
|
owner := &Owner{
|
||||||
ID: globalMinioDefaultOwnerID,
|
ID: globalMinioDefaultOwnerID,
|
||||||
DisplayName: "minio",
|
DisplayName: "minio",
|
||||||
}
|
}
|
||||||
data := ListVersionsResponse{}
|
data := ListVersionsResponse{}
|
||||||
|
var lastObjMetaName string
|
||||||
|
var tagErr, metaErr APIErrorCode = -1, -1
|
||||||
|
|
||||||
for _, object := range resp.Objects {
|
for _, object := range resp.Objects {
|
||||||
if object.Name == "" {
|
if object.Name == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// Cache checks for the same object
|
||||||
|
if metadata != nil && lastObjMetaName != object.Name {
|
||||||
|
tagErr = metadata(object.Name, policy.GetObjectTaggingAction)
|
||||||
|
metaErr = metadata(object.Name, policy.GetObjectAction)
|
||||||
|
lastObjMetaName = object.Name
|
||||||
|
}
|
||||||
content := ObjectVersion{}
|
content := ObjectVersion{}
|
||||||
content.Key = s3EncodeName(object.Name, encodingType)
|
content.Key = s3EncodeName(object.Name, encodingType)
|
||||||
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
|
content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
|
||||||
@@ -517,6 +577,29 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
|||||||
} else {
|
} else {
|
||||||
content.StorageClass = globalMinioDefaultStorageClass
|
content.StorageClass = globalMinioDefaultStorageClass
|
||||||
}
|
}
|
||||||
|
if tagErr == ErrNone {
|
||||||
|
content.UserTags = object.UserTags
|
||||||
|
}
|
||||||
|
if metaErr == ErrNone {
|
||||||
|
content.UserMetadata = &Metadata{}
|
||||||
|
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
|
||||||
|
case crypto.S3:
|
||||||
|
content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
|
||||||
|
case crypto.S3KMS:
|
||||||
|
content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
|
||||||
|
case crypto.SSEC:
|
||||||
|
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
|
||||||
|
}
|
||||||
|
for k, v := range cleanReservedKeys(object.UserDefined) {
|
||||||
|
content.UserMetadata.Set(k, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||||
|
content.Internal = &ObjectInternalInfo{
|
||||||
|
K: object.DataBlocks,
|
||||||
|
M: object.ParityBlocks,
|
||||||
|
}
|
||||||
|
}
|
||||||
content.Owner = owner
|
content.Owner = owner
|
||||||
content.VersionID = object.VersionID
|
content.VersionID = object.VersionID
|
||||||
if content.VersionID == "" {
|
if content.VersionID == "" {
|
||||||
@@ -553,7 +636,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
|||||||
// generates an ListObjectsV1 response for the said bucket with other enumerated options.
|
// generates an ListObjectsV1 response for the said bucket with other enumerated options.
|
||||||
func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
|
func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
|
||||||
contents := make([]Object, 0, len(resp.Objects))
|
contents := make([]Object, 0, len(resp.Objects))
|
||||||
owner := Owner{
|
owner := &Owner{
|
||||||
ID: globalMinioDefaultOwnerID,
|
ID: globalMinioDefaultOwnerID,
|
||||||
DisplayName: "minio",
|
DisplayName: "minio",
|
||||||
}
|
}
|
||||||
@@ -600,12 +683,16 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
|
|||||||
}
|
}
|
||||||
|
|
||||||
// generates an ListObjectsV2 response for the said bucket with other enumerated options.
|
// generates an ListObjectsV2 response for the said bucket with other enumerated options.
|
||||||
func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
|
func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
|
||||||
contents := make([]Object, 0, len(objects))
|
contents := make([]Object, 0, len(objects))
|
||||||
owner := Owner{
|
var owner *Owner
|
||||||
|
if fetchOwner {
|
||||||
|
owner = &Owner{
|
||||||
ID: globalMinioDefaultOwnerID,
|
ID: globalMinioDefaultOwnerID,
|
||||||
DisplayName: "minio",
|
DisplayName: "minio",
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
data := ListObjectsV2Response{}
|
data := ListObjectsV2Response{}
|
||||||
|
|
||||||
for _, object := range objects {
|
for _, object := range objects {
|
||||||
@@ -625,7 +712,11 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
|||||||
content.StorageClass = globalMinioDefaultStorageClass
|
content.StorageClass = globalMinioDefaultStorageClass
|
||||||
}
|
}
|
||||||
content.Owner = owner
|
content.Owner = owner
|
||||||
if metadata {
|
if metadata != nil {
|
||||||
|
if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone {
|
||||||
|
content.UserTags = object.UserTags
|
||||||
|
}
|
||||||
|
if metadata(object.Name, policy.GetObjectAction) == ErrNone {
|
||||||
content.UserMetadata = &Metadata{}
|
content.UserMetadata = &Metadata{}
|
||||||
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
|
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
|
||||||
case crypto.S3:
|
case crypto.S3:
|
||||||
@@ -635,18 +726,15 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
|||||||
case crypto.SSEC:
|
case crypto.SSEC:
|
||||||
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
|
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
|
||||||
}
|
}
|
||||||
for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
|
for k, v := range cleanReservedKeys(object.UserDefined) {
|
||||||
if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
|
|
||||||
// Do not need to send any internal metadata
|
|
||||||
// values to client.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
|
||||||
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
content.UserMetadata.Set(k, v)
|
content.UserMetadata.Set(k, v)
|
||||||
}
|
}
|
||||||
|
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||||
|
content.Internal = &ObjectInternalInfo{
|
||||||
|
K: object.DataBlocks,
|
||||||
|
M: object.ParityBlocks,
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
contents = append(contents, content)
|
contents = append(contents, content)
|
||||||
}
|
}
|
||||||
@@ -673,6 +761,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
|||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
||||||
|
|
||||||
// generates CopyObjectResponse from etag and lastModified time.
|
// generates CopyObjectResponse from etag and lastModified time.
|
||||||
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
||||||
return CopyObjectResponse{
|
return CopyObjectResponse{
|
||||||
@@ -700,7 +790,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
|
|||||||
|
|
||||||
// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
|
// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
|
||||||
func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
|
func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
|
||||||
cs := oi.decryptChecksums()
|
cs := oi.decryptChecksums(0)
|
||||||
c := CompleteMultipartUploadResponse{
|
c := CompleteMultipartUploadResponse{
|
||||||
Location: location,
|
Location: location,
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
@@ -875,12 +965,14 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
|
|||||||
|
|
||||||
// Generate error response.
|
// Generate error response.
|
||||||
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
|
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
|
||||||
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
|
||||||
encodedErrorResponse := encodeResponse(errorResponse)
|
encodedErrorResponse := encodeResponse(errorResponse)
|
||||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
|
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
|
func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
|
||||||
|
w.Header().Set(xMinIOErrCodeHeader, err.Code)
|
||||||
|
w.Header().Set(xMinIOErrDescHeader, "\""+err.Description+"\"")
|
||||||
writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
|
writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -893,7 +985,7 @@ func writeErrorResponseString(ctx context.Context, w http.ResponseWriter, err AP
|
|||||||
// useful for admin APIs.
|
// useful for admin APIs.
|
||||||
func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
|
func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
|
||||||
// Generate error response.
|
// Generate error response.
|
||||||
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
|
||||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||||
}
|
}
|
||||||
@@ -912,7 +1004,7 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
|
|||||||
BucketName: reqInfo.BucketName,
|
BucketName: reqInfo.BucketName,
|
||||||
Key: reqInfo.ObjectName,
|
Key: reqInfo.ObjectName,
|
||||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||||
HostID: globalDeploymentID,
|
HostID: globalDeploymentID(),
|
||||||
}
|
}
|
||||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||||
|
|||||||
+29
-27
@@ -23,11 +23,11 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/klauspost/compress/gzhttp"
|
"github.com/klauspost/compress/gzhttp"
|
||||||
"github.com/minio/console/restapi"
|
consoleapi "github.com/minio/console/api"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/mux"
|
"github.com/minio/mux"
|
||||||
"github.com/minio/pkg/wildcard"
|
"github.com/minio/pkg/v2/wildcard"
|
||||||
"github.com/rs/cors"
|
"github.com/rs/cors"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -43,13 +43,13 @@ func setHTTPServer(h *xhttp.Server) {
|
|||||||
globalObjLayerMutex.Unlock()
|
globalObjLayerMutex.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
func newConsoleServerFn() *restapi.Server {
|
func newConsoleServerFn() *consoleapi.Server {
|
||||||
globalObjLayerMutex.RLock()
|
globalObjLayerMutex.RLock()
|
||||||
defer globalObjLayerMutex.RUnlock()
|
defer globalObjLayerMutex.RUnlock()
|
||||||
return globalConsoleSrv
|
return globalConsoleSrv
|
||||||
}
|
}
|
||||||
|
|
||||||
func setConsoleSrv(srv *restapi.Server) {
|
func setConsoleSrv(srv *consoleapi.Server) {
|
||||||
globalObjLayerMutex.Lock()
|
globalObjLayerMutex.Lock()
|
||||||
globalConsoleSrv = srv
|
globalConsoleSrv = srv
|
||||||
globalObjLayerMutex.Unlock()
|
globalObjLayerMutex.Unlock()
|
||||||
@@ -61,18 +61,6 @@ func newObjectLayerFn() ObjectLayer {
|
|||||||
return globalObjectAPI
|
return globalObjectAPI
|
||||||
}
|
}
|
||||||
|
|
||||||
func newCachedObjectLayerFn() CacheObjectLayer {
|
|
||||||
globalObjLayerMutex.RLock()
|
|
||||||
defer globalObjLayerMutex.RUnlock()
|
|
||||||
return globalCacheObjectAPI
|
|
||||||
}
|
|
||||||
|
|
||||||
func setCacheObjectLayer(c CacheObjectLayer) {
|
|
||||||
globalObjLayerMutex.Lock()
|
|
||||||
globalCacheObjectAPI = c
|
|
||||||
globalObjLayerMutex.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func setObjectLayer(o ObjectLayer) {
|
func setObjectLayer(o ObjectLayer) {
|
||||||
globalObjLayerMutex.Lock()
|
globalObjLayerMutex.Lock()
|
||||||
globalObjectAPI = o
|
globalObjectAPI = o
|
||||||
@@ -82,7 +70,6 @@ func setObjectLayer(o ObjectLayer) {
|
|||||||
// objectAPIHandler implements and provides http handlers for S3 API.
|
// objectAPIHandler implements and provides http handlers for S3 API.
|
||||||
type objectAPIHandlers struct {
|
type objectAPIHandlers struct {
|
||||||
ObjectAPI func() ObjectLayer
|
ObjectAPI func() ObjectLayer
|
||||||
CacheAPI func() CacheObjectLayer
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// getHost tries its best to return the request host.
|
// getHost tries its best to return the request host.
|
||||||
@@ -189,7 +176,6 @@ func registerAPIRouter(router *mux.Router) {
|
|||||||
// Initialize API.
|
// Initialize API.
|
||||||
api := objectAPIHandlers{
|
api := objectAPIHandlers{
|
||||||
ObjectAPI: newObjectLayerFn,
|
ObjectAPI: newObjectLayerFn,
|
||||||
CacheAPI: newCachedObjectLayerFn,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// API Router
|
// API Router
|
||||||
@@ -241,14 +227,17 @@ func registerAPIRouter(router *mux.Router) {
|
|||||||
// HeadObject
|
// HeadObject
|
||||||
router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
||||||
collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
|
collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
|
||||||
|
// GetObjectAttribytes
|
||||||
|
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
|
collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "")
|
||||||
// CopyObjectPart
|
// CopyObjectPart
|
||||||
router.Methods(http.MethodPut).Path("/{object:.+}").
|
router.Methods(http.MethodPut).Path("/{object:.+}").
|
||||||
HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
|
HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
|
||||||
HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
|
HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
|
||||||
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
|
||||||
// PutObjectPart
|
// PutObjectPart
|
||||||
router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||||
collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
|
||||||
// ListObjectParts
|
// ListObjectParts
|
||||||
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
|
collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
|
||||||
@@ -285,7 +274,10 @@ func registerAPIRouter(router *mux.Router) {
|
|||||||
// GetObjectLegalHold
|
// GetObjectLegalHold
|
||||||
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
|
collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
|
||||||
// GetObject - note gzip compression is *not* added due to Range requests.
|
// GetObject with lambda ARNs
|
||||||
|
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
|
collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
|
||||||
|
// GetObject
|
||||||
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||||
collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
|
collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
|
||||||
// CopyObject
|
// CopyObject
|
||||||
@@ -389,6 +381,9 @@ func registerAPIRouter(router *mux.Router) {
|
|||||||
router.Methods(http.MethodGet).HandlerFunc(
|
router.Methods(http.MethodGet).HandlerFunc(
|
||||||
collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
|
collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
|
||||||
// ListObjectVersions
|
// ListObjectVersions
|
||||||
|
router.Methods(http.MethodGet).HandlerFunc(
|
||||||
|
collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
|
||||||
|
// ListObjectVersions
|
||||||
router.Methods(http.MethodGet).HandlerFunc(
|
router.Methods(http.MethodGet).HandlerFunc(
|
||||||
collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
|
collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
|
||||||
// GetBucketPolicyStatus
|
// GetBucketPolicyStatus
|
||||||
@@ -431,8 +426,9 @@ func registerAPIRouter(router *mux.Router) {
|
|||||||
router.Methods(http.MethodHead).HandlerFunc(
|
router.Methods(http.MethodHead).HandlerFunc(
|
||||||
collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
|
collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
|
||||||
// PostPolicy
|
// PostPolicy
|
||||||
router.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
|
router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
|
||||||
collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
|
return isRequestPostPolicySignatureV4(r)
|
||||||
|
}).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
|
||||||
// DeleteMultipleObjects
|
// DeleteMultipleObjects
|
||||||
router.Methods(http.MethodPost).HandlerFunc(
|
router.Methods(http.MethodPost).HandlerFunc(
|
||||||
collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")
|
collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")
|
||||||
@@ -454,10 +450,16 @@ func registerAPIRouter(router *mux.Router) {
|
|||||||
|
|
||||||
// MinIO extension API for replication.
|
// MinIO extension API for replication.
|
||||||
//
|
//
|
||||||
// GetBucketReplicationMetrics
|
router.Methods(http.MethodGet).HandlerFunc(
|
||||||
|
collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2")
|
||||||
|
// deprecated handler
|
||||||
router.Methods(http.MethodGet).HandlerFunc(
|
router.Methods(http.MethodGet).HandlerFunc(
|
||||||
collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
|
collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
|
||||||
|
|
||||||
|
// ValidateBucketReplicationCreds
|
||||||
|
router.Methods(http.MethodGet).HandlerFunc(
|
||||||
|
collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
|
||||||
|
|
||||||
// Register rejected bucket APIs
|
// Register rejected bucket APIs
|
||||||
for _, r := range rejectedBucketAPIs {
|
for _, r := range rejectedBucketAPIs {
|
||||||
router.Methods(r.methods...).
|
router.Methods(r.methods...).
|
||||||
@@ -513,8 +515,7 @@ func corsHandler(handler http.Handler) http.Handler {
|
|||||||
"x-amz*",
|
"x-amz*",
|
||||||
"*",
|
"*",
|
||||||
}
|
}
|
||||||
|
opts := cors.Options{
|
||||||
return cors.New(cors.Options{
|
|
||||||
AllowOriginFunc: func(origin string) bool {
|
AllowOriginFunc: func(origin string) bool {
|
||||||
for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
|
for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
|
||||||
if wildcard.MatchSimple(allowedOrigin, origin) {
|
if wildcard.MatchSimple(allowedOrigin, origin) {
|
||||||
@@ -535,5 +536,6 @@ func corsHandler(handler http.Handler) http.Handler {
|
|||||||
AllowedHeaders: commonS3Headers,
|
AllowedHeaders: commonS3Headers,
|
||||||
ExposedHeaders: commonS3Headers,
|
ExposedHeaders: commonS3Headers,
|
||||||
AllowCredentials: true,
|
AllowCredentials: true,
|
||||||
}).Handler(handler)
|
}
|
||||||
|
return cors.New(opts).Handler(handler)
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-8
@@ -94,14 +94,8 @@ func s3URLEncode(s string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// s3EncodeName encodes string in response when encodingType is specified in AWS S3 requests.
|
// s3EncodeName encodes string in response when encodingType is specified in AWS S3 requests.
|
||||||
func s3EncodeName(name string, encodingType string) (result string) {
|
func s3EncodeName(name, encodingType string) string {
|
||||||
// Quick path to exit
|
if strings.ToLower(encodingType) == "url" {
|
||||||
if encodingType == "" {
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
encodingType = strings.ToLower(encodingType)
|
|
||||||
switch encodingType {
|
|
||||||
case "url":
|
|
||||||
return s3URLEncode(name)
|
return s3URLEncode(name)
|
||||||
}
|
}
|
||||||
return name
|
return name
|
||||||
|
|||||||
+313
-297
File diff suppressed because one or more lines are too long
+104
-52
@@ -25,6 +25,7 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
|
"mime"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -40,8 +41,7 @@ import (
|
|||||||
xjwt "github.com/minio/minio/internal/jwt"
|
xjwt "github.com/minio/minio/internal/jwt"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
"github.com/minio/minio/internal/mcontext"
|
"github.com/minio/minio/internal/mcontext"
|
||||||
"github.com/minio/pkg/bucket/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Verify if request has JWT.
|
// Verify if request has JWT.
|
||||||
@@ -74,8 +74,11 @@ func isRequestPresignedSignatureV2(r *http.Request) bool {
|
|||||||
|
|
||||||
// Verify if request has AWS Post policy Signature Version '4'.
|
// Verify if request has AWS Post policy Signature Version '4'.
|
||||||
func isRequestPostPolicySignatureV4(r *http.Request) bool {
|
func isRequestPostPolicySignatureV4(r *http.Request) bool {
|
||||||
return strings.Contains(r.Header.Get(xhttp.ContentType), "multipart/form-data") &&
|
mediaType, _, err := mime.ParseMediaType(r.Header.Get(xhttp.ContentType))
|
||||||
r.Method == http.MethodPost
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return mediaType == "multipart/form-data" && r.Method == http.MethodPost
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
|
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
|
||||||
@@ -84,6 +87,18 @@ func isRequestSignStreamingV4(r *http.Request) bool {
|
|||||||
r.Method == http.MethodPut
|
r.Method == http.MethodPut
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
|
||||||
|
func isRequestSignStreamingTrailerV4(r *http.Request) bool {
|
||||||
|
return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256Trailer &&
|
||||||
|
r.Method == http.MethodPut
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer.
|
||||||
|
func isRequestUnsignedTrailerV4(r *http.Request) bool {
|
||||||
|
return r.Header.Get(xhttp.AmzContentSha256) == unsignedPayloadTrailer &&
|
||||||
|
r.Method == http.MethodPut && strings.Contains(r.Header.Get(xhttp.ContentEncoding), streamingContentEncoding)
|
||||||
|
}
|
||||||
|
|
||||||
// Authorization type.
|
// Authorization type.
|
||||||
//
|
//
|
||||||
//go:generate stringer -type=authType -trimprefix=authType $GOFILE
|
//go:generate stringer -type=authType -trimprefix=authType $GOFILE
|
||||||
@@ -101,10 +116,12 @@ const (
|
|||||||
authTypeSignedV2
|
authTypeSignedV2
|
||||||
authTypeJWT
|
authTypeJWT
|
||||||
authTypeSTS
|
authTypeSTS
|
||||||
|
authTypeStreamingSignedTrailer
|
||||||
|
authTypeStreamingUnsignedTrailer
|
||||||
)
|
)
|
||||||
|
|
||||||
// Get request authentication type.
|
// Get request authentication type.
|
||||||
func getRequestAuthType(r *http.Request) authType {
|
func getRequestAuthType(r *http.Request) (at authType) {
|
||||||
if r.URL != nil {
|
if r.URL != nil {
|
||||||
var err error
|
var err error
|
||||||
r.Form, err = url.ParseQuery(r.URL.RawQuery)
|
r.Form, err = url.ParseQuery(r.URL.RawQuery)
|
||||||
@@ -119,6 +136,10 @@ func getRequestAuthType(r *http.Request) authType {
|
|||||||
return authTypePresignedV2
|
return authTypePresignedV2
|
||||||
} else if isRequestSignStreamingV4(r) {
|
} else if isRequestSignStreamingV4(r) {
|
||||||
return authTypeStreamingSigned
|
return authTypeStreamingSigned
|
||||||
|
} else if isRequestSignStreamingTrailerV4(r) {
|
||||||
|
return authTypeStreamingSignedTrailer
|
||||||
|
} else if isRequestUnsignedTrailerV4(r) {
|
||||||
|
return authTypeStreamingUnsignedTrailer
|
||||||
} else if isRequestSignatureV4(r) {
|
} else if isRequestSignatureV4(r) {
|
||||||
return authTypeSigned
|
return authTypeSigned
|
||||||
} else if isRequestPresignedSignatureV4(r) {
|
} else if isRequestPresignedSignatureV4(r) {
|
||||||
@@ -135,43 +156,48 @@ func getRequestAuthType(r *http.Request) authType {
|
|||||||
return authTypeUnknown
|
return authTypeUnknown
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, map[string]interface{}, bool, APIErrorCode) {
|
func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, bool, APIErrorCode) {
|
||||||
var cred auth.Credentials
|
var cred auth.Credentials
|
||||||
var owner bool
|
var owner bool
|
||||||
s3Err := ErrAccessDenied
|
s3Err := ErrAccessDenied
|
||||||
if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
|
if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
|
||||||
getRequestAuthType(r) == authTypeSigned {
|
getRequestAuthType(r) == authTypeSigned {
|
||||||
// We only support admin credentials to access admin APIs.
|
|
||||||
|
// Get credential information from the request.
|
||||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||||
if s3Err != ErrNone {
|
if s3Err != ErrNone {
|
||||||
return cred, nil, owner, s3Err
|
return cred, owner, s3Err
|
||||||
}
|
}
|
||||||
|
|
||||||
// we only support V4 (no presign) with auth body
|
// we only support V4 (no presign) with auth body
|
||||||
s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
|
s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
|
||||||
}
|
}
|
||||||
if s3Err != ErrNone {
|
if s3Err != ErrNone {
|
||||||
return cred, nil, owner, s3Err
|
return cred, owner, s3Err
|
||||||
}
|
}
|
||||||
|
|
||||||
return cred, cred.Claims, owner, ErrNone
|
logger.GetReqInfo(ctx).Cred = cred
|
||||||
|
logger.GetReqInfo(ctx).Owner = owner
|
||||||
|
logger.GetReqInfo(ctx).Region = globalSite.Region
|
||||||
|
|
||||||
|
return cred, owner, ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkAdminRequestAuth checks for authentication and authorization for the incoming
|
// checkAdminRequestAuth checks for authentication and authorization for the incoming
|
||||||
// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
|
// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
|
||||||
// are automatically rejected.
|
// are automatically rejected.
|
||||||
func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
|
func checkAdminRequestAuth(ctx context.Context, r *http.Request, action policy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
|
||||||
cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
|
cred, owner, s3Err := validateAdminSignature(ctx, r, region)
|
||||||
if s3Err != ErrNone {
|
if s3Err != ErrNone {
|
||||||
return cred, s3Err
|
return cred, s3Err
|
||||||
}
|
}
|
||||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
if globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: iampolicy.Action(action),
|
Action: policy.Action(action),
|
||||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
IsOwner: owner,
|
IsOwner: owner,
|
||||||
Claims: claims,
|
Claims: cred.Claims,
|
||||||
}) {
|
}) {
|
||||||
// Request is allowed return the appropriate access key.
|
// Request is allowed return the appropriate access key.
|
||||||
return cred, ErrNone
|
return cred, ErrNone
|
||||||
@@ -203,7 +229,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
|
|||||||
// that clients cannot decode the token using the temp
|
// that clients cannot decode the token using the temp
|
||||||
// secret keys and generate an entirely new claim by essentially
|
// secret keys and generate an entirely new claim by essentially
|
||||||
// hijacking the policies. We need to make sure that this is
|
// hijacking the policies. We need to make sure that this is
|
||||||
// based an admin credential such that token cannot be decoded
|
// based on admin credential such that token cannot be decoded
|
||||||
// on the client side and is treated like an opaque value.
|
// on the client side and is treated like an opaque value.
|
||||||
claims, err := auth.ExtractClaims(token, secret)
|
claims, err := auth.ExtractClaims(token, secret)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -222,7 +248,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check if a session policy is set. If so, decode it here.
|
// Check if a session policy is set. If so, decode it here.
|
||||||
sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
|
sp, spok := claims.Lookup(policy.SessionPolicyName)
|
||||||
if spok {
|
if spok {
|
||||||
// Looks like subpolicy is set and is a string, if set then its
|
// Looks like subpolicy is set and is a string, if set then its
|
||||||
// base64 encoded, decode it. Decoding fails reject such
|
// base64 encoded, decode it. Decoding fails reject such
|
||||||
@@ -253,7 +279,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
|
|||||||
return nil, ErrNoAccessKey
|
return nil, ErrNoAccessKey
|
||||||
}
|
}
|
||||||
|
|
||||||
if token == "" && cred.IsTemp() {
|
if token == "" && cred.IsTemp() && !cred.IsServiceAccount() {
|
||||||
// Temporary credentials should always have x-amz-security-token
|
// Temporary credentials should always have x-amz-security-token
|
||||||
return nil, ErrInvalidToken
|
return nil, ErrInvalidToken
|
||||||
}
|
}
|
||||||
@@ -263,7 +289,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
|
|||||||
return nil, ErrInvalidToken
|
return nil, ErrInvalidToken
|
||||||
}
|
}
|
||||||
|
|
||||||
if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
|
if !cred.IsServiceAccount() && cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
|
||||||
// validate token for temporary credentials only.
|
// validate token for temporary credentials only.
|
||||||
return nil, ErrInvalidToken
|
return nil, ErrInvalidToken
|
||||||
}
|
}
|
||||||
@@ -300,6 +326,17 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
|||||||
return s3Err
|
return s3Err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
|
||||||
|
// passes versionID additionally.
|
||||||
|
func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
|
||||||
|
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||||
|
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||||
|
logger.GetReqInfo(ctx).VersionID = versionID
|
||||||
|
|
||||||
|
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
|
||||||
|
return s3Err
|
||||||
|
}
|
||||||
|
|
||||||
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||||
if logger.GetReqInfo(ctx) == nil {
|
if logger.GetReqInfo(ctx) == nil {
|
||||||
logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
|
logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
|
||||||
@@ -333,6 +370,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
|
|||||||
|
|
||||||
logger.GetReqInfo(ctx).Cred = cred
|
logger.GetReqInfo(ctx).Cred = cred
|
||||||
logger.GetReqInfo(ctx).Owner = owner
|
logger.GetReqInfo(ctx).Owner = owner
|
||||||
|
logger.GetReqInfo(ctx).Region = globalSite.Region
|
||||||
|
|
||||||
// region is valid only for CreateBucketAction.
|
// region is valid only for CreateBucketAction.
|
||||||
var region string
|
var region string
|
||||||
@@ -375,11 +413,12 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
|||||||
|
|
||||||
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
|
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
|
||||||
// Anonymous checks are not meant for ListAllBuckets action
|
// Anonymous checks are not meant for ListAllBuckets action
|
||||||
if globalPolicySys.IsAllowed(policy.Args{
|
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
|
Groups: cred.Groups,
|
||||||
Action: action,
|
Action: action,
|
||||||
BucketName: bucket,
|
BucketName: bucket,
|
||||||
ConditionValues: getConditionValues(r, region, "", nil),
|
ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
|
||||||
IsOwner: false,
|
IsOwner: false,
|
||||||
ObjectName: object,
|
ObjectName: object,
|
||||||
}) {
|
}) {
|
||||||
@@ -390,11 +429,12 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
|||||||
if action == policy.ListBucketVersionsAction {
|
if action == policy.ListBucketVersionsAction {
|
||||||
// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
|
// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
|
||||||
// verify as a fallback.
|
// verify as a fallback.
|
||||||
if globalPolicySys.IsAllowed(policy.Args{
|
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
|
Groups: cred.Groups,
|
||||||
Action: policy.ListBucketAction,
|
Action: policy.ListBucketAction,
|
||||||
BucketName: bucket,
|
BucketName: bucket,
|
||||||
ConditionValues: getConditionValues(r, region, "", nil),
|
ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
|
||||||
IsOwner: false,
|
IsOwner: false,
|
||||||
ObjectName: object,
|
ObjectName: object,
|
||||||
}) {
|
}) {
|
||||||
@@ -406,12 +446,12 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
|||||||
return ErrAccessDenied
|
return ErrAccessDenied
|
||||||
}
|
}
|
||||||
if action == policy.DeleteObjectAction && versionID != "" {
|
if action == policy.DeleteObjectAction && versionID != "" {
|
||||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
if !globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: iampolicy.Action(policy.DeleteObjectVersionAction),
|
Action: policy.Action(policy.DeleteObjectVersionAction),
|
||||||
BucketName: bucket,
|
BucketName: bucket,
|
||||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
ObjectName: object,
|
ObjectName: object,
|
||||||
IsOwner: owner,
|
IsOwner: owner,
|
||||||
Claims: cred.Claims,
|
Claims: cred.Claims,
|
||||||
@@ -420,12 +460,12 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
|||||||
return ErrAccessDenied
|
return ErrAccessDenied
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
if globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: iampolicy.Action(action),
|
Action: action,
|
||||||
BucketName: bucket,
|
BucketName: bucket,
|
||||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
ObjectName: object,
|
ObjectName: object,
|
||||||
IsOwner: owner,
|
IsOwner: owner,
|
||||||
Claims: cred.Claims,
|
Claims: cred.Claims,
|
||||||
@@ -437,12 +477,12 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
|||||||
if action == policy.ListBucketVersionsAction {
|
if action == policy.ListBucketVersionsAction {
|
||||||
// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
|
// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
|
||||||
// verify as a fallback.
|
// verify as a fallback.
|
||||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
if globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: iampolicy.ListBucketAction,
|
Action: policy.ListBucketAction,
|
||||||
BucketName: bucket,
|
BucketName: bucket,
|
||||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
ObjectName: object,
|
ObjectName: object,
|
||||||
IsOwner: owner,
|
IsOwner: owner,
|
||||||
Claims: cred.Claims,
|
Claims: cred.Claims,
|
||||||
@@ -528,7 +568,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
|||||||
|
|
||||||
// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
|
// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
|
||||||
// The verification happens implicit during reading.
|
// The verification happens implicit during reading.
|
||||||
reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
|
reader, err := hash.NewReader(ctx, r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return toAPIErrorCode(ctx, err)
|
return toAPIErrorCode(ctx, err)
|
||||||
}
|
}
|
||||||
@@ -545,6 +585,8 @@ var supportedS3AuthTypes = map[authType]struct{}{
|
|||||||
authTypeSignedV2: {},
|
authTypeSignedV2: {},
|
||||||
authTypePostPolicy: {},
|
authTypePostPolicy: {},
|
||||||
authTypeStreamingSigned: {},
|
authTypeStreamingSigned: {},
|
||||||
|
authTypeStreamingSignedTrailer: {},
|
||||||
|
authTypeStreamingUnsignedTrailer: {},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate if the authType is valid and supported.
|
// Validate if the authType is valid and supported.
|
||||||
@@ -553,14 +595,15 @@ func isSupportedS3AuthType(aType authType) bool {
|
|||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
// setAuthHandler to validate authorization header for the incoming request.
|
// setAuthMiddleware to validate authorization header for the incoming request.
|
||||||
func setAuthHandler(h http.Handler) http.Handler {
|
func setAuthMiddleware(h http.Handler) http.Handler {
|
||||||
// handler for validating incoming authorization headers.
|
// handler for validating incoming authorization headers.
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
|
tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
|
||||||
|
|
||||||
aType := getRequestAuthType(r)
|
aType := getRequestAuthType(r)
|
||||||
if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
|
switch aType {
|
||||||
|
case authTypeSigned, authTypeSignedV2, authTypeStreamingSigned, authTypeStreamingSignedTrailer:
|
||||||
// Verify if date headers are set, if not reject the request
|
// Verify if date headers are set, if not reject the request
|
||||||
amzDate, errCode := parseAmzDateHeader(r)
|
amzDate, errCode := parseAmzDateHeader(r)
|
||||||
if errCode != ErrNone {
|
if errCode != ErrNone {
|
||||||
@@ -572,6 +615,7 @@ func setAuthHandler(h http.Handler) http.Handler {
|
|||||||
// All our internal APIs are sensitive towards Date
|
// All our internal APIs are sensitive towards Date
|
||||||
// header, for all requests where Date header is not
|
// header, for all requests where Date header is not
|
||||||
// present we will reject such clients.
|
// present we will reject such clients.
|
||||||
|
defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
|
||||||
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
|
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
|
||||||
atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
|
atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
|
||||||
return
|
return
|
||||||
@@ -585,14 +629,21 @@ func setAuthHandler(h http.Handler) http.Handler {
|
|||||||
tc.ResponseRecorder.LogErrBody = true
|
tc.ResponseRecorder.LogErrBody = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
|
||||||
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
|
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
|
||||||
atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
|
atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if isSupportedS3AuthType(aType) || aType == authTypeJWT || aType == authTypeSTS {
|
|
||||||
h.ServeHTTP(w, r)
|
h.ServeHTTP(w, r)
|
||||||
return
|
return
|
||||||
|
case authTypeJWT, authTypeSTS:
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
if isSupportedS3AuthType(aType) {
|
||||||
|
h.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if ok {
|
if ok {
|
||||||
@@ -600,6 +651,7 @@ func setAuthHandler(h http.Handler) http.Handler {
|
|||||||
tc.ResponseRecorder.LogErrBody = true
|
tc.ResponseRecorder.LogErrBody = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
|
||||||
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
|
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
|
||||||
atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
|
atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
|
||||||
})
|
})
|
||||||
@@ -637,17 +689,17 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
|||||||
return ErrAccessDenied
|
return ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
|
conditions := getConditionValues(r, "", cred)
|
||||||
conditions["object-lock-mode"] = []string{string(retMode)}
|
conditions["object-lock-mode"] = []string{string(retMode)}
|
||||||
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
|
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
|
||||||
if retDays > 0 {
|
if retDays > 0 {
|
||||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
||||||
}
|
}
|
||||||
if retMode == objectlock.RetGovernance && byPassSet {
|
if retMode == objectlock.RetGovernance && byPassSet {
|
||||||
byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
|
byPassSet = globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: iampolicy.BypassGovernanceRetentionAction,
|
Action: policy.BypassGovernanceRetentionAction,
|
||||||
BucketName: bucketName,
|
BucketName: bucketName,
|
||||||
ObjectName: objectName,
|
ObjectName: objectName,
|
||||||
ConditionValues: conditions,
|
ConditionValues: conditions,
|
||||||
@@ -655,10 +707,10 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
|||||||
Claims: cred.Claims,
|
Claims: cred.Claims,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
if globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: iampolicy.PutObjectRetentionAction,
|
Action: policy.PutObjectRetentionAction,
|
||||||
BucketName: bucketName,
|
BucketName: bucketName,
|
||||||
ConditionValues: conditions,
|
ConditionValues: conditions,
|
||||||
ObjectName: objectName,
|
ObjectName: objectName,
|
||||||
@@ -676,7 +728,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
|||||||
// isPutActionAllowed - check if PUT operation is allowed on the resource, this
|
// isPutActionAllowed - check if PUT operation is allowed on the resource, this
|
||||||
// call verifies bucket policies and IAM policies, supports multi user
|
// call verifies bucket policies and IAM policies, supports multi user
|
||||||
// checks etc.
|
// checks etc.
|
||||||
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
|
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||||
var cred auth.Credentials
|
var cred auth.Credentials
|
||||||
var owner bool
|
var owner bool
|
||||||
region := globalSite.Region
|
region := globalSite.Region
|
||||||
@@ -685,7 +737,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
|||||||
return ErrSignatureVersionNotSupported
|
return ErrSignatureVersionNotSupported
|
||||||
case authTypeSignedV2, authTypePresignedV2:
|
case authTypeSignedV2, authTypePresignedV2:
|
||||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||||
case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
|
case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
|
||||||
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
|
||||||
}
|
}
|
||||||
if s3Err != ErrNone {
|
if s3Err != ErrNone {
|
||||||
@@ -699,19 +751,19 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
|||||||
// Do not check for PutObjectRetentionAction permission,
|
// Do not check for PutObjectRetentionAction permission,
|
||||||
// if mode and retain until date are not set.
|
// if mode and retain until date are not set.
|
||||||
// Can happen when bucket has default lock config set
|
// Can happen when bucket has default lock config set
|
||||||
if action == iampolicy.PutObjectRetentionAction &&
|
if action == policy.PutObjectRetentionAction &&
|
||||||
r.Header.Get(xhttp.AmzObjectLockMode) == "" &&
|
r.Header.Get(xhttp.AmzObjectLockMode) == "" &&
|
||||||
r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" {
|
r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" {
|
||||||
return ErrNone
|
return ErrNone
|
||||||
}
|
}
|
||||||
|
|
||||||
if cred.AccessKey == "" {
|
if cred.AccessKey == "" {
|
||||||
if globalPolicySys.IsAllowed(policy.Args{
|
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: policy.Action(action),
|
Action: action,
|
||||||
BucketName: bucketName,
|
BucketName: bucketName,
|
||||||
ConditionValues: getConditionValues(r, "", "", nil),
|
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
||||||
IsOwner: false,
|
IsOwner: false,
|
||||||
ObjectName: objectName,
|
ObjectName: objectName,
|
||||||
}) {
|
}) {
|
||||||
@@ -720,12 +772,12 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
|||||||
return ErrAccessDenied
|
return ErrAccessDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
if globalIAMSys.IsAllowed(policy.Args{
|
||||||
AccountName: cred.AccessKey,
|
AccountName: cred.AccessKey,
|
||||||
Groups: cred.Groups,
|
Groups: cred.Groups,
|
||||||
Action: action,
|
Action: action,
|
||||||
BucketName: bucketName,
|
BucketName: bucketName,
|
||||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
|
ConditionValues: getConditionValues(r, "", cred),
|
||||||
ObjectName: objectName,
|
ObjectName: objectName,
|
||||||
IsOwner: owner,
|
IsOwner: owner,
|
||||||
Claims: cred.Claims,
|
Claims: cred.Claims,
|
||||||
|
|||||||
+13
-12
@@ -28,7 +28,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
iampolicy "github.com/minio/pkg/iam/policy"
|
"github.com/minio/pkg/v2/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
type nullReader struct{}
|
type nullReader struct{}
|
||||||
@@ -287,7 +287,7 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
|
|||||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||||
cred := globalActiveCred
|
cred := globalActiveCred
|
||||||
if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
|
if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
|
||||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
t.Fatalf("Unable to initialized new signed http request %s", err)
|
||||||
}
|
}
|
||||||
return req
|
return req
|
||||||
}
|
}
|
||||||
@@ -298,7 +298,7 @@ func mustNewSignedV2Request(method string, urlStr string, contentLength int64, b
|
|||||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||||
cred := globalActiveCred
|
cred := globalActiveCred
|
||||||
if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil {
|
if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil {
|
||||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
t.Fatalf("Unable to initialized new signed http request %s", err)
|
||||||
}
|
}
|
||||||
return req
|
return req
|
||||||
}
|
}
|
||||||
@@ -309,7 +309,7 @@ func mustNewPresignedV2Request(method string, urlStr string, contentLength int64
|
|||||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||||
cred := globalActiveCred
|
cred := globalActiveCred
|
||||||
if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
|
if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
|
||||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
t.Fatalf("Unable to initialized new signed http request %s", err)
|
||||||
}
|
}
|
||||||
return req
|
return req
|
||||||
}
|
}
|
||||||
@@ -320,7 +320,7 @@ func mustNewPresignedRequest(method string, urlStr string, contentLength int64,
|
|||||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||||
cred := globalActiveCred
|
cred := globalActiveCred
|
||||||
if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
|
if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
|
||||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
t.Fatalf("Unable to initialized new signed http request %s", err)
|
||||||
}
|
}
|
||||||
return req
|
return req
|
||||||
}
|
}
|
||||||
@@ -375,8 +375,6 @@ func TestIsReqAuthenticated(t *testing.T) {
|
|||||||
|
|
||||||
initConfigSubsystem(ctx, objLayer)
|
initConfigSubsystem(ctx, objLayer)
|
||||||
|
|
||||||
globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
|
|
||||||
|
|
||||||
creds, err := auth.CreateCredentials("myuser", "mypassword")
|
creds, err := auth.CreateCredentials("myuser", "mypassword")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unable create credential, %s", err)
|
t.Fatalf("unable create credential, %s", err)
|
||||||
@@ -384,6 +382,8 @@ func TestIsReqAuthenticated(t *testing.T) {
|
|||||||
|
|
||||||
globalActiveCred = creds
|
globalActiveCred = creds
|
||||||
|
|
||||||
|
globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
|
||||||
|
|
||||||
// List of test cases for validating http request authentication.
|
// List of test cases for validating http request authentication.
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
req *http.Request
|
req *http.Request
|
||||||
@@ -443,7 +443,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
|
|||||||
{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||||
}
|
}
|
||||||
for i, testCase := range testCases {
|
for i, testCase := range testCases {
|
||||||
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
|
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
|
||||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -464,9 +464,8 @@ func TestValidateAdminSignature(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
initAllSubsystems(ctx)
|
initAllSubsystems(ctx)
|
||||||
initConfigSubsystem(ctx, objLayer)
|
|
||||||
|
|
||||||
globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
|
initConfigSubsystem(ctx, objLayer)
|
||||||
|
|
||||||
creds, err := auth.CreateCredentials("admin", "mypassword")
|
creds, err := auth.CreateCredentials("admin", "mypassword")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -474,6 +473,8 @@ func TestValidateAdminSignature(t *testing.T) {
|
|||||||
}
|
}
|
||||||
globalActiveCred = creds
|
globalActiveCred = creds
|
||||||
|
|
||||||
|
globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
|
||||||
|
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
AccessKey string
|
AccessKey string
|
||||||
SecretKey string
|
SecretKey string
|
||||||
@@ -490,9 +491,9 @@ func TestValidateAdminSignature(t *testing.T) {
|
|||||||
for i, testCase := range testCases {
|
for i, testCase := range testCases {
|
||||||
req := mustNewRequest(http.MethodGet, "http://localhost:9000/", 0, nil, t)
|
req := mustNewRequest(http.MethodGet, "http://localhost:9000/", 0, nil, t)
|
||||||
if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
|
if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
|
||||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
t.Fatalf("Unable to initialized new signed http request %s", err)
|
||||||
}
|
}
|
||||||
_, _, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
|
_, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
|
||||||
if s3Error != testCase.ErrCode {
|
if s3Error != testCase.ErrCode {
|
||||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i+1, testCase.ErrCode, s3Error)
|
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i+1, testCase.ErrCode, s3Error)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,11 +18,13 @@ func _() {
|
|||||||
_ = x[authTypeSignedV2-7]
|
_ = x[authTypeSignedV2-7]
|
||||||
_ = x[authTypeJWT-8]
|
_ = x[authTypeJWT-8]
|
||||||
_ = x[authTypeSTS-9]
|
_ = x[authTypeSTS-9]
|
||||||
|
_ = x[authTypeStreamingSignedTrailer-10]
|
||||||
|
_ = x[authTypeStreamingUnsignedTrailer-11]
|
||||||
}
|
}
|
||||||
|
|
||||||
const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTS"
|
const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTSStreamingSignedTrailerStreamingUnsignedTrailer"
|
||||||
|
|
||||||
var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81}
|
var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81, 103, 127}
|
||||||
|
|
||||||
func (i authType) String() string {
|
func (i authType) String() string {
|
||||||
if i < 0 || i >= authType(len(_authType_index)-1) {
|
if i < 0 || i >= authType(len(_authType_index)-1) {
|
||||||
|
|||||||
@@ -19,9 +19,14 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
|
"github.com/minio/minio/internal/logger"
|
||||||
|
"github.com/minio/pkg/v2/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
// healTask represents what to heal along with options
|
// healTask represents what to heal along with options
|
||||||
@@ -56,13 +61,43 @@ func activeListeners() int {
|
|||||||
return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
|
return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
|
||||||
}
|
}
|
||||||
|
|
||||||
func waitForLowHTTPReq() {
|
func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) {
|
||||||
var currentIO func() int
|
// No need to wait run at full speed.
|
||||||
if httpServer := newHTTPServerFn(); httpServer != nil {
|
if maxIO <= 0 {
|
||||||
currentIO = httpServer.GetRequestCount
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
globalHealConfig.Wait(currentIO, activeListeners)
|
const waitTick = 100 * time.Millisecond
|
||||||
|
|
||||||
|
tmpMaxWait := maxWait
|
||||||
|
|
||||||
|
for currentIO() >= maxIO {
|
||||||
|
if tmpMaxWait > 0 {
|
||||||
|
if tmpMaxWait < waitTick {
|
||||||
|
time.Sleep(tmpMaxWait)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(waitTick)
|
||||||
|
tmpMaxWait -= waitTick
|
||||||
|
}
|
||||||
|
if tmpMaxWait <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func currentHTTPIO() int {
|
||||||
|
httpServer := newHTTPServerFn()
|
||||||
|
if httpServer == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return httpServer.GetRequestCount() - activeListeners()
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitForLowHTTPReq() {
|
||||||
|
maxIO, maxWait, _ := globalHealConfig.Clone()
|
||||||
|
waitForLowIO(maxIO, maxWait, currentHTTPIO)
|
||||||
}
|
}
|
||||||
|
|
||||||
func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
|
func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
|
||||||
@@ -111,9 +146,19 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
|
|||||||
|
|
||||||
func newHealRoutine() *healRoutine {
|
func newHealRoutine() *healRoutine {
|
||||||
workers := runtime.GOMAXPROCS(0) / 2
|
workers := runtime.GOMAXPROCS(0) / 2
|
||||||
|
|
||||||
|
if envHealWorkers := env.Get("_MINIO_HEAL_WORKERS", ""); envHealWorkers != "" {
|
||||||
|
if numHealers, err := strconv.Atoi(envHealWorkers); err != nil {
|
||||||
|
logger.LogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
|
||||||
|
} else {
|
||||||
|
workers = numHealers
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if workers == 0 {
|
if workers == 0 {
|
||||||
workers = 4
|
workers = 4
|
||||||
}
|
}
|
||||||
|
|
||||||
return &healRoutine{
|
return &healRoutine{
|
||||||
tasks: make(chan healTask),
|
tasks: make(chan healTask),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
|
|||||||
@@ -26,12 +26,15 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/minio/madmin-go/v2"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio-go/v7/pkg/set"
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
|
"github.com/minio/minio/internal/config"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
|
"github.com/minio/pkg/v2/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -44,6 +47,7 @@ const (
|
|||||||
// healingTracker is used to persist healing information during a heal.
|
// healingTracker is used to persist healing information during a heal.
|
||||||
type healingTracker struct {
|
type healingTracker struct {
|
||||||
disk StorageAPI `msg:"-"`
|
disk StorageAPI `msg:"-"`
|
||||||
|
mu *sync.RWMutex `msg:"-"`
|
||||||
|
|
||||||
ID string
|
ID string
|
||||||
PoolIndex int
|
PoolIndex int
|
||||||
@@ -112,22 +116,76 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
|
|||||||
}
|
}
|
||||||
h.disk = disk
|
h.disk = disk
|
||||||
h.ID = diskID
|
h.ID = diskID
|
||||||
|
h.mu = &sync.RWMutex{}
|
||||||
return &h, nil
|
return &h, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newHealingTracker will create a new healing tracker for the disk.
|
// newHealingTracker will create a new healing tracker for the disk.
|
||||||
func newHealingTracker(disk StorageAPI, healID string) *healingTracker {
|
func newHealingTracker() *healingTracker {
|
||||||
diskID, _ := disk.GetDiskID()
|
return &healingTracker{
|
||||||
h := healingTracker{
|
mu: &sync.RWMutex{},
|
||||||
disk: disk,
|
|
||||||
ID: diskID,
|
|
||||||
HealID: healID,
|
|
||||||
Path: disk.String(),
|
|
||||||
Endpoint: disk.Endpoint().String(),
|
|
||||||
Started: time.Now().UTC(),
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
|
||||||
|
h := newHealingTracker()
|
||||||
|
diskID, _ := disk.GetDiskID()
|
||||||
|
h.disk = disk
|
||||||
|
h.ID = diskID
|
||||||
|
h.HealID = healID
|
||||||
|
h.Path = disk.String()
|
||||||
|
h.Endpoint = disk.Endpoint().String()
|
||||||
|
h.Started = time.Now().UTC()
|
||||||
h.PoolIndex, h.SetIndex, h.DiskIndex = disk.GetDiskLoc()
|
h.PoolIndex, h.SetIndex, h.DiskIndex = disk.GetDiskLoc()
|
||||||
return &h
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h healingTracker) getLastUpdate() time.Time {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
|
|
||||||
|
return h.LastUpdate
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h healingTracker) getBucket() string {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
|
|
||||||
|
return h.Bucket
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *healingTracker) setBucket(bucket string) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
|
h.Bucket = bucket
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h healingTracker) getObject() string {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
|
|
||||||
|
return h.Object
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *healingTracker) setObject(object string) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
|
h.Object = object
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *healingTracker) updateProgress(success bool, bytes uint64) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
|
if success {
|
||||||
|
h.ItemsHealed++
|
||||||
|
h.BytesDone += bytes
|
||||||
|
} else {
|
||||||
|
h.ItemsFailed++
|
||||||
|
h.BytesFailed += bytes
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// update will update the tracker on the disk.
|
// update will update the tracker on the disk.
|
||||||
@@ -136,15 +194,18 @@ func (h *healingTracker) update(ctx context.Context) error {
|
|||||||
if h.disk.Healing() == nil {
|
if h.disk.Healing() == nil {
|
||||||
return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
|
return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
|
||||||
}
|
}
|
||||||
|
h.mu.Lock()
|
||||||
if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
|
if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
|
||||||
h.ID, _ = h.disk.GetDiskID()
|
h.ID, _ = h.disk.GetDiskID()
|
||||||
h.PoolIndex, h.SetIndex, h.DiskIndex = h.disk.GetDiskLoc()
|
h.PoolIndex, h.SetIndex, h.DiskIndex = h.disk.GetDiskLoc()
|
||||||
}
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
return h.save(ctx)
|
return h.save(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
// save will unconditionally save the tracker and will be created if not existing.
|
// save will unconditionally save the tracker and will be created if not existing.
|
||||||
func (h *healingTracker) save(ctx context.Context) error {
|
func (h *healingTracker) save(ctx context.Context) error {
|
||||||
|
h.mu.Lock()
|
||||||
if h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
|
if h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
|
||||||
// Attempt to get location.
|
// Attempt to get location.
|
||||||
if api := newObjectLayerFn(); api != nil {
|
if api := newObjectLayerFn(); api != nil {
|
||||||
@@ -155,6 +216,7 @@ func (h *healingTracker) save(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
h.LastUpdate = time.Now().UTC()
|
h.LastUpdate = time.Now().UTC()
|
||||||
htrackerBytes, err := h.MarshalMsg(nil)
|
htrackerBytes, err := h.MarshalMsg(nil)
|
||||||
|
h.mu.Unlock()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -170,12 +232,14 @@ func (h *healingTracker) delete(ctx context.Context) error {
|
|||||||
pathJoin(bucketMetaPrefix, healingTrackerFilename),
|
pathJoin(bucketMetaPrefix, healingTrackerFilename),
|
||||||
DeleteOptions{
|
DeleteOptions{
|
||||||
Recursive: false,
|
Recursive: false,
|
||||||
Force: false,
|
Immediate: false,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *healingTracker) isHealed(bucket string) bool {
|
func (h *healingTracker) isHealed(bucket string) bool {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
for _, v := range h.HealedBuckets {
|
for _, v := range h.HealedBuckets {
|
||||||
if v == bucket {
|
if v == bucket {
|
||||||
return true
|
return true
|
||||||
@@ -186,6 +250,9 @@ func (h *healingTracker) isHealed(bucket string) bool {
|
|||||||
|
|
||||||
// resume will reset progress to the numbers at the start of the bucket.
|
// resume will reset progress to the numbers at the start of the bucket.
|
||||||
func (h *healingTracker) resume() {
|
func (h *healingTracker) resume() {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
h.ItemsHealed = h.ResumeItemsHealed
|
h.ItemsHealed = h.ResumeItemsHealed
|
||||||
h.ItemsFailed = h.ResumeItemsFailed
|
h.ItemsFailed = h.ResumeItemsFailed
|
||||||
h.BytesDone = h.ResumeBytesDone
|
h.BytesDone = h.ResumeBytesDone
|
||||||
@@ -195,6 +262,9 @@ func (h *healingTracker) resume() {
|
|||||||
// bucketDone should be called when a bucket is done healing.
|
// bucketDone should be called when a bucket is done healing.
|
||||||
// Adds the bucket to the list of healed buckets and updates resume numbers.
|
// Adds the bucket to the list of healed buckets and updates resume numbers.
|
||||||
func (h *healingTracker) bucketDone(bucket string) {
|
func (h *healingTracker) bucketDone(bucket string) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
h.ResumeItemsHealed = h.ItemsHealed
|
h.ResumeItemsHealed = h.ItemsHealed
|
||||||
h.ResumeItemsFailed = h.ItemsFailed
|
h.ResumeItemsFailed = h.ItemsFailed
|
||||||
h.ResumeBytesDone = h.BytesDone
|
h.ResumeBytesDone = h.BytesDone
|
||||||
@@ -211,6 +281,9 @@ func (h *healingTracker) bucketDone(bucket string) {
|
|||||||
// setQueuedBuckets will add buckets, but exclude any that is already in h.HealedBuckets.
|
// setQueuedBuckets will add buckets, but exclude any that is already in h.HealedBuckets.
|
||||||
// Order is preserved.
|
// Order is preserved.
|
||||||
func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
|
func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
s := set.CreateStringSet(h.HealedBuckets...)
|
s := set.CreateStringSet(h.HealedBuckets...)
|
||||||
h.QueuedBuckets = make([]string, 0, len(buckets))
|
h.QueuedBuckets = make([]string, 0, len(buckets))
|
||||||
for _, b := range buckets {
|
for _, b := range buckets {
|
||||||
@@ -221,15 +294,22 @@ func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (h *healingTracker) printTo(writer io.Writer) {
|
func (h *healingTracker) printTo(writer io.Writer) {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
|
|
||||||
b, err := json.MarshalIndent(h, "", " ")
|
b, err := json.MarshalIndent(h, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writer.Write([]byte(err.Error()))
|
writer.Write([]byte(err.Error()))
|
||||||
|
return
|
||||||
}
|
}
|
||||||
writer.Write(b)
|
writer.Write(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
// toHealingDisk converts the information to madmin.HealingDisk
|
// toHealingDisk converts the information to madmin.HealingDisk
|
||||||
func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
|
func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
|
|
||||||
return madmin.HealingDisk{
|
return madmin.HealingDisk{
|
||||||
ID: h.ID,
|
ID: h.ID,
|
||||||
HealID: h.HealID,
|
HealID: h.HealID,
|
||||||
@@ -265,13 +345,17 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
|
|||||||
|
|
||||||
initBackgroundHealing(ctx, objAPI) // start quick background healing
|
initBackgroundHealing(ctx, objAPI) // start quick background healing
|
||||||
|
|
||||||
|
if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
|
||||||
globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
|
globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
|
||||||
|
|
||||||
go monitorLocalDisksAndHeal(ctx, z)
|
go monitorLocalDisksAndHeal(ctx, z)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getLocalDisksToHeal() (disksToHeal Endpoints) {
|
func getLocalDisksToHeal() (disksToHeal Endpoints) {
|
||||||
for _, disk := range globalLocalDrives {
|
globalLocalDrivesMu.RLock()
|
||||||
|
localDrives := globalLocalDrives
|
||||||
|
globalLocalDrivesMu.RUnlock()
|
||||||
|
for _, disk := range localDrives {
|
||||||
_, err := disk.GetDiskID()
|
_, err := disk.GetDiskID()
|
||||||
if errors.Is(err, errUnformattedDisk) {
|
if errors.Is(err, errUnformattedDisk) {
|
||||||
disksToHeal = append(disksToHeal, disk.Endpoint())
|
disksToHeal = append(disksToHeal, disk.Endpoint())
|
||||||
@@ -292,14 +376,14 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
|
|||||||
var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||||
|
|
||||||
func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
|
func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
|
||||||
disk, format, err := connectEndpoint(endpoint)
|
disk, format, _, err := connectEndpoint(endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("Error: %w, %s", err, endpoint)
|
return fmt.Errorf("Error: %w, %s", err, endpoint)
|
||||||
}
|
}
|
||||||
defer disk.Close()
|
defer disk.Close()
|
||||||
poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
|
poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
|
||||||
if poolIdx < 0 {
|
if poolIdx < 0 {
|
||||||
return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
|
return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate the set index where the current endpoint belongs
|
// Calculate the set index where the current endpoint belongs
|
||||||
@@ -310,14 +394,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if setIdx < 0 {
|
if setIdx < 0 {
|
||||||
return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
|
return fmt.Errorf("unexpected set index (%d) found for %s", setIdx, disk.Endpoint())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prevent parallel erasure set healing
|
// Prevent parallel erasure set healing
|
||||||
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
|
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
|
||||||
lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
|
lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("Healing of drive '%v' on %s pool, belonging to %s erasure set already in progress: %w",
|
||||||
|
disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1), err)
|
||||||
}
|
}
|
||||||
ctx = lkctx.Context()
|
ctx = lkctx.Context()
|
||||||
defer locker.Unlock(lkctx)
|
defer locker.Unlock(lkctx)
|
||||||
@@ -325,19 +410,20 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
|||||||
// Load healing tracker in this disk
|
// Load healing tracker in this disk
|
||||||
tracker, err := loadHealingTracker(ctx, disk)
|
tracker, err := loadHealingTracker(ctx, disk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// A healing track can be not found when another disk in the same
|
// A healing tracker may be deleted if another disk in the
|
||||||
// erasure set and same healing-id successfully finished healing.
|
// same erasure set with same healing-id successfully finished
|
||||||
if err == errFileNotFound {
|
// healing.
|
||||||
|
if errors.Is(err, errFileNotFound) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
logger.LogIf(ctx, fmt.Errorf("Unable to load a healing tracker on '%s': %w", disk, err))
|
logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
|
||||||
tracker = newHealingTracker(disk, mustGetUUID())
|
tracker = initHealingTracker(disk, mustGetUUID())
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
|
logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
|
||||||
|
|
||||||
buckets, _ := z.ListBuckets(ctx, BucketOptions{})
|
buckets, _ := z.ListBuckets(ctx, BucketOptions{})
|
||||||
// Buckets data are dispersed in multiple zones/sets, make
|
// Buckets data are dispersed in multiple pools/sets, make
|
||||||
// sure to heal all bucket metadata configuration.
|
// sure to heal all bucket metadata configuration.
|
||||||
buckets = append(buckets, BucketInfo{
|
buckets = append(buckets, BucketInfo{
|
||||||
Name: pathJoin(minioMetaBucket, minioConfigPrefix),
|
Name: pathJoin(minioMetaBucket, minioConfigPrefix),
|
||||||
@@ -355,7 +441,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
|||||||
})
|
})
|
||||||
|
|
||||||
if serverDebugLog {
|
if serverDebugLog {
|
||||||
logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
|
logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load bucket totals
|
// Load bucket totals
|
||||||
@@ -378,9 +464,13 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
|||||||
}
|
}
|
||||||
|
|
||||||
if tracker.ItemsFailed > 0 {
|
if tracker.ItemsFailed > 0 {
|
||||||
logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||||
} else {
|
} else {
|
||||||
logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(tracker.QueuedBuckets) > 0 {
|
||||||
|
return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
|
||||||
}
|
}
|
||||||
|
|
||||||
if serverDebugLog {
|
if serverDebugLog {
|
||||||
@@ -388,16 +478,25 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
|
|||||||
logger.Info("\n")
|
logger.Info("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
if tracker.HealID == "" { // HealID is empty only before Feb 2023
|
if tracker.HealID == "" { // HealID was empty only before Feb 2023
|
||||||
logger.LogIf(ctx, tracker.delete(ctx))
|
logger.LogIf(ctx, tracker.delete(ctx))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove .healing.bin from all disks with similar heal-id
|
// Remove .healing.bin from all disks with similar heal-id
|
||||||
for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
|
disks, err := z.GetDisks(poolIdx, setIdx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, disk := range disks {
|
||||||
|
if disk == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
t, err := loadHealingTracker(ctx, disk)
|
t, err := loadHealingTracker(ctx, disk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err != errFileNotFound {
|
if !errors.Is(err, errFileNotFound) {
|
||||||
logger.LogIf(ctx, err)
|
logger.LogIf(ctx, err)
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -442,10 +541,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
|
|||||||
for _, disk := range healDisks {
|
for _, disk := range healDisks {
|
||||||
go func(disk Endpoint) {
|
go func(disk Endpoint) {
|
||||||
globalBackgroundHealState.setDiskHealingStatus(disk, true)
|
globalBackgroundHealState.setDiskHealingStatus(disk, true)
|
||||||
err := healFreshDisk(ctx, z, disk)
|
if err := healFreshDisk(ctx, z, disk); err != nil {
|
||||||
if err != nil {
|
|
||||||
globalBackgroundHealState.setDiskHealingStatus(disk, false)
|
globalBackgroundHealState.setDiskHealingStatus(disk, false)
|
||||||
|
timedout := OperationTimedOut{}
|
||||||
|
if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) {
|
||||||
printEndpointError(disk, err, false)
|
printEndpointError(disk, err, false)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Only upon success pop the healed disk.
|
// Only upon success pop the healed disk.
|
||||||
|
|||||||
@@ -0,0 +1,746 @@
|
|||||||
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio-go/v7/pkg/tags"
|
||||||
|
"github.com/minio/minio/internal/bucket/versioning"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
xioutil "github.com/minio/minio/internal/ioutil"
|
||||||
|
"github.com/minio/minio/internal/logger"
|
||||||
|
"github.com/minio/pkg/v2/env"
|
||||||
|
"github.com/minio/pkg/v2/wildcard"
|
||||||
|
"github.com/minio/pkg/v2/workers"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// expire: # Expire objects that match a condition
|
||||||
|
// apiVersion: v1
|
||||||
|
// bucket: mybucket # Bucket where this batch job will expire matching objects from
|
||||||
|
// prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
|
||||||
|
// rules:
|
||||||
|
// - type: object # regular objects with zero or more older versions
|
||||||
|
// name: NAME # match object names that satisfy the wildcard expression.
|
||||||
|
// olderThan: 70h # match objects older than this value
|
||||||
|
// createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||||
|
// tags:
|
||||||
|
// - key: name
|
||||||
|
// value: pick* # match objects with tag 'name', all values starting with 'pick'
|
||||||
|
// metadata:
|
||||||
|
// - key: content-type
|
||||||
|
// value: image/* # match objects with 'content-type', all values starting with 'image/'
|
||||||
|
// size:
|
||||||
|
// lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
|
||||||
|
// greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
|
||||||
|
// purge:
|
||||||
|
// # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||||
|
// # retainVersions: 5 # keep the latest 5 versions of the object.
|
||||||
|
//
|
||||||
|
// - type: deleted # objects with delete marker as their latest version
|
||||||
|
// name: NAME # match object names that satisfy the wildcard expression.
|
||||||
|
// olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
|
||||||
|
// createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||||
|
// purge:
|
||||||
|
// # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||||
|
// # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
|
||||||
|
//
|
||||||
|
// notify:
|
||||||
|
// endpoint: https://notify.endpoint # notification endpoint to receive job completion status
|
||||||
|
// token: Bearer xxxxx # optional authentication token for the notification endpoint
|
||||||
|
//
|
||||||
|
// retry:
|
||||||
|
// attempts: 10 # number of retries for the job before giving up
|
||||||
|
// delay: 500ms # least amount of delay between each retry
|
||||||
|
|
||||||
|
//go:generate msgp -file $GOFILE
|
||||||
|
|
||||||
|
// BatchJobExpirePurge type accepts non-negative versions to be retained
|
||||||
|
type BatchJobExpirePurge struct {
|
||||||
|
line, col int
|
||||||
|
RetainVersions int `yaml:"retainVersions" json:"retainVersions"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ yaml.Unmarshaler = &BatchJobExpirePurge{}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobExpirePurge extends unmarshal to extract line, col
|
||||||
|
func (p *BatchJobExpirePurge) UnmarshalYAML(val *yaml.Node) error {
|
||||||
|
type purge BatchJobExpirePurge
|
||||||
|
var tmp purge
|
||||||
|
err := val.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*p = BatchJobExpirePurge(tmp)
|
||||||
|
p.line, p.col = val.Line, val.Column
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate returns nil if value is valid, ie > 0.
|
||||||
|
func (p BatchJobExpirePurge) Validate() error {
|
||||||
|
if p.RetainVersions < 0 {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: p.line,
|
||||||
|
col: p.col,
|
||||||
|
msg: "retainVersions must be >= 0",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobExpireFilter holds all the filters currently supported for batch replication
|
||||||
|
type BatchJobExpireFilter struct {
|
||||||
|
line, col int
|
||||||
|
OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
|
||||||
|
CreatedBefore *time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"`
|
||||||
|
Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"`
|
||||||
|
Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"`
|
||||||
|
Size BatchJobSizeFilter `yaml:"size" json:"size"`
|
||||||
|
Type string `yaml:"type" json:"type"`
|
||||||
|
Name string `yaml:"name" json:"name"`
|
||||||
|
Purge BatchJobExpirePurge `yaml:"purge" json:"purge"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ yaml.Unmarshaler = &BatchJobExpireFilter{}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobExpireFilter extends unmarshal to extract line, col
|
||||||
|
// information
|
||||||
|
func (ef *BatchJobExpireFilter) UnmarshalYAML(value *yaml.Node) error {
|
||||||
|
type expFilter BatchJobExpireFilter
|
||||||
|
var tmp expFilter
|
||||||
|
err := value.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*ef = BatchJobExpireFilter(tmp)
|
||||||
|
ef.line, ef.col = value.Line, value.Column
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Matches returns true if obj matches the filter conditions specified in ef.
|
||||||
|
func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
|
||||||
|
switch ef.Type {
|
||||||
|
case BatchJobExpireObject:
|
||||||
|
if obj.DeleteMarker {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
case BatchJobExpireDeleted:
|
||||||
|
if !obj.DeleteMarker {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
// we should never come here, Validate should have caught this.
|
||||||
|
logger.LogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if ef.CreatedBefore != nil && !obj.ModTime.Before(*ef.CreatedBefore) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ef.Tags) > 0 && !obj.DeleteMarker {
|
||||||
|
// Only parse object tags if tags filter is specified.
|
||||||
|
var tagMap map[string]string
|
||||||
|
if len(obj.UserTags) != 0 {
|
||||||
|
t, err := tags.ParseObjectTags(obj.UserTags)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
tagMap = t.ToMap()
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, kv := range ef.Tags {
|
||||||
|
// Object (version) must match all tags specified in
|
||||||
|
// the filter
|
||||||
|
var match bool
|
||||||
|
for t, v := range tagMap {
|
||||||
|
if kv.Match(BatchJobKV{Key: t, Value: v}) {
|
||||||
|
match = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !match {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
if len(ef.Metadata) > 0 && !obj.DeleteMarker {
|
||||||
|
for _, kv := range ef.Metadata {
|
||||||
|
// Object (version) must match all x-amz-meta and
|
||||||
|
// standard metadata headers
|
||||||
|
// specified in the filter
|
||||||
|
var match bool
|
||||||
|
for k, v := range obj.UserDefined {
|
||||||
|
if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// We only need to match x-amz-meta or standardHeaders
|
||||||
|
if kv.Match(BatchJobKV{Key: k, Value: v}) {
|
||||||
|
match = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !match {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ef.Size.InRange(obj.Size)
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// BatchJobExpireObject - object type
|
||||||
|
BatchJobExpireObject string = "object"
|
||||||
|
// BatchJobExpireDeleted - delete marker type
|
||||||
|
BatchJobExpireDeleted string = "deleted"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Validate returns nil if ef has valid fields, validation error otherwise.
|
||||||
|
func (ef BatchJobExpireFilter) Validate() error {
|
||||||
|
switch ef.Type {
|
||||||
|
case BatchJobExpireObject:
|
||||||
|
case BatchJobExpireDeleted:
|
||||||
|
if len(ef.Tags) > 0 || len(ef.Metadata) > 0 {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: ef.line,
|
||||||
|
col: ef.col,
|
||||||
|
msg: "delete type filter can't have tags or metadata",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: ef.line,
|
||||||
|
col: ef.col,
|
||||||
|
msg: "invalid batch-expire type",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tag := range ef.Tags {
|
||||||
|
if err := tag.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, meta := range ef.Metadata {
|
||||||
|
if err := meta.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ef.Purge.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ef.Size.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if ef.CreatedBefore != nil && !ef.CreatedBefore.Before(time.Now()) {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: ef.line,
|
||||||
|
col: ef.col,
|
||||||
|
msg: "CreatedBefore is in the future",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobExpire represents configuration parameters for a batch expiration
|
||||||
|
// job typically supplied in yaml form
|
||||||
|
type BatchJobExpire struct {
|
||||||
|
line, col int
|
||||||
|
APIVersion string `yaml:"apiVersion" json:"apiVersion"`
|
||||||
|
Bucket string `yaml:"bucket" json:"bucket"`
|
||||||
|
Prefix string `yaml:"prefix" json:"prefix"`
|
||||||
|
NotificationCfg BatchJobNotification `yaml:"notify" json:"notify"`
|
||||||
|
Retry BatchJobRetry `yaml:"retry" json:"retry"`
|
||||||
|
Rules []BatchJobExpireFilter `yaml:"rules" json:"rules"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ yaml.Unmarshaler = &BatchJobExpire{}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
|
||||||
|
func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
|
||||||
|
type expireJob BatchJobExpire
|
||||||
|
var tmp expireJob
|
||||||
|
err := val.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*r = BatchJobExpire(tmp)
|
||||||
|
r.line, r.col = val.Line, val.Column
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Notify notifies notification endpoint if configured regarding job failure or success.
|
||||||
|
func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
|
||||||
|
if r.NotificationCfg.Endpoint == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.NotificationCfg.Endpoint, body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.NotificationCfg.Token != "" {
|
||||||
|
req.Header.Set("Authorization", r.NotificationCfg.Token)
|
||||||
|
}
|
||||||
|
|
||||||
|
clnt := http.Client{Transport: getRemoteInstanceTransport}
|
||||||
|
resp, err := clnt.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
xhttp.DrainBody(resp.Body)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return errors.New(resp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expire expires object versions which have already matched supplied filter conditions
|
||||||
|
func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
|
||||||
|
opts := ObjectOptions{
|
||||||
|
PrefixEnabledFn: vc.PrefixEnabled,
|
||||||
|
VersionSuspended: vc.Suspended(),
|
||||||
|
}
|
||||||
|
_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel, opts)
|
||||||
|
return errs
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
batchExpireName = "batch-expire.bin"
|
||||||
|
batchExpireFormat = 1
|
||||||
|
batchExpireVersionV1 = 1
|
||||||
|
batchExpireVersion = batchExpireVersionV1
|
||||||
|
batchExpireAPIVersion = "v1"
|
||||||
|
batchExpireJobDefaultRetries = 3
|
||||||
|
batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
|
||||||
|
)
|
||||||
|
|
||||||
|
type objInfoCache map[string]*ObjectInfo
|
||||||
|
|
||||||
|
func newObjInfoCache() objInfoCache {
|
||||||
|
return objInfoCache(make(map[string]*ObjectInfo))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
|
||||||
|
oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
|
||||||
|
}
|
||||||
|
|
||||||
|
func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
|
||||||
|
oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
|
||||||
|
return oi, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
|
||||||
|
vc, _ := globalBucketVersioningSys.Get(r.Bucket)
|
||||||
|
retryAttempts := r.Retry.Attempts
|
||||||
|
delay := job.Expire.Retry.Delay
|
||||||
|
if delay == 0 {
|
||||||
|
delay = batchExpireJobDefaultRetryDelay
|
||||||
|
}
|
||||||
|
|
||||||
|
var i int
|
||||||
|
for toExpire := range expireCh {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
if i > 0 {
|
||||||
|
if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
|
||||||
|
time.Sleep(wait)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
wk.Take()
|
||||||
|
go func(toExpire []expireObjInfo) {
|
||||||
|
defer wk.Give()
|
||||||
|
|
||||||
|
toExpireAll := make([]ObjectInfo, 0, len(toExpire))
|
||||||
|
toDel := make([]ObjectToDelete, 0, len(toExpire))
|
||||||
|
oiCache := newObjInfoCache()
|
||||||
|
for _, exp := range toExpire {
|
||||||
|
if exp.ExpireAll {
|
||||||
|
toExpireAll = append(toExpireAll, exp.ObjectInfo)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Cache ObjectInfo value via pointers for
|
||||||
|
// subsequent use to track objects which
|
||||||
|
// couldn't be deleted.
|
||||||
|
od := ObjectToDelete{
|
||||||
|
ObjectV: ObjectV{
|
||||||
|
ObjectName: exp.Name,
|
||||||
|
VersionID: exp.VersionID,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
toDel = append(toDel, od)
|
||||||
|
oiCache.Add(od, &exp.ObjectInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
var done bool
|
||||||
|
// DeleteObject(deletePrefix: true) to expire all versions of an object
|
||||||
|
for _, exp := range toExpireAll {
|
||||||
|
var success bool
|
||||||
|
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
done = true
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||||
|
_, err := api.DeleteObject(ctx, exp.Bucket, exp.Name, ObjectOptions{
|
||||||
|
DeletePrefix: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
stopFn(exp, err)
|
||||||
|
logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
|
||||||
|
} else {
|
||||||
|
stopFn(exp, err)
|
||||||
|
success = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ri.trackMultipleObjectVersions(r.Bucket, exp, success)
|
||||||
|
if done {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if done {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteMultiple objects
|
||||||
|
toDelCopy := make([]ObjectToDelete, len(toDel))
|
||||||
|
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||||
|
// Copying toDel to select from objects whose
|
||||||
|
// deletion failed
|
||||||
|
copy(toDelCopy, toDel)
|
||||||
|
var failed int
|
||||||
|
errs := r.Expire(ctx, api, vc, toDel)
|
||||||
|
// reslice toDel in preparation for next retry
|
||||||
|
// attempt
|
||||||
|
toDel = toDel[:0]
|
||||||
|
for i, err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
stopFn(toDelCopy[i], err)
|
||||||
|
logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
|
||||||
|
failed++
|
||||||
|
if attempts == retryAttempts { // all retry attempts failed, record failure
|
||||||
|
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
|
||||||
|
ri.trackCurrentBucketObject(r.Bucket, *oi, false)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
toDel = append(toDel, toDelCopy[i])
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
stopFn(toDelCopy[i], nil)
|
||||||
|
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
|
||||||
|
ri.trackCurrentBucketObject(r.Bucket, *oi, true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
globalBatchJobsMetrics.save(ri.JobID, ri)
|
||||||
|
|
||||||
|
if failed == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add a delay between retry attempts
|
||||||
|
if attempts < retryAttempts {
|
||||||
|
time.Sleep(delay)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}(toExpire)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type expireObjInfo struct {
|
||||||
|
ObjectInfo
|
||||||
|
ExpireAll bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start the batch expiration job, resumes if there was a pending job via "job.ID"
|
||||||
|
func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
|
||||||
|
ri := &batchJobInfo{
|
||||||
|
JobID: job.ID,
|
||||||
|
JobType: string(job.Type()),
|
||||||
|
StartTime: job.Started,
|
||||||
|
}
|
||||||
|
if err := ri.load(ctx, api, job); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
globalBatchJobsMetrics.save(job.ID, ri)
|
||||||
|
lastObject := ri.Object
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
|
||||||
|
workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_EXPIRATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
wk, err := workers.New(workerSize)
|
||||||
|
if err != nil {
|
||||||
|
// invalid worker size.
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
results := make(chan ObjectInfo, workerSize)
|
||||||
|
if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
|
||||||
|
Marker: lastObject,
|
||||||
|
LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions
|
||||||
|
VersionsSort: WalkVersionsSortDesc,
|
||||||
|
}); err != nil {
|
||||||
|
// Do not need to retry if we can't list objects on source.
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Goroutine to periodically save batch-expire job's in-memory state
|
||||||
|
saverQuitCh := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
saveTicker := time.NewTicker(10 * time.Second)
|
||||||
|
defer saveTicker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-saveTicker.C:
|
||||||
|
// persist in-memory state to disk after every 10secs.
|
||||||
|
logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
|
||||||
|
|
||||||
|
case <-ctx.Done():
|
||||||
|
// persist in-memory state immediately before exiting due to context cancellation.
|
||||||
|
logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
|
||||||
|
return
|
||||||
|
|
||||||
|
case <-saverQuitCh:
|
||||||
|
// persist in-memory state immediately to disk.
|
||||||
|
logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
expireCh := make(chan []expireObjInfo, workerSize)
|
||||||
|
go batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
|
||||||
|
|
||||||
|
var (
|
||||||
|
prevObj ObjectInfo
|
||||||
|
matchedFilter BatchJobExpireFilter
|
||||||
|
versionsCount int
|
||||||
|
toDel []expireObjInfo
|
||||||
|
)
|
||||||
|
for result := range results {
|
||||||
|
// Apply filter to find the matching rule to apply expiry
|
||||||
|
// actions accordingly.
|
||||||
|
// nolint:gocritic
|
||||||
|
if result.IsLatest {
|
||||||
|
// send down filtered entries to be deleted using
|
||||||
|
// DeleteObjects method
|
||||||
|
if len(toDel) > 10 { // batch up to 10 objects/versions to be expired simultaneously.
|
||||||
|
xfer := make([]expireObjInfo, len(toDel))
|
||||||
|
copy(xfer, toDel)
|
||||||
|
|
||||||
|
var done bool
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
done = true
|
||||||
|
case expireCh <- xfer:
|
||||||
|
toDel = toDel[:0] // resetting toDel
|
||||||
|
}
|
||||||
|
if done {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var match BatchJobExpireFilter
|
||||||
|
var found bool
|
||||||
|
for _, rule := range r.Rules {
|
||||||
|
if rule.Matches(result, now) {
|
||||||
|
match = rule
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
prevObj = result
|
||||||
|
matchedFilter = match
|
||||||
|
versionsCount = 1
|
||||||
|
// Include the latest version
|
||||||
|
if matchedFilter.Purge.RetainVersions == 0 {
|
||||||
|
toDel = append(toDel, expireObjInfo{
|
||||||
|
ObjectInfo: result,
|
||||||
|
ExpireAll: true,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else if prevObj.Name == result.Name {
|
||||||
|
if matchedFilter.Purge.RetainVersions == 0 {
|
||||||
|
continue // including latest version in toDel suffices, skipping other versions
|
||||||
|
}
|
||||||
|
versionsCount++
|
||||||
|
} else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if versionsCount <= matchedFilter.Purge.RetainVersions {
|
||||||
|
continue // retain versions
|
||||||
|
}
|
||||||
|
toDel = append(toDel, expireObjInfo{
|
||||||
|
ObjectInfo: result,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// Send any remaining objects downstream
|
||||||
|
if len(toDel) > 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case expireCh <- toDel:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
xioutil.SafeClose(expireCh)
|
||||||
|
|
||||||
|
wk.Wait() // waits for all expire goroutines to complete
|
||||||
|
|
||||||
|
ri.Complete = ri.ObjectsFailed == 0
|
||||||
|
ri.Failed = ri.ObjectsFailed > 0
|
||||||
|
globalBatchJobsMetrics.save(job.ID, ri)
|
||||||
|
|
||||||
|
// Close the saverQuitCh - this also triggers saving in-memory state
|
||||||
|
// immediately one last time before we exit this method.
|
||||||
|
xioutil.SafeClose(saverQuitCh)
|
||||||
|
|
||||||
|
// Notify expire jobs final status to the configured endpoint
|
||||||
|
buf, _ := json.Marshal(ri)
|
||||||
|
if err := r.Notify(context.Background(), bytes.NewReader(buf)); err != nil {
|
||||||
|
logger.LogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
//msgp:ignore batchExpireJobError
|
||||||
|
type batchExpireJobError struct {
|
||||||
|
Code string
|
||||||
|
Description string
|
||||||
|
HTTPStatusCode int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e batchExpireJobError) Error() string {
|
||||||
|
return e.Description
|
||||||
|
}
|
||||||
|
|
||||||
|
// maxBatchRules maximum number of rules a batch-expiry job supports
|
||||||
|
const maxBatchRules = 50
|
||||||
|
|
||||||
|
// Validate validates the job definition input
|
||||||
|
func (r *BatchJobExpire) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
|
||||||
|
if r == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.APIVersion != batchExpireAPIVersion {
|
||||||
|
return batchExpireJobError{
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Unsupported batch expire API version",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Bucket == "" {
|
||||||
|
return batchExpireJobError{
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Bucket argument missing",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
|
||||||
|
if isErrBucketNotFound(err) {
|
||||||
|
return batchExpireJobError{
|
||||||
|
Code: "NoSuchSourceBucket",
|
||||||
|
Description: "The specified source bucket does not exist",
|
||||||
|
HTTPStatusCode: http.StatusNotFound,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(r.Rules) > maxBatchRules {
|
||||||
|
return batchExpireJobError{
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: "Too many rules. Batch expire job can't have more than 100 rules",
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, rule := range r.Rules {
|
||||||
|
if err := rule.Validate(); err != nil {
|
||||||
|
return batchExpireJobError{
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: fmt.Sprintf("Invalid batch expire rule: %s", err),
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := r.Retry.Validate(); err != nil {
|
||||||
|
return batchExpireJobError{
|
||||||
|
Code: "InvalidArgument",
|
||||||
|
Description: fmt.Sprintf("Invalid batch expire retry configuration: %s", err),
|
||||||
|
HTTPStatusCode: http.StatusBadRequest,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,856 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *BatchJobExpire) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "APIVersion":
|
||||||
|
z.APIVersion, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "APIVersion")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Bucket":
|
||||||
|
z.Bucket, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Prefix":
|
||||||
|
z.Prefix, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Prefix")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "NotificationCfg":
|
||||||
|
err = z.NotificationCfg.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "NotificationCfg")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Retry":
|
||||||
|
err = z.Retry.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Retry")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Rules":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, err = dc.ReadArrayHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Rules")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Rules) >= int(zb0002) {
|
||||||
|
z.Rules = (z.Rules)[:zb0002]
|
||||||
|
} else {
|
||||||
|
z.Rules = make([]BatchJobExpireFilter, zb0002)
|
||||||
|
}
|
||||||
|
for za0001 := range z.Rules {
|
||||||
|
err = z.Rules[za0001].DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Rules", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeMsg implements msgp.Encodable
|
||||||
|
func (z *BatchJobExpire) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
|
// map header, size 6
|
||||||
|
// write "APIVersion"
|
||||||
|
err = en.Append(0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.APIVersion)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "APIVersion")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Bucket"
|
||||||
|
err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Bucket)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Prefix"
|
||||||
|
err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Prefix)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Prefix")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "NotificationCfg"
|
||||||
|
err = en.Append(0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = z.NotificationCfg.EncodeMsg(en)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "NotificationCfg")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Retry"
|
||||||
|
err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = z.Retry.EncodeMsg(en)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Retry")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Rules"
|
||||||
|
err = en.Append(0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteArrayHeader(uint32(len(z.Rules)))
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Rules")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0001 := range z.Rules {
|
||||||
|
err = z.Rules[za0001].EncodeMsg(en)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Rules", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalMsg implements msgp.Marshaler
|
||||||
|
func (z *BatchJobExpire) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
|
o = msgp.Require(b, z.Msgsize())
|
||||||
|
// map header, size 6
|
||||||
|
// string "APIVersion"
|
||||||
|
o = append(o, 0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
|
||||||
|
o = msgp.AppendString(o, z.APIVersion)
|
||||||
|
// string "Bucket"
|
||||||
|
o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
|
||||||
|
o = msgp.AppendString(o, z.Bucket)
|
||||||
|
// string "Prefix"
|
||||||
|
o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
|
||||||
|
o = msgp.AppendString(o, z.Prefix)
|
||||||
|
// string "NotificationCfg"
|
||||||
|
o = append(o, 0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
|
||||||
|
o, err = z.NotificationCfg.MarshalMsg(o)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "NotificationCfg")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// string "Retry"
|
||||||
|
o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
|
||||||
|
o, err = z.Retry.MarshalMsg(o)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Retry")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// string "Rules"
|
||||||
|
o = append(o, 0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
|
||||||
|
o = msgp.AppendArrayHeader(o, uint32(len(z.Rules)))
|
||||||
|
for za0001 := range z.Rules {
|
||||||
|
o, err = z.Rules[za0001].MarshalMsg(o)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Rules", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *BatchJobExpire) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "APIVersion":
|
||||||
|
z.APIVersion, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "APIVersion")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Bucket":
|
||||||
|
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Prefix":
|
||||||
|
z.Prefix, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Prefix")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "NotificationCfg":
|
||||||
|
bts, err = z.NotificationCfg.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "NotificationCfg")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Retry":
|
||||||
|
bts, err = z.Retry.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Retry")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Rules":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Rules")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Rules) >= int(zb0002) {
|
||||||
|
z.Rules = (z.Rules)[:zb0002]
|
||||||
|
} else {
|
||||||
|
z.Rules = make([]BatchJobExpireFilter, zb0002)
|
||||||
|
}
|
||||||
|
for za0001 := range z.Rules {
|
||||||
|
bts, err = z.Rules[za0001].UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Rules", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z *BatchJobExpire) Msgsize() (s int) {
|
||||||
|
s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 16 + z.NotificationCfg.Msgsize() + 6 + z.Retry.Msgsize() + 6 + msgp.ArrayHeaderSize
|
||||||
|
for za0001 := range z.Rules {
|
||||||
|
s += z.Rules[za0001].Msgsize()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "OlderThan":
|
||||||
|
z.OlderThan, err = dc.ReadDuration()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "OlderThan")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "CreatedBefore":
|
||||||
|
if dc.IsNil() {
|
||||||
|
err = dc.ReadNil()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "CreatedBefore")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.CreatedBefore = nil
|
||||||
|
} else {
|
||||||
|
if z.CreatedBefore == nil {
|
||||||
|
z.CreatedBefore = new(time.Time)
|
||||||
|
}
|
||||||
|
*z.CreatedBefore, err = dc.ReadTime()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "CreatedBefore")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "Tags":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, err = dc.ReadArrayHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Tags")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Tags) >= int(zb0002) {
|
||||||
|
z.Tags = (z.Tags)[:zb0002]
|
||||||
|
} else {
|
||||||
|
z.Tags = make([]BatchJobKV, zb0002)
|
||||||
|
}
|
||||||
|
for za0001 := range z.Tags {
|
||||||
|
err = z.Tags[za0001].DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Tags", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "Metadata":
|
||||||
|
var zb0003 uint32
|
||||||
|
zb0003, err = dc.ReadArrayHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Metadata")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Metadata) >= int(zb0003) {
|
||||||
|
z.Metadata = (z.Metadata)[:zb0003]
|
||||||
|
} else {
|
||||||
|
z.Metadata = make([]BatchJobKV, zb0003)
|
||||||
|
}
|
||||||
|
for za0002 := range z.Metadata {
|
||||||
|
err = z.Metadata[za0002].DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Metadata", za0002)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "Size":
|
||||||
|
err = z.Size.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Size")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Type":
|
||||||
|
z.Type, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Type")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Name":
|
||||||
|
z.Name, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Name")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Purge":
|
||||||
|
var zb0004 uint32
|
||||||
|
zb0004, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0004 > 0 {
|
||||||
|
zb0004--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "RetainVersions":
|
||||||
|
z.Purge.RetainVersions, err = dc.ReadInt()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge", "RetainVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeMsg implements msgp.Encodable
|
||||||
|
func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
|
// map header, size 8
|
||||||
|
// write "OlderThan"
|
||||||
|
err = en.Append(0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteDuration(z.OlderThan)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "OlderThan")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "CreatedBefore"
|
||||||
|
err = en.Append(0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if z.CreatedBefore == nil {
|
||||||
|
err = en.WriteNil()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
err = en.WriteTime(*z.CreatedBefore)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "CreatedBefore")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// write "Tags"
|
||||||
|
err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteArrayHeader(uint32(len(z.Tags)))
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Tags")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0001 := range z.Tags {
|
||||||
|
err = z.Tags[za0001].EncodeMsg(en)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Tags", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// write "Metadata"
|
||||||
|
err = en.Append(0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteArrayHeader(uint32(len(z.Metadata)))
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Metadata")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0002 := range z.Metadata {
|
||||||
|
err = z.Metadata[za0002].EncodeMsg(en)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Metadata", za0002)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// write "Size"
|
||||||
|
err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = z.Size.EncodeMsg(en)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Size")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Type"
|
||||||
|
err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Type)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Type")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Name"
|
||||||
|
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Name)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Name")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "Purge"
|
||||||
|
err = en.Append(0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// map header, size 1
|
||||||
|
// write "RetainVersions"
|
||||||
|
err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt(z.Purge.RetainVersions)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge", "RetainVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalMsg implements msgp.Marshaler
|
||||||
|
func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
|
o = msgp.Require(b, z.Msgsize())
|
||||||
|
// map header, size 8
|
||||||
|
// string "OlderThan"
|
||||||
|
o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
|
||||||
|
o = msgp.AppendDuration(o, z.OlderThan)
|
||||||
|
// string "CreatedBefore"
|
||||||
|
o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
|
||||||
|
if z.CreatedBefore == nil {
|
||||||
|
o = msgp.AppendNil(o)
|
||||||
|
} else {
|
||||||
|
o = msgp.AppendTime(o, *z.CreatedBefore)
|
||||||
|
}
|
||||||
|
// string "Tags"
|
||||||
|
o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73)
|
||||||
|
o = msgp.AppendArrayHeader(o, uint32(len(z.Tags)))
|
||||||
|
for za0001 := range z.Tags {
|
||||||
|
o, err = z.Tags[za0001].MarshalMsg(o)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Tags", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// string "Metadata"
|
||||||
|
o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
|
||||||
|
o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata)))
|
||||||
|
for za0002 := range z.Metadata {
|
||||||
|
o, err = z.Metadata[za0002].MarshalMsg(o)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Metadata", za0002)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// string "Size"
|
||||||
|
o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
||||||
|
o, err = z.Size.MarshalMsg(o)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Size")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// string "Type"
|
||||||
|
o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
|
||||||
|
o = msgp.AppendString(o, z.Type)
|
||||||
|
// string "Name"
|
||||||
|
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||||
|
o = msgp.AppendString(o, z.Name)
|
||||||
|
// string "Purge"
|
||||||
|
o = append(o, 0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
|
||||||
|
// map header, size 1
|
||||||
|
// string "RetainVersions"
|
||||||
|
o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||||
|
o = msgp.AppendInt(o, z.Purge.RetainVersions)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "OlderThan":
|
||||||
|
z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "OlderThan")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "CreatedBefore":
|
||||||
|
if msgp.IsNil(bts) {
|
||||||
|
bts, err = msgp.ReadNilBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.CreatedBefore = nil
|
||||||
|
} else {
|
||||||
|
if z.CreatedBefore == nil {
|
||||||
|
z.CreatedBefore = new(time.Time)
|
||||||
|
}
|
||||||
|
*z.CreatedBefore, bts, err = msgp.ReadTimeBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "CreatedBefore")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "Tags":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Tags")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Tags) >= int(zb0002) {
|
||||||
|
z.Tags = (z.Tags)[:zb0002]
|
||||||
|
} else {
|
||||||
|
z.Tags = make([]BatchJobKV, zb0002)
|
||||||
|
}
|
||||||
|
for za0001 := range z.Tags {
|
||||||
|
bts, err = z.Tags[za0001].UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Tags", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "Metadata":
|
||||||
|
var zb0003 uint32
|
||||||
|
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Metadata")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Metadata) >= int(zb0003) {
|
||||||
|
z.Metadata = (z.Metadata)[:zb0003]
|
||||||
|
} else {
|
||||||
|
z.Metadata = make([]BatchJobKV, zb0003)
|
||||||
|
}
|
||||||
|
for za0002 := range z.Metadata {
|
||||||
|
bts, err = z.Metadata[za0002].UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Metadata", za0002)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "Size":
|
||||||
|
bts, err = z.Size.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Size")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Type":
|
||||||
|
z.Type, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Type")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Name":
|
||||||
|
z.Name, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Name")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Purge":
|
||||||
|
var zb0004 uint32
|
||||||
|
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0004 > 0 {
|
||||||
|
zb0004--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "RetainVersions":
|
||||||
|
z.Purge.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge", "RetainVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Purge")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z *BatchJobExpireFilter) Msgsize() (s int) {
|
||||||
|
s = 1 + 10 + msgp.DurationSize + 14
|
||||||
|
if z.CreatedBefore == nil {
|
||||||
|
s += msgp.NilSize
|
||||||
|
} else {
|
||||||
|
s += msgp.TimeSize
|
||||||
|
}
|
||||||
|
s += 5 + msgp.ArrayHeaderSize
|
||||||
|
for za0001 := range z.Tags {
|
||||||
|
s += z.Tags[za0001].Msgsize()
|
||||||
|
}
|
||||||
|
s += 9 + msgp.ArrayHeaderSize
|
||||||
|
for za0002 := range z.Metadata {
|
||||||
|
s += z.Metadata[za0002].Msgsize()
|
||||||
|
}
|
||||||
|
s += 5 + z.Size.Msgsize() + 5 + msgp.StringPrefixSize + len(z.Type) + 5 + msgp.StringPrefixSize + len(z.Name) + 6 + 1 + 15 + msgp.IntSize
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *BatchJobExpirePurge) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "RetainVersions":
|
||||||
|
z.RetainVersions, err = dc.ReadInt()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "RetainVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeMsg implements msgp.Encodable
|
||||||
|
func (z BatchJobExpirePurge) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
|
// map header, size 1
|
||||||
|
// write "RetainVersions"
|
||||||
|
err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt(z.RetainVersions)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "RetainVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalMsg implements msgp.Marshaler
|
||||||
|
func (z BatchJobExpirePurge) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
|
o = msgp.Require(b, z.Msgsize())
|
||||||
|
// map header, size 1
|
||||||
|
// string "RetainVersions"
|
||||||
|
o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||||
|
o = msgp.AppendInt(o, z.RetainVersions)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *BatchJobExpirePurge) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "RetainVersions":
|
||||||
|
z.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "RetainVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z BatchJobExpirePurge) Msgsize() (s int) {
|
||||||
|
s = 1 + 15 + msgp.IntSize
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -0,0 +1,349 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobExpire(t *testing.T) {
|
||||||
|
v := BatchJobExpire{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobExpire(b *testing.B) {
|
||||||
|
v := BatchJobExpire{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobExpire(b *testing.B) {
|
||||||
|
v := BatchJobExpire{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobExpire(b *testing.B) {
|
||||||
|
v := BatchJobExpire{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobExpire(t *testing.T) {
|
||||||
|
v := BatchJobExpire{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobExpire Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobExpire{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobExpire(b *testing.B) {
|
||||||
|
v := BatchJobExpire{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobExpire(b *testing.B) {
|
||||||
|
v := BatchJobExpire{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobExpireFilter(t *testing.T) {
|
||||||
|
v := BatchJobExpireFilter{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobExpireFilter(b *testing.B) {
|
||||||
|
v := BatchJobExpireFilter{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobExpireFilter(b *testing.B) {
|
||||||
|
v := BatchJobExpireFilter{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobExpireFilter(b *testing.B) {
|
||||||
|
v := BatchJobExpireFilter{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobExpireFilter(t *testing.T) {
|
||||||
|
v := BatchJobExpireFilter{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobExpireFilter Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobExpireFilter{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobExpireFilter(b *testing.B) {
|
||||||
|
v := BatchJobExpireFilter{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobExpireFilter(b *testing.B) {
|
||||||
|
v := BatchJobExpireFilter{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobExpirePurge(t *testing.T) {
|
||||||
|
v := BatchJobExpirePurge{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobExpirePurge(b *testing.B) {
|
||||||
|
v := BatchJobExpirePurge{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobExpirePurge(b *testing.B) {
|
||||||
|
v := BatchJobExpirePurge{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobExpirePurge(b *testing.B) {
|
||||||
|
v := BatchJobExpirePurge{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobExpirePurge(t *testing.T) {
|
||||||
|
v := BatchJobExpirePurge{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobExpirePurge Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobExpirePurge{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobExpirePurge(b *testing.B) {
|
||||||
|
v := BatchJobExpirePurge{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobExpirePurge(b *testing.B) {
|
||||||
|
v := BatchJobExpirePurge{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseBatchJobExpire(t *testing.T) {
|
||||||
|
expireYaml := `
|
||||||
|
expire: # Expire objects that match a condition
|
||||||
|
apiVersion: v1
|
||||||
|
bucket: mybucket # Bucket where this batch job will expire matching objects from
|
||||||
|
prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
|
||||||
|
rules:
|
||||||
|
- type: object # regular objects with zero or more older versions
|
||||||
|
name: NAME # match object names that satisfy the wildcard expression.
|
||||||
|
olderThan: 70h # match objects older than this value
|
||||||
|
createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||||
|
tags:
|
||||||
|
- key: name
|
||||||
|
value: pick* # match objects with tag 'name', all values starting with 'pick'
|
||||||
|
metadata:
|
||||||
|
- key: content-type
|
||||||
|
value: image/* # match objects with 'content-type', all values starting with 'image/'
|
||||||
|
size:
|
||||||
|
lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
|
||||||
|
greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
|
||||||
|
purge:
|
||||||
|
# retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||||
|
# retainVersions: 5 # keep the latest 5 versions of the object.
|
||||||
|
|
||||||
|
- type: deleted # objects with delete marker as their latest version
|
||||||
|
name: NAME # match object names that satisfy the wildcard expression.
|
||||||
|
olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
|
||||||
|
createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||||
|
purge:
|
||||||
|
# retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||||
|
# retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
|
||||||
|
|
||||||
|
notify:
|
||||||
|
endpoint: https://notify.endpoint # notification endpoint to receive job completion status
|
||||||
|
token: Bearer xxxxx # optional authentication token for the notification endpoint
|
||||||
|
|
||||||
|
retry:
|
||||||
|
attempts: 10 # number of retries for the job before giving up
|
||||||
|
delay: 500ms # least amount of delay between each retry
|
||||||
|
`
|
||||||
|
var job BatchJobRequest
|
||||||
|
err := yaml.UnmarshalStrict([]byte(expireYaml), &job)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal("Failed to parse batch-job-expire yaml", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+1150
-337
File diff suppressed because it is too large
Load Diff
+138
-2195
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,287 @@
|
|||||||
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/dustin/go-humanize"
|
||||||
|
"github.com/minio/pkg/v2/wildcard"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:generate msgp -file $GOFILE
|
||||||
|
//msgp:ignore BatchJobYamlErr
|
||||||
|
|
||||||
|
// BatchJobYamlErr can be used to return yaml validation errors with line,
|
||||||
|
// column information guiding user to fix syntax errors
|
||||||
|
type BatchJobYamlErr struct {
|
||||||
|
line, col int
|
||||||
|
msg string
|
||||||
|
}
|
||||||
|
|
||||||
|
// message returns the error message excluding line, col information.
|
||||||
|
// Intended to be used in unit tests.
|
||||||
|
func (b BatchJobYamlErr) message() string {
|
||||||
|
return b.msg
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error implements Error interface
|
||||||
|
func (b BatchJobYamlErr) Error() string {
|
||||||
|
return fmt.Sprintf("%s\n Hint: error near line: %d, col: %d", b.msg, b.line, b.col)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobKV is a key-value data type which supports wildcard matching
|
||||||
|
type BatchJobKV struct {
|
||||||
|
line, col int
|
||||||
|
Key string `yaml:"key" json:"key"`
|
||||||
|
Value string `yaml:"value" json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ yaml.Unmarshaler = &BatchJobKV{}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobKV extends default unmarshal to extract line, col information.
|
||||||
|
func (kv *BatchJobKV) UnmarshalYAML(val *yaml.Node) error {
|
||||||
|
type jobKV BatchJobKV
|
||||||
|
var tmp jobKV
|
||||||
|
err := val.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*kv = BatchJobKV(tmp)
|
||||||
|
kv.line, kv.col = val.Line, val.Column
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate returns an error if key is empty
|
||||||
|
func (kv BatchJobKV) Validate() error {
|
||||||
|
if kv.Key == "" {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: kv.line,
|
||||||
|
col: kv.col,
|
||||||
|
msg: "key can't be empty",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty indicates if kv is not set
|
||||||
|
func (kv BatchJobKV) Empty() bool {
|
||||||
|
return kv.Key == "" && kv.Value == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Match matches input kv with kv, value will be wildcard matched depending on the user input
|
||||||
|
func (kv BatchJobKV) Match(ikv BatchJobKV) bool {
|
||||||
|
if kv.Empty() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if strings.EqualFold(kv.Key, ikv.Key) {
|
||||||
|
return wildcard.Match(kv.Value, ikv.Value)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobNotification stores notification endpoint and token information.
|
||||||
|
// Used by batch jobs to notify of their status.
|
||||||
|
type BatchJobNotification struct {
|
||||||
|
line, col int
|
||||||
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||||
|
Token string `yaml:"token" json:"token"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ yaml.Unmarshaler = &BatchJobNotification{}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobNotification extends unmarshal to extract line, column information
|
||||||
|
func (b *BatchJobNotification) UnmarshalYAML(val *yaml.Node) error {
|
||||||
|
type notification BatchJobNotification
|
||||||
|
var tmp notification
|
||||||
|
err := val.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*b = BatchJobNotification(tmp)
|
||||||
|
b.line, b.col = val.Line, val.Column
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobRetry stores retry configuration used in the event of failures.
|
||||||
|
type BatchJobRetry struct {
|
||||||
|
line, col int
|
||||||
|
Attempts int `yaml:"attempts" json:"attempts"` // number of retry attempts
|
||||||
|
Delay time.Duration `yaml:"delay" json:"delay"` // delay between each retries
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ yaml.Unmarshaler = &BatchJobRetry{}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobRetry extends unmarshal to extract line, column information
|
||||||
|
func (r *BatchJobRetry) UnmarshalYAML(val *yaml.Node) error {
|
||||||
|
type retry BatchJobRetry
|
||||||
|
var tmp retry
|
||||||
|
err := val.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*r = BatchJobRetry(tmp)
|
||||||
|
r.line, r.col = val.Line, val.Column
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate validates input replicate retries.
|
||||||
|
func (r BatchJobRetry) Validate() error {
|
||||||
|
if r.Attempts < 0 {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: r.line,
|
||||||
|
col: r.col,
|
||||||
|
msg: "Invalid arguments specified",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Delay < 0 {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: r.line,
|
||||||
|
col: r.col,
|
||||||
|
msg: "Invalid arguments specified",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// # snowball based archive transfer is by default enabled when source
|
||||||
|
// # is local and target is remote which is also minio.
|
||||||
|
// snowball:
|
||||||
|
// disable: false # optionally turn-off snowball archive transfer
|
||||||
|
// batch: 100 # upto this many objects per archive
|
||||||
|
// inmemory: true # indicates if the archive must be staged locally or in-memory
|
||||||
|
// compress: true # S2/Snappy compressed archive
|
||||||
|
// smallerThan: 5MiB # create archive for all objects smaller than 5MiB
|
||||||
|
// skipErrs: false # skips any source side read() errors
|
||||||
|
|
||||||
|
// BatchJobSnowball describes the snowball feature when replicating objects from a local source to a remote target
|
||||||
|
type BatchJobSnowball struct {
|
||||||
|
line, col int
|
||||||
|
Disable *bool `yaml:"disable" json:"disable"`
|
||||||
|
Batch *int `yaml:"batch" json:"batch"`
|
||||||
|
InMemory *bool `yaml:"inmemory" json:"inmemory"`
|
||||||
|
Compress *bool `yaml:"compress" json:"compress"`
|
||||||
|
SmallerThan *string `yaml:"smallerThan" json:"smallerThan"`
|
||||||
|
SkipErrs *bool `yaml:"skipErrs" json:"skipErrs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ yaml.Unmarshaler = &BatchJobSnowball{}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobSnowball extends unmarshal to extract line, column information
|
||||||
|
func (b *BatchJobSnowball) UnmarshalYAML(val *yaml.Node) error {
|
||||||
|
type snowball BatchJobSnowball
|
||||||
|
var tmp snowball
|
||||||
|
err := val.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*b = BatchJobSnowball(tmp)
|
||||||
|
b.line, b.col = val.Line, val.Column
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate the snowball parameters in the job description
|
||||||
|
func (b BatchJobSnowball) Validate() error {
|
||||||
|
if *b.Batch <= 0 {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: b.line,
|
||||||
|
col: b.col,
|
||||||
|
msg: "batch number should be non positive zero",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err := humanize.ParseBytes(*b.SmallerThan)
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: b.line,
|
||||||
|
col: b.col,
|
||||||
|
msg: err.Error(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan
|
||||||
|
type BatchJobSizeFilter struct {
|
||||||
|
line, col int
|
||||||
|
UpperBound BatchJobSize `yaml:"lessThan" json:"lessThan"`
|
||||||
|
LowerBound BatchJobSize `yaml:"greaterThan" json:"greaterThan"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalYAML - BatchJobSizeFilter extends unmarshal to extract line, column information
|
||||||
|
func (sf *BatchJobSizeFilter) UnmarshalYAML(val *yaml.Node) error {
|
||||||
|
type sizeFilter BatchJobSizeFilter
|
||||||
|
var tmp sizeFilter
|
||||||
|
err := val.Decode(&tmp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
*sf = BatchJobSizeFilter(tmp)
|
||||||
|
sf.line, sf.col = val.Line, val.Column
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// InRange returns true in the following cases and false otherwise,
|
||||||
|
// - sf.LowerBound < sz, when sf.LowerBound alone is specified
|
||||||
|
// - sz < sf.UpperBound, when sf.UpperBound alone is specified
|
||||||
|
// - sf.LowerBound < sz < sf.UpperBound when both are specified,
|
||||||
|
func (sf BatchJobSizeFilter) InRange(sz int64) bool {
|
||||||
|
if sf.UpperBound > 0 && sz > int64(sf.UpperBound) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if sf.LowerBound > 0 && sz < int64(sf.LowerBound) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks if sf is a valid batch-job size filter
|
||||||
|
func (sf BatchJobSizeFilter) Validate() error {
|
||||||
|
if sf.LowerBound > 0 && sf.UpperBound > 0 && sf.LowerBound >= sf.UpperBound {
|
||||||
|
return BatchJobYamlErr{
|
||||||
|
line: sf.line,
|
||||||
|
col: sf.col,
|
||||||
|
msg: "invalid batch-job size filter",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobSize supports humanized byte values in yaml files type BatchJobSize uint64
|
||||||
|
type BatchJobSize int64
|
||||||
|
|
||||||
|
// UnmarshalYAML to parse humanized byte values
|
||||||
|
func (s *BatchJobSize) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
var batchExpireSz string
|
||||||
|
err := unmarshal(&batchExpireSz)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sz, err := humanize.ParseBytes(batchExpireSz)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*s = BatchJobSize(sz)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,575 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobKV(t *testing.T) {
|
||||||
|
v := BatchJobKV{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobKV(b *testing.B) {
|
||||||
|
v := BatchJobKV{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobKV(b *testing.B) {
|
||||||
|
v := BatchJobKV{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobKV(b *testing.B) {
|
||||||
|
v := BatchJobKV{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobKV(t *testing.T) {
|
||||||
|
v := BatchJobKV{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobKV Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobKV{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobKV(b *testing.B) {
|
||||||
|
v := BatchJobKV{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobKV(b *testing.B) {
|
||||||
|
v := BatchJobKV{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobNotification(t *testing.T) {
|
||||||
|
v := BatchJobNotification{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobNotification(b *testing.B) {
|
||||||
|
v := BatchJobNotification{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobNotification(b *testing.B) {
|
||||||
|
v := BatchJobNotification{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobNotification(b *testing.B) {
|
||||||
|
v := BatchJobNotification{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobNotification(t *testing.T) {
|
||||||
|
v := BatchJobNotification{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobNotification Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobNotification{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobNotification(b *testing.B) {
|
||||||
|
v := BatchJobNotification{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobNotification(b *testing.B) {
|
||||||
|
v := BatchJobNotification{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobRetry(t *testing.T) {
|
||||||
|
v := BatchJobRetry{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobRetry(b *testing.B) {
|
||||||
|
v := BatchJobRetry{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobRetry(b *testing.B) {
|
||||||
|
v := BatchJobRetry{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobRetry(b *testing.B) {
|
||||||
|
v := BatchJobRetry{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobRetry(t *testing.T) {
|
||||||
|
v := BatchJobRetry{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobRetry Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobRetry{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobRetry(b *testing.B) {
|
||||||
|
v := BatchJobRetry{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobRetry(b *testing.B) {
|
||||||
|
v := BatchJobRetry{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobSizeFilter(t *testing.T) {
|
||||||
|
v := BatchJobSizeFilter{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobSizeFilter(b *testing.B) {
|
||||||
|
v := BatchJobSizeFilter{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobSizeFilter(b *testing.B) {
|
||||||
|
v := BatchJobSizeFilter{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobSizeFilter(b *testing.B) {
|
||||||
|
v := BatchJobSizeFilter{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobSizeFilter(t *testing.T) {
|
||||||
|
v := BatchJobSizeFilter{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobSizeFilter Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobSizeFilter{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobSizeFilter(b *testing.B) {
|
||||||
|
v := BatchJobSizeFilter{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobSizeFilter(b *testing.B) {
|
||||||
|
v := BatchJobSizeFilter{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalBatchJobSnowball(t *testing.T) {
|
||||||
|
v := BatchJobSnowball{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgBatchJobSnowball(b *testing.B) {
|
||||||
|
v := BatchJobSnowball{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgBatchJobSnowball(b *testing.B) {
|
||||||
|
v := BatchJobSnowball{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalBatchJobSnowball(b *testing.B) {
|
||||||
|
v := BatchJobSnowball{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeBatchJobSnowball(t *testing.T) {
|
||||||
|
v := BatchJobSnowball{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeBatchJobSnowball Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := BatchJobSnowball{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeBatchJobSnowball(b *testing.B) {
|
||||||
|
v := BatchJobSnowball{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeBatchJobSnowball(b *testing.B) {
|
||||||
|
v := BatchJobSnowball{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBatchJobSizeInRange(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
objSize int64
|
||||||
|
sizeFilter BatchJobSizeFilter
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// 1Mib < 2Mib < 10MiB -> in range
|
||||||
|
objSize: 2 << 20,
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 10 << 20,
|
||||||
|
LowerBound: 1 << 20,
|
||||||
|
},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// 2KiB < 1 MiB -> out of range from left
|
||||||
|
objSize: 2 << 10,
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 10 << 20,
|
||||||
|
LowerBound: 1 << 20,
|
||||||
|
},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// 11MiB > 10 MiB -> out of range from right
|
||||||
|
objSize: 11 << 20,
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 10 << 20,
|
||||||
|
LowerBound: 1 << 20,
|
||||||
|
},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// 2MiB < 10MiB -> in range
|
||||||
|
objSize: 2 << 20,
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 10 << 20,
|
||||||
|
},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// 2MiB > 1MiB -> in range
|
||||||
|
objSize: 2 << 20,
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
LowerBound: 1 << 20,
|
||||||
|
},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, test := range tests {
|
||||||
|
t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
|
||||||
|
if got := test.sizeFilter.InRange(test.objSize); got != test.want {
|
||||||
|
t.Fatalf("Expected %v but got %v", test.want, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBatchJobSizeValidate(t *testing.T) {
|
||||||
|
errInvalidBatchJobSizeFilter := BatchJobYamlErr{
|
||||||
|
msg: "invalid batch-job size filter",
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
sizeFilter BatchJobSizeFilter
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
// Unspecified size filter is a valid filter
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 0,
|
||||||
|
LowerBound: 0,
|
||||||
|
},
|
||||||
|
err: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 0,
|
||||||
|
LowerBound: 1 << 20,
|
||||||
|
},
|
||||||
|
err: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 10 << 20,
|
||||||
|
LowerBound: 0,
|
||||||
|
},
|
||||||
|
err: nil,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// LowerBound > UpperBound -> empty range
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 1 << 20,
|
||||||
|
LowerBound: 10 << 20,
|
||||||
|
},
|
||||||
|
err: errInvalidBatchJobSizeFilter,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// LowerBound == UpperBound -> empty range
|
||||||
|
sizeFilter: BatchJobSizeFilter{
|
||||||
|
UpperBound: 1 << 20,
|
||||||
|
LowerBound: 1 << 20,
|
||||||
|
},
|
||||||
|
err: errInvalidBatchJobSizeFilter,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for i, test := range tests {
|
||||||
|
t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
|
||||||
|
err := test.sizeFilter.Validate()
|
||||||
|
if err != nil {
|
||||||
|
gotErr := err.(BatchJobYamlErr)
|
||||||
|
testErr := test.err.(BatchJobYamlErr)
|
||||||
|
if gotErr.message() != testErr.message() {
|
||||||
|
t.Fatalf("Expected %v but got %v", test.err, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err == nil && test.err != nil {
|
||||||
|
t.Fatalf("Expected %v but got nil", test.err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
miniogo "github.com/minio/minio-go/v7"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:generate msgp -file $GOFILE
|
||||||
|
|
||||||
|
// replicate:
|
||||||
|
// # source of the objects to be replicated
|
||||||
|
// source:
|
||||||
|
// type: "minio"
|
||||||
|
// bucket: "testbucket"
|
||||||
|
// prefix: "spark/"
|
||||||
|
//
|
||||||
|
// # optional flags based filtering criteria
|
||||||
|
// # for source objects
|
||||||
|
// flags:
|
||||||
|
// filter:
|
||||||
|
// newerThan: "7d"
|
||||||
|
// olderThan: "7d"
|
||||||
|
// createdAfter: "date"
|
||||||
|
// createdBefore: "date"
|
||||||
|
// tags:
|
||||||
|
// - key: "name"
|
||||||
|
// value: "value*"
|
||||||
|
// metadata:
|
||||||
|
// - key: "content-type"
|
||||||
|
// value: "image/*"
|
||||||
|
// notify:
|
||||||
|
// endpoint: "https://splunk-hec.dev.com"
|
||||||
|
// token: "Splunk ..." # e.g. "Bearer token"
|
||||||
|
//
|
||||||
|
// # target where the objects must be replicated
|
||||||
|
// target:
|
||||||
|
// type: "minio"
|
||||||
|
// bucket: "testbucket1"
|
||||||
|
// endpoint: "https://play.min.io"
|
||||||
|
// path: "on"
|
||||||
|
// credentials:
|
||||||
|
// accessKey: "minioadmin"
|
||||||
|
// secretKey: "minioadmin"
|
||||||
|
// sessionToken: ""
|
||||||
|
|
||||||
|
// BatchReplicateFilter holds all the filters currently supported for batch replication
|
||||||
|
type BatchReplicateFilter struct {
|
||||||
|
NewerThan time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
|
||||||
|
OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
|
||||||
|
CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"`
|
||||||
|
CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"`
|
||||||
|
Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"`
|
||||||
|
Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobReplicateFlags various configurations for replication job definition currently includes
|
||||||
|
// - filter
|
||||||
|
// - notify
|
||||||
|
// - retry
|
||||||
|
type BatchJobReplicateFlags struct {
|
||||||
|
Filter BatchReplicateFilter `yaml:"filter" json:"filter"`
|
||||||
|
Notify BatchJobNotification `yaml:"notify" json:"notify"`
|
||||||
|
Retry BatchJobRetry `yaml:"retry" json:"retry"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobReplicateResourceType defines the type of batch jobs
|
||||||
|
type BatchJobReplicateResourceType string
|
||||||
|
|
||||||
|
// Validate validates if the replicate resource type is recognized and supported
|
||||||
|
func (t BatchJobReplicateResourceType) Validate() error {
|
||||||
|
switch t {
|
||||||
|
case BatchJobReplicateResourceMinIO:
|
||||||
|
case BatchJobReplicateResourceS3:
|
||||||
|
default:
|
||||||
|
return errInvalidArgument
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t BatchJobReplicateResourceType) isMinio() bool {
|
||||||
|
return t == BatchJobReplicateResourceMinIO
|
||||||
|
}
|
||||||
|
|
||||||
|
// Different types of batch jobs..
|
||||||
|
const (
|
||||||
|
BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
|
||||||
|
BatchJobReplicateResourceS3 BatchJobReplicateResourceType = "s3"
|
||||||
|
|
||||||
|
// add future targets
|
||||||
|
)
|
||||||
|
|
||||||
|
// BatchJobReplicateCredentials access credentials for batch replication it may
|
||||||
|
// be either for target or source.
|
||||||
|
type BatchJobReplicateCredentials struct {
|
||||||
|
AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"`
|
||||||
|
SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"`
|
||||||
|
SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty indicates if credentials are not set
|
||||||
|
func (c BatchJobReplicateCredentials) Empty() bool {
|
||||||
|
return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate validates if credentials are valid
|
||||||
|
func (c BatchJobReplicateCredentials) Validate() error {
|
||||||
|
if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
|
||||||
|
return errInvalidArgument
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobReplicateTarget describes target element of the replication job that receives
|
||||||
|
// the filtered data from source
|
||||||
|
type BatchJobReplicateTarget struct {
|
||||||
|
Type BatchJobReplicateResourceType `yaml:"type" json:"type"`
|
||||||
|
Bucket string `yaml:"bucket" json:"bucket"`
|
||||||
|
Prefix string `yaml:"prefix" json:"prefix"`
|
||||||
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||||
|
Path string `yaml:"path" json:"path"`
|
||||||
|
Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidPath returns true if path is valid
|
||||||
|
func (t BatchJobReplicateTarget) ValidPath() bool {
|
||||||
|
return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobReplicateSource describes source element of the replication job that is
|
||||||
|
// the source of the data for the target
|
||||||
|
type BatchJobReplicateSource struct {
|
||||||
|
Type BatchJobReplicateResourceType `yaml:"type" json:"type"`
|
||||||
|
Bucket string `yaml:"bucket" json:"bucket"`
|
||||||
|
Prefix string `yaml:"prefix" json:"prefix"`
|
||||||
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||||
|
Path string `yaml:"path" json:"path"`
|
||||||
|
Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"`
|
||||||
|
Snowball BatchJobSnowball `yaml:"snowball" json:"snowball"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidPath returns true if path is valid
|
||||||
|
func (s BatchJobReplicateSource) ValidPath() bool {
|
||||||
|
switch s.Path {
|
||||||
|
case "on", "off", "auto", "":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchJobReplicateV1 v1 of batch job replication
|
||||||
|
type BatchJobReplicateV1 struct {
|
||||||
|
APIVersion string `yaml:"apiVersion" json:"apiVersion"`
|
||||||
|
Flags BatchJobReplicateFlags `yaml:"flags" json:"flags"`
|
||||||
|
Target BatchJobReplicateTarget `yaml:"target" json:"target"`
|
||||||
|
Source BatchJobReplicateSource `yaml:"source" json:"source"`
|
||||||
|
|
||||||
|
clnt *miniogo.Core `msg:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RemoteToLocal returns true if source is remote and target is local
|
||||||
|
func (r BatchJobReplicateV1) RemoteToLocal() bool {
|
||||||
|
return !r.Source.Creds.Empty()
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user