Compare commits

...

329 Commits

Author SHA1 Message Date
Harshavardhana 752e18e795 upgrade console to v0.21.2 2022-11-07 21:27:07 -08:00
Aditya Manthramurthy 76d822bf1e Add LDAP policy entities API (#15908) 2022-11-07 14:35:09 -08:00
Klaus Post ddeca9f12a fix: filter rest errors and logs returned (#16019) 2022-11-07 10:38:08 -08:00
Shireesh Anjal 19d0340ddf Update version of madmin-go to v1.7.3 (#15994) 2022-11-07 09:32:18 -08:00
Harshavardhana 21251d8c22 initialize streaming events without lazy initialization (#16016) 2022-11-07 08:01:24 -08:00
Harshavardhana 1f3db03bf0 allow changing argument for path for SNSD setup (#16013) 2022-11-07 00:11:58 -08:00
Harshavardhana 944c62daf4 skip flaky tests on windows OS (#16015) 2022-11-07 00:11:21 -08:00
Harshavardhana 9547b7d0e9 add deadlineConnections on remoteTransport (#16010) 2022-11-05 11:09:21 -07:00
Harshavardhana 76c4ea7682 force all internal MinIO operations to be under UTC (#16009) 2022-11-04 16:44:38 -07:00
Klaus Post 808ecfe0f2 merge versions across sets when listing (#16003) 2022-11-04 11:33:22 -07:00
Klaus Post 2894dd4d1a fix: hold lock while serializing replication stats (#16007) 2022-11-04 09:59:14 -07:00
yanggang 797fa7f97b update Elasticsearch dependency to 7.17.7 (#15992) 2022-11-04 08:23:33 -07:00
jiuker fd8750e959 fix: http body must be drained in downloadBinary() (#16001) 2022-11-04 08:22:38 -07:00
Harshavardhana 7be65f66b8 support HS256 series of JWT signature for OpenID connect (#15993) 2022-11-03 16:41:53 -07:00
Poorna 4f5d38a4b1 site replication edit: validate endpoint belongs to deployment (#16000) 2022-11-03 16:23:45 -07:00
Anis Elleuch 7e73fc2870 Implement inspect data API v2 (#15474)
Co-authored-by: Klaus Post <klauspost@gmail.com>
2022-11-02 13:36:38 -07:00
yanggang d2c9a9e395 add windows port allot by "netsh dynamicport" (#15986) 2022-11-02 09:10:26 -07:00
Harshavardhana 0d49b365ff converge SNSD deployments into single code (#15988) 2022-11-01 16:41:01 -07:00
Anis Elleuch 7721595aa9 config: Deprecated delay/max_wait/scanner and introduce speed (#15941) 2022-11-01 08:04:07 -07:00
Harshavardhana fd6f6fc8df cleanup stale parent multipart directories (#15980) 2022-11-01 08:00:02 -07:00
Aditya Manthramurthy 4fb47cd568 fix: update admin IDP APIs to be more RESTful (#15896) 2022-10-31 14:52:26 -07:00
Klaus Post ecc932d5dd Clean entire tmp-old on restart (#15979) 2022-10-31 07:27:50 -07:00
Harshavardhana b57fbff7c1 ignore background healInfo in single drive setup (#15968) 2022-10-31 07:26:10 -07:00
Harshavardhana 4892a766a8 do not panic if webhook returns an error (#15970) 2022-10-30 16:45:53 -07:00
Minio Trusted 0303cd8625 Update yaml files to latest version RELEASE.2022-10-29T06-21-33Z 2022-10-29 09:29:37 +00:00
Poorna d765b89a63 improve validation for replication resync API (#15964) 2022-10-28 23:21:33 -07:00
Harshavardhana 6e4acf0504 add a message of removal for gateway and hide the command (#15965) 2022-10-28 14:11:20 -07:00
Klaus Post 71954faa3a mark pubsub type safe via generics (#15961) 2022-10-28 10:55:42 -07:00
Shireesh Anjal 6d22e74d11 mark SUBNET config keys as sensitive info (#15962) 2022-10-28 10:54:44 -07:00
Poorna dc92bb4646 xl-meta: update metadata version (#15958) 2022-10-28 02:48:43 -07:00
Klaus Post 0f0e154315 fix: inconsistent replication delete marker timestamps (#15956) 2022-10-27 09:46:52 -07:00
Harshavardhana 136d41775f remove numAvailableDisks check as it doesn't serve any purpose (#15954) 2022-10-27 09:05:24 -07:00
Harshavardhana ec77d28e62 make subnet subsys dynamic and simplify callhome (#15927) 2022-10-27 00:20:01 -07:00
Klaus Post 86420a1f46 Store multipart checksums (#15953) 2022-10-26 18:14:58 -07:00
Poorna 7dd8b6c8ed ensure ILM expiry creates non null deleteMarker for versioned bucket (#15947) 2022-10-26 16:09:27 -07:00
杨刚 8afa6fefd8 add cross-build linux/amd64 (#15949) 2022-10-26 16:05:26 -07:00
Anis Elleuch 533c9d4fe3 fix: lockName to disallow parallel same erasure set healing (#15951) 2022-10-26 12:43:54 -07:00
Anis Elleuch a35ef155fc return appropriate error status code in the lock handler (#15950) 2022-10-26 09:51:26 -07:00
Poorna 8dd3c41b2a allow MakeBucket errors to be handled lazily (#15945)
remote error is not required to be passed back to the 
client - this is mostly because we have healing that should 
eventually, catch up on this and heal the bucket.
2022-10-25 23:32:37 -07:00
Krishnan Parthasarathi 4523da6543 feat: introduce pool-level rebalance (#15483) 2022-10-25 12:36:57 -07:00
Poorna ce8456a1a9 proxy multipart to peers via multipart uploadID (#15926) 2022-10-25 10:52:29 -07:00
Minio Trusted 1673778633 add missing helm 5.0.0
Signed-off-by: Harshavardhana <harsha@minio.io>
2022-10-25 03:20:22 -07:00
Poorna 9ce1884732 reject editing bucket replication config when site replication is enabled (#15937) 2022-10-24 20:24:32 -07:00
Harshavardhana 23b329b9df remove gateway completely (#15929) 2022-10-24 17:44:15 -07:00
Krishnan Parthasarathi 0c34e51a75 Filter out tiering metadata during CopyObject (#15936) 2022-10-24 16:32:31 -07:00
Minio Trusted 1633b30979 update helm v4.1.0
Signed-off-by: Harshavardhana <harsha@minio.io>
2022-10-24 13:37:05 -07:00
Minio Trusted 630dabf4b9 Update yaml files to latest version RELEASE.2022-10-24T18-35-07Z 2022-10-24 19:29:30 +00:00
Anis Elleuch fc6c794972 Audit dangling object removal (#15933) 2022-10-24 11:35:07 -07:00
Brien Dieterle 2e33b99c6b helm: move Prometheus TLSConfig to job spec (#15739) 2022-10-24 08:46:41 -07:00
Harshavardhana 3b7292b637 update console v0.21.1 2022-10-24 02:20:47 -07:00
Harshavardhana e4f469ae7a update console dependency to master 2022-10-23 22:36:24 -07:00
Minio Trusted c921dc75c7 Update yaml files to latest version RELEASE.2022-10-21T22-37-48Z 2022-10-22 00:04:37 +00:00
Klaus Post 86d543d0f6 Check for s3zip content offset (#15924) 2022-10-21 15:37:48 -07:00
Poorna e4e90b53c1 fix: delete-marker replication check properly (#15923) 2022-10-21 14:45:06 -07:00
Anis Elleuch 58d776daa0 Set CONSOLE_MINIO_SERVER to 127.0.0.1 by default (#15887) 2022-10-21 14:42:28 -07:00
Krishnan Parthasarathi f6b2e89109 Pass encrypted etag as is for immediate tiering (#15925) 2022-10-21 14:40:50 -07:00
Anis Elleuch ac85c2af76 lifecycle: refactor rules filtering and tagging support (#15914) 2022-10-21 10:46:53 -07:00
Shireesh Anjal 5aba2aedb3 Do not freeze s3 traffic in healthinfo api (#15912) 2022-10-21 00:34:32 -07:00
Minio Trusted bd77f1df4c Update yaml files to latest version RELEASE.2022-10-20T00-55-09Z 2022-10-20 06:38:33 +00:00
Harshavardhana a8332efa94 fix: Delete() of bucket metadata should not parse the config (#15904) 2022-10-19 17:55:09 -07:00
Aditya Manthramurthy 3dbef72dc7 fix: AccountInfo API for roleARN based accounts (#15907) 2022-10-19 17:54:41 -07:00
Aditya Manthramurthy 2d16e74f38 Add LDAP IDP Configuration APIs (#15840) 2022-10-19 11:00:10 -07:00
Anis Elleuch de5070446d Deprecate --listeners flag (#15900) 2022-10-19 08:45:50 -07:00
Harshavardhana 374abd1e7d add filter support for tags and metadata in batch replication (#15885) 2022-10-18 21:22:21 -07:00
Anis Elleuch 0506d9e83d storage: Return errDiskNotFound when a peer is during shutdown (#15868) 2022-10-18 13:50:46 -07:00
Klaus Post bd3dfad8b9 Add concurrent Snowball extraction + options (#15836) 2022-10-18 13:50:21 -07:00
Harshavardhana 9fff315555 do not need to trace ignored objects (#15894) 2022-10-18 13:47:55 -07:00
Harshavardhana 07b6dce1a5 remove nancy, we rely on vulncheck from now on (#15893) 2022-10-18 10:45:44 -07:00
Anis Elleuch 18fb86b7be convert context.DeadlineExceed to offline disk in DiskInfo() (#15886) 2022-10-18 03:01:16 -07:00
Harshavardhana 58a8275e84 do not assume invalid buf to be non-xl.meta (#15843) 2022-10-17 09:39:21 -07:00
Minio Trusted 196fab6834 Update yaml files to latest version RELEASE.2022-10-15T19-57-03Z 2022-10-15 23:20:56 +00:00
Aditya Manthramurthy 85fc7cea97 Pass role ARN for OIDC providers to console (#15862) 2022-10-15 12:57:03 -07:00
Harshavardhana 328d660106 support CRC32 Checksums on single drive setup (#15873) 2022-10-15 11:58:47 -07:00
Harshavardhana c68910005b validate bucket before attempting batch replication (#15861) 2022-10-15 11:58:31 -07:00
Harshavardhana c79bcc8838 Revert "convert context.DeadlineExceed to offline disk in DiskInfo() (#15869)"
This reverts commit 0fe58dbb34.
2022-10-14 20:37:50 -07:00
Anis Elleuch 0fe58dbb34 convert context.DeadlineExceed to offline disk in DiskInfo() (#15869) 2022-10-14 19:32:13 -07:00
Harshavardhana 6cb2f56395 Revert "Revert "tests: Add context cancelation (#15374)""
This reverts commit 564a0afae1.
2022-10-14 03:08:40 -07:00
Harshavardhana 59e33b3b21 validate setBucketTarget properly as per BucketExists() call (#15860) 2022-10-13 17:46:49 -07:00
Poorna 0e3c92c027 attempt delete marker replication after object is replicated (#15857)
Ensure delete marker replication success, especially since the
recent optimizations to heal on HEAD, LIST and GET can force
replication attempts on delete marker before underlying object
version could have synced.
2022-10-13 17:45:23 -07:00
Anis Elleuch db7a9b2c37 heal-info: Return the endpoint of a disk with unknown state (#15854) 2022-10-13 16:41:44 -07:00
Harshavardhana 44097faec1 support deleteMarkers and all versions in batch replication (#15858) 2022-10-13 14:42:10 -07:00
Anis Elleuch ff5fca76ab Bump golang.org/x/text to 0.3.8 to fix CVE-2022-32149 (#15855) 2022-10-13 11:43:18 -07:00
Klaus Post bf3da5081f Omit empty checksums in responses (#15850) 2022-10-13 00:49:46 -07:00
Harshavardhana 5532982857 do not disable IsKubernetes(), IsDocker() checks with MINIO_CI_CD (#15852) 2022-10-12 23:40:48 -07:00
Anis Elleuch 783dd875f7 refactor objectQuorumFromMeta() to search for parity quorum (#15844) 2022-10-12 16:42:45 -07:00
Harshavardhana 97112c69be fix: replication stats() to not crash under any situation (#15851)
Co-authored-by: Daniel Valdivia <18384552+dvaldivia@users.noreply.github.com>
2022-10-12 15:47:41 -07:00
hellivan b0b573052a fix: unrecognized openid config parameters (#15847) 2022-10-12 12:19:44 -07:00
Javier Adriel 2939000342 Add metrics, version and apis handlers (#15839) 2022-10-12 12:08:03 -07:00
Harshavardhana 41e1654f9a remove spurious logging for object not found (#15842) 2022-10-12 04:28:21 -07:00
Harshavardhana e3cb0278ce honor specified target prefix under batch replication (#15834) 2022-10-11 14:36:06 -07:00
Harshavardhana 0c81f1bdb3 indicate how long it took to bring the drive online (#15835) 2022-10-11 11:33:56 -07:00
Klaus Post 6220875803 Add missing server info fields (#15826) 2022-10-11 11:31:26 -07:00
Anis Elleuch afd4279cd8 lock tests: Initialize different DRWMutex for each lock (#15833) 2022-10-10 15:14:32 -07:00
Minio Trusted 0c8dd8046a Update yaml files to latest version RELEASE.2022-10-08T20-11-00Z 2022-10-09 19:44:19 +00:00
Harshavardhana 3c4ef4338f marshal retention XML in expected format (#15821) 2022-10-08 13:11:00 -07:00
Aditya Manthramurthy 64cf887b28 use LDAP config from minio/pkg to share with console (#15810) 2022-10-07 22:12:36 -07:00
Harshavardhana 927a879052 authenticate the request first for headObject() (#15820) 2022-10-07 21:45:53 -07:00
Anis Elleuch dfe0c96b87 preserve Version and DeleteMarker sort order in the list XML response (#15819) 2022-10-07 16:12:36 -07:00
Anis Elleuch e856e10ac2 ignore VersionNotFound in addition to ObjectNotFound while replicating (#15814) 2022-10-07 16:11:41 -07:00
Krishnan Parthasarathi 6d6a731d6d Handle overlapping and conflicting ILM rules (#15812) 2022-10-07 14:36:23 -07:00
Harshavardhana 928feb0889 remove unused debug param from evalActionFromLifecycle (#15813) 2022-10-07 10:24:12 -07:00
Javier Adriel b3febe2d24 Update pkg to v1.4.5 (#15808) 2022-10-06 16:58:54 -07:00
Poorna b6b26dba87 fix: GetObjectRetention to parse in ISO8601 time format (#15809) 2022-10-06 13:53:56 -07:00
Minio Trusted 5c034e26bd fix: govulncheck must use go1.19.2 2022-10-06 12:42:59 -07:00
Harshavardhana cef0fb1434 remove build asset caching (#15807) 2022-10-06 11:36:07 -07:00
Anis Elleuch 158d0e26a2 decom: Ignore object/version error during deletion (#15806) 2022-10-06 09:41:58 -07:00
Harshavardhana 78385bfbeb set bucket creation timestamp properly for legacy FS backend (#15800) 2022-10-06 02:46:31 -07:00
Harshavardhana 2a13cc28f2 feat: implement support batch replication (#15554) 2022-10-05 23:00:43 -07:00
Minio Trusted 4d761fda81 Update yaml files to latest version RELEASE.2022-10-05T14-58-27Z 2022-10-05 22:50:50 +00:00
Lenin Alevski 4bdf41a6c7 Removing unused getUpdateReaderFromFile function (#15794)
Signed-off-by: Lenin Alevski <alevsk.8772@gmail.com>
2022-10-05 07:58:27 -07:00
Klaus Post 3c605c93fe warn when 0 parity has been set as default parity (#15790) 2022-10-04 22:41:42 -07:00
Anis Elleuch 121f18a443 Use admin request check for ReplicationDiff handler (#15793) 2022-10-04 17:47:31 -07:00
Harshavardhana 538aeef27a fix: heal service accounts for LDAP users in site replication (#15785) 2022-10-04 10:41:47 -07:00
Poorna be0d2537b7 site replication: fix typo in meta collection (#15792) 2022-10-04 10:19:17 -07:00
Javier Adriel 3307aa1260 Implement KMS handlers (#15737) 2022-10-04 10:05:09 -07:00
Harshavardhana 57cfdfd8fb remove 'perf' tests from health diagnostics (#15780) 2022-10-03 00:18:41 -07:00
Minio Trusted dc6733dacc Update yaml files to latest version RELEASE.2022-10-02T19-29-29Z 2022-10-03 02:29:56 +00:00
Harshavardhana f696a221af allow tagging policy condition for GetObject (#15777) 2022-10-02 12:29:29 -07:00
Harshavardhana ed5b67720c rename deprecated 'mc policy' -> 'mc anonymous' (#15779) 2022-10-01 11:47:48 -07:00
Harshavardhana 2aac50571d fix: de-duplicate conflicting object names on namespace (#15772) 2022-09-30 15:44:21 -07:00
Shireesh Anjal 45edd27ad7 Re-load config after 'mc admin config reset' (#15771) 2022-09-30 10:55:53 -07:00
Minio Trusted c302d1cfc8 update console v0.20.5 2022-09-30 02:13:12 -07:00
Anis Elleuch 6287e8c571 fix: race when accessing REST TCP dial values (#15770) 2022-09-29 09:27:58 -07:00
Anis Elleuch f69a98ce49 fix: loading Audit kafka configuration loading (#15766) 2022-09-29 08:35:08 -07:00
Daryl White d44f3526dc Update links to documentation site (#15750) 2022-09-28 21:28:45 -07:00
Harshavardhana 41b633f5ea support tagging based policy conditions (#15763) 2022-09-28 11:25:46 -07:00
Minio Trusted 4f1ff9c4d9 update console v0.20.4 2022-09-28 00:00:22 -07:00
Anis Elleuch 86bb48792c non-blocking initialization of bucket target notifications (#15571) 2022-09-27 17:23:28 -07:00
Harshavardhana 94dbb4a427 fix: generalize SC config and also skip healing sub-sys under SD (#15757) 2022-09-26 09:04:54 -07:00
Anis Elleuch 048a46ec2a Add RPC tcp timeout/errs and AVG duration to prometheus (#15747) 2022-09-26 09:04:26 -07:00
Minio Trusted 1480340830 Update yaml files to latest version RELEASE.2022-09-25T15-44-53Z 2022-09-26 04:54:15 +00:00
Harshavardhana 877bd95fa3 remove unused package internal/smart (#15758) 2022-09-25 08:44:53 -07:00
Poorna 8ea6fb368d Add auto configuration of replication workers (#15636) 2022-09-24 16:20:28 -07:00
Poorna 5fd5ddea23 relax retention date validation on replication target (#15752) 2022-09-23 21:19:03 -07:00
Harshavardhana b04c0697e1 validate correct ETag for the parts sent during CompleteMultipart (#15751) 2022-09-23 21:17:08 -07:00
Harshavardhana 50a8ba6a6f fix: parse and save retainUntilDate in correct time format (#15741) 2022-09-23 08:49:27 -07:00
Minio Trusted 334f1ed45a Update yaml files to latest version RELEASE.2022-09-22T18-57-27Z 2022-09-23 06:01:11 +00:00
Anis Elleuch 20c89ebbb3 freeze before exit when _MINIO_DEBUG_NO_EXIT is defined (#15709)
this is to ensure keep k8s pods running, when they reach a "crashloop" stage
2022-09-22 11:57:27 -07:00
Krishnan Parthasarathi 6f56ba80b3 lifecycle: Assign unique id to rules with empty id (#15731) 2022-09-22 10:51:54 -07:00
Anis Elleuch 6e84283c66 fix: ignoring O_DIRECT in case of erasure single disk (#15734)
fixes #15733 
fixes #15735
2022-09-22 10:41:06 -07:00
Harshavardhana 9d6fddcfdf persist the non-default creds in config (#15711) 2022-09-21 16:14:47 -07:00
Minio Trusted a83105df9d remove deprecated GO111MODULE=on env 2022-09-21 10:34:45 -07:00
Minio Trusted 9528b55c25 update helm chart v4.0.15 2022-09-21 04:07:01 -07:00
jiuker 749ce107ee fix: context leak with replication endpoint hearbeat (#15721) 2022-09-21 03:08:45 -07:00
Minio Trusted b2a67834ac upgrade console to v0.20.3 2022-09-21 02:29:50 -07:00
Poorna aec2aa3497 site replication: clear config if remove --all specified (#15716) 2022-09-20 14:32:23 -07:00
Jan Šafařík c7dcbfd6c1 helm: specify service account for the jobs (#15706) 2022-09-19 11:06:16 -07:00
Klaus Post ff12080ff5 Remove deprecated io/ioutil (#15707) 2022-09-19 11:05:16 -07:00
Javier Adriel 0b6175b742 Implement KMS methods for keys, policies and identities (#15673) 2022-09-19 11:04:40 -07:00
Harshavardhana cf49da387b enable cross compile for openbsd/amd64 (#15701) 2022-09-19 07:01:22 -07:00
Josh Kasuboski ac714e7e3d helm: default additional labels/annotations as object (#15698) 2022-09-18 08:03:50 -07:00
mersl 79fb79b71c helm: adds support for policy conditions (#15599) 2022-09-18 08:01:58 -07:00
Minio Trusted 98874c3baf Update yaml files to latest version RELEASE.2022-09-17T00-09-45Z 2022-09-17 08:07:32 +00:00
Minio Trusted d89f6af6c4 avoid replication stats crash in Prometheus 2022-09-16 17:09:45 -07:00
Minio Trusted d4bca00df9 update console v0.20.2 2022-09-16 16:45:59 -07:00
Harshavardhana 2c68a19dfd upgrade all deps and update CREDITS (#15650) 2022-09-16 01:59:45 -07:00
Harshavardhana 9e5853ecc0 optimize double reads by reusing results from checkUploadIDExists() (#15692)
Move to using `xl.meta` data structure to keep temporary partInfo,
this allows for a future change where we move to different parts to
different drives.
2022-09-15 12:43:49 -07:00
Harshavardhana 124544d834 add pre-conditions support for PUT calls during replication (#15674)
PUT shall only proceed if pre-conditions are met, the new
code uses

- x-minio-source-mtime
- x-minio-source-etag

to verify if the object indeed needs to be replicated
or not, allowing us to avoid StatObject() call.
2022-09-14 18:44:04 -07:00
Poorna b910904fa6 change replication stats save path for windows (#15690) 2022-09-14 13:49:13 -07:00
Klaus Post eee1ce305c When listing, do not count delete markers (#15689)
When limiting listing do not count delete, since they may be discarded.

Extend limit, since we may be discarding the forward-to marker.

Fix directories always being sent to resolve, since they didn't return as match.
2022-09-14 12:11:27 -07:00
Klaus Post 5c61c3ccdc Fix flaky TestGetObjectWithOutdatedDisks (#15687)
On occasion this test fails:

```
2022-09-12T17:22:44.6562737Z === RUN   TestGetObjectWithOutdatedDisks
2022-09-12T17:22:44.6563751Z     erasure-object_test.go:1214: Test 2: Expected data to have md5sum = `c946b71bb69c07daf25470742c967e7c`, found `7d16d23f07072af1a809707ba101ae07`
2
```

Theory: Both objects are written with the same timestamp due to lower timer resolution on Windows. This results in secondary resolution, which is deterministic, but random.

Solution: Instead of hacking in a wait we request the specific version we want. Should still keep the test relevant.

Bonus: Remote action dependency for vulncheck
2022-09-14 08:17:39 -07:00
Ashish Kumar Sinha fb8d512f58 update docs to add cycle in scanner (#15686) 2022-09-13 22:46:02 -07:00
Poorna a0fb0c1835 panic if replication config could not be read from disk (#15685)
If replication config could not be read from bucket metadata for some
reason, issue a panic so that unexpected replication outcomes can
be avoided for replicated buckets.

For similar reasons, adding a panic while fetching object-lock config
if it failed for reason other than non-existence of config.
2022-09-13 21:23:33 -07:00
Aditya Manthramurthy e152b2a975 Pass groups claim into condition values (#15679)
This allows using `jwt:groups` as a multi-valued condition key in policies.
2022-09-13 09:45:36 -07:00
Harshavardhana a71629d4dd use go1.19.1 for vulncheck (#15680) 2022-09-12 20:16:39 -07:00
Klaus Post c22f3ca7a8 fix: S3 Select CSV -> JSON with variable field count (#15677)
When there are fewer fields than expected, output fewer fields.
2022-09-12 17:00:59 -07:00
Anis Elleuch 4a92134235 prometheus: track errors during REST read/write calls (#15678)
minio_inter_node_traffic_errors_total currently does not track
requests body write/read errors of internode REST communications.

This commit fixes this by wrapping resp.Body.
2022-09-12 12:40:51 -07:00
Poorna 6b9fd256e1 Persist in-memory replication stats to disk (#15594)
to avoid relying on scanner-calculated replication metrics.
This will improve the accuracy of the replication stats reported.

This PR also adds on to #15556 by handing replication
traffic that could not be queued by available workers to the 
MRF queue so that entries in `PENDING` status are healed faster.
2022-09-12 12:40:02 -07:00
Andreas Auernhammer d6132b854f add vulncheck action to CI (#15676)
Signed-off-by: Andreas Auernhammer <hi@aead.dev>
2022-09-12 10:10:30 -07:00
Klaus Post ff9a74b91f Add fast max-keys=1 support for Listing (#15670)
Add a listing option to stop when the limit is reached.  
This can be used by stateless listings for fast results.
2022-09-09 08:13:06 -07:00
Harshavardhana b579163802 limit number of buckets to 500k (#15668)
500k is a reasonable limit for any single MinIO
cluster deployment, in future we may increase this
value.

However for now we are going to keep this limit.
2022-09-09 03:06:34 -07:00
Minio Trusted 87f0c8e7e8 Update yaml files to latest version RELEASE.2022-09-07T22-25-02Z 2022-09-08 00:44:12 +00:00
Anis Elleuch bb855499e1 Update PR template to make it easy to request for doc changes (#15665)
A link can be clicked to open a new request in minio/docs if something
in the documentation needs to be changed.
2022-09-07 15:25:02 -07:00
Krishnan Parthasarathi 96bfa77856 serialize updates to healing tracker (#15647)
When healing is parallelized by setting the ` _MINIO_HEAL_WORKERS` 
environment variable, multiple goroutines may race while updating the disk's 
healing tracker. This change serializes only these concurrent updates using a
channel. Note, the healing tracker is still not concurrency safe in other contexts.
2022-09-07 08:47:21 -07:00
Harshavardhana 8e997eba4a fix: trigger Heal when xl.meta needs healing during PUT (#15661)
This PR is a continuation of the previous change instead
of returning an error, instead trigger a spot heal on the
'xl.meta' and return only after the healing is complete.

This allows for future GETs on the same resource to be
consistent for any version of the object.
2022-09-07 07:25:39 -07:00
Harshavardhana 228c6686f8 allow non-standards fallback for all http.TimeFormats (#15662)
fixes #15645
2022-09-07 07:24:54 -07:00
Minio Trusted 52861d3aea update console to v0.20.1 2022-09-06 20:36:16 -07:00
Brien Dieterle cc26911c46 helm: use a Prometheus Operator Probe to scrape Cluster metrics (#15659)
use Probe to scrape Cluster metrics. ServiceMonitor to scrape Node metrics
2022-09-06 09:41:03 -07:00
Harshavardhana 7776d064cf allow non-standards fallback for Expires header (#15655)
fixes #15645
2022-09-05 19:18:18 -07:00
Harshavardhana 2d9b5a65f1 verify RenameData() versions to be consistent (#15649)
xl.meta gets written and never rolled back, however
we definitely need to validate the state that is
persisted on the disk, if there are inconsistencies

- more than write quorum we should return an error
  to the client

- if write quorum was achieved however there are
  inconsistent xl.meta's we should simply trigger
  an MRF on them
2022-09-05 16:51:37 -07:00
Shireesh Anjal c240da6568 Reuse madmin.ClusterRegistrationInfo (#15654)
The `clusterInfo` struct in admin-handlers is same as
madmin.ClusterRegistrationInfo, except for small differences in field
names.

Removing this and using madmin.ClusterRegistrationInfo in its place will
help in following ways:

- The JSON payload generated by mc in case of cluster registration will
  be consistent (same keys) with cluster.info generated by minio as part
  of the profile and inspect zip
- health-analyzer can parse the cluster.info using the same struct and
  won't have to define it's own
2022-09-05 10:02:25 -07:00
Harshavardhana 157272dc5b fix: use optimized json.NewEncoder instead for metrics (#15648) 2022-09-05 08:06:35 -07:00
Minio Trusted 9065274d02 update helm v4.0.14
Signed-off-by: Minio Trusted <trusted@minio.io>
2022-09-04 18:06:49 -07:00
yudoutingle f4c56026a2 fix: potential deadLock caused by unlocking a non-existing lock (#15635) 2022-09-02 14:24:32 -07:00
Harshavardhana 37e3f5de10 do not print object not found errors in MRF healing (#15646) 2022-09-02 14:22:40 -07:00
Harshavardhana 5ea629beb2 avoid printing io.ErrUnexpectedEOF for .metacache objects (#15642) 2022-09-02 12:47:17 -07:00
Minio Trusted 240164560f Update yaml files to latest version RELEASE.2022-09-01T23-53-36Z 2022-09-02 01:52:19 +00:00
Anis Elleuch cf52691959 Save resync status in the backend using a last update timestamp (#15638)
Currently, there is a short time window where the code is allowed 
to save the status of a replication resync. Currently, the window is
`now.Sub(st.EndTime) <= resyncTimeInterval`. Also, any failure to 
write in the backend disks is not retried.

Refactor the code a little bit to rely on the last timestamp of a
successful write of the resync status of any given bucket in the 
backend disks.
2022-09-01 16:53:36 -07:00
Anis Elleuch 10e75116ef Avoid replicating dirs in listing with replication enabled (#15641)
When replication is enabled in a particular bucket, the listing will send
objects to bucket replication, but it is also sending prefixes for non
recursive listing which is useless and shows a lot of error logs.

This commit will ignore prefixes.
2022-09-01 15:22:11 -07:00
Harshavardhana f649968c69 tier: avoid stats infinite loop in forwardTo method (#15640)
under some sequence of events following code would
reach an infinite loop.

```
idx1, idx2 := 0, 1
for ; idx2 != idx1; idx2++ {
        fmt.Println(idx2)
}
```

fixes #15639
2022-09-01 13:51:06 -07:00
Krishnan Parthasarathi 5ce1448049 audit: account for response headers separately (#15610) 2022-09-01 12:51:04 -07:00
Harshavardhana bcedc2b0d9 fix: add healing metric type for heal tracing (#15631)
changes the `heal.checkBucket` to `heal.Bucket` instead
since the latter is more meaningful.
2022-08-31 12:28:03 -07:00
Klaus Post 8e4a45ec41 fix: encrypt checksums in metadata (#15620) 2022-08-31 08:13:23 -07:00
Klaus Post dec942beb6 feat: Add healing trace (#15616) 2022-08-31 01:56:12 -07:00
Abirdcfly d4e0f13bb3 chore: remove duplicate word in comments (#15607)
Signed-off-by: Abirdcfly <fp544037857@gmail.com>

Signed-off-by: Abirdcfly <fp544037857@gmail.com>
2022-08-30 08:26:43 -07:00
Anis Elleuch 1f28a3bb80 Avoid messages from go test output (#15601)
A lot of warning messages are printed in CI/CD failures generated by go
test. Avoid that by requiring at least Error level for logging when
doing go test.
2022-08-30 08:23:40 -07:00
Krishnan Parthasarathi 3a1d3a7952 audit-log: Add time to get/restore object from remote-tier (#15602) 2022-08-29 21:33:59 -07:00
Klaus Post a9f1ad7924 Add extended checksum support (#15433) 2022-08-29 16:57:16 -07:00
Poorna 929b9e164e site replication: Avoid returning root svcacct info in sr metadata (#15608)
Service accounts of root users should not be replicated.
2022-08-29 11:19:51 -07:00
Harshavardhana 97376f6e8f improve performance for inlined data (#15603)
inlined data often is bigger than the allowed
O_DIRECT alignment, so potentially we can write
'xl.meta' without O_DSYNC instead we can rely on
O_DIRECT + fdatasync() instead.

This PR allows O_DIRECT on inlined data that
would gain the benefits of performing O_DIRECT,
eventually performing an fdatasync() at the end.

Performance boost can be observed here for small
objects < 128KiB. The performance boost is mainly
seen on HDD, and marginal on NVMe setups.
2022-08-29 11:19:29 -07:00
Jan Šafařík 92a0a59de2 helm: add extra volumes and extra volume mounts option for job resources (#15590)
* add a new line to the end of the credentials file when creating a user
* add extra volumes and mounts option into helm chart
* add extra volumes and extra volume mounts option for job resources
2022-08-28 21:17:30 -07:00
Minio Trusted cd18599e7b Update yaml files to latest version RELEASE.2022-08-26T19-53-15Z 2022-08-29 00:10:22 +00:00
Febriananda Wida Pramudita 1f22a16b15 fix: endpoints for single local disks must retain port info (#15585) 2022-08-26 12:53:15 -07:00
Harshavardhana 433b6fa8fe upgrade golang-lint to the latest (#15600) 2022-08-26 12:52:29 -07:00
jiuker d7cd857c7c fix: .golangci.yml key 'linters-settings' duplicate entry (#15589) 2022-08-25 23:06:06 -07:00
Krishnan Parthasarathi 99fbfe2421 Add concurrency to healing objects on a fresh disk (#15575) 2022-08-25 13:07:15 -07:00
Poorna b1b6264bea fix: validate deployment id when adding peer clusters (#15591)
Fixes: #15573
2022-08-25 11:30:52 -07:00
Minio Trusted 1fd72d5aea Update yaml files to latest version RELEASE.2022-08-25T07-17-05Z 2022-08-25 07:47:18 +00:00
Aditya Manthramurthy 18dffb26e7 Allow querying a single target in config get API (#15587) 2022-08-25 00:17:05 -07:00
Harshavardhana edba7c987b fix: objects matching prefixes should not leave delete markers (#15586)
This is needed to ensure that we do not leave prefixes where
version is suspended, instead we never leave versions on
these paths.
2022-08-24 13:46:29 -07:00
Anis Elleuch b737c83a66 Ensure that only one node performs site replication healing (#15584)
When a node finds a change in the other replication cluster and applies
to itself will already notify other peers. No need for all nodes in a
given cluster to do site replication healing, only one node is
sufficient.
2022-08-24 13:46:09 -07:00
Anis Elleuch 97a6322de1 Fix regression in notifying peers about new policy mapping (#15583)
Switch from mux.Vars() to r.Form to avoid the issue of missing arguments
passed to LoadPolicyMappingHandler.
2022-08-24 12:34:52 -07:00
Klaus Post 037fe4afdc Add listing block reuse (#15579)
When streaming results, pool metadata slices when sent.
2022-08-24 09:11:16 -07:00
Aditya Manthramurthy afbb63a197 Factor out external event notification funcs (#15574)
This change moves external event notification functionality into
`event-notification.go`. This simplifies notification related code.
2022-08-24 06:42:36 -07:00
Harshavardhana 8902561f3c use new xxml for XML responses to support rare control characters (#15511)
use new xxml/XML responses to support rare control characters

fixes #15023
2022-08-23 17:04:11 -07:00
Minio Trusted a67116b5bc helm: update to release v4.0.13 2022-08-23 11:18:14 -07:00
Jan Šafařík 0f7aa4125f helm: add extra volumes and extra volume mounts option (#15568)
- add a new line to the end of the credentials file when creating a user
- add extra volumes and mounts option into helm chart
2022-08-23 11:15:26 -07:00
X b62a5c954c helm: Supply proper initial values for annotations (#15570) 2022-08-23 11:14:54 -07:00
Anis Elleuch b8cdf060c8 Properly replicate policy mapping for virtual users (#15558)
Currently, replicating policy mapping for STS users does not work. Fix
it is by passing user type to PolicyDBSet.
2022-08-23 11:11:45 -07:00
Minio Trusted 9fb937986e update console to v0.20.0 2022-08-23 00:03:18 -07:00
Minio Trusted 2c48f6a02b Update yaml files to latest version RELEASE.2022-08-22T23-53-06Z 2022-08-23 05:45:31 +00:00
Poorna 4155c5b695 replication: improve MRF healing. (#15556)
This PR improves the replication failure healing by persisting
most recent failures to disk and re-queuing them until the replication
is successful.

While this does not eliminate the need for healing during a full scan, 
queuing MRF vastly improves the ETA to keeping replicated buckets 
in sync as it does not wait for the scanner visit to detect unreplicated 
object versions.
2022-08-22 16:53:06 -07:00
Poorna 471467d310 fix: ensure metadata update happens after deletemarker replication (#15564)
Fixes regression caused by #15521
2022-08-22 15:59:06 -07:00
Aditya Manthramurthy c54c13831a fix: error types in internal config package (#15559) 2022-08-20 15:11:49 -07:00
Harshavardhana ae4ee95d25 change default lock retry interval to 50ms (#15560)
competing calls on the same object on versioned bucket
mutating calls on the same object may unexpected have
higher delays.

This can be reproduced with a replicated bucket
overwriting the same object writes, deletes repeatedly.

For longer locks like scanner keep the 1sec interval
2022-08-19 16:21:05 -07:00
Klaus Frank a2e037f0ec doc: certgen typo in parameter name (#15555)
"-ca" is not a valid parameter remove it.
2022-08-19 10:48:34 -07:00
Harshavardhana e9055e9ef7 fix: walk() should cancel itself upon context cancellation (#15553)
This PR fixes possible leaks that may emanate from not
listening on context cancelation or timeouts.

```
goroutine 60957610 [chan send, 16 minutes]:
github.com/minio/minio/cmd.(*erasureServerPools).Walk.func1.1.1(...)
        github.com/minio/minio/cmd/erasure-server-pool.go:1724 +0x368
github.com/minio/minio/cmd.listPathRaw({0x4a9a740, 0xc0666dffc0},...
        github.com/minio/minio/cmd/metacache-set.go:1022 +0xfc4
github.com/minio/minio/cmd.(*erasureServerPools).Walk.func1.1()
        github.com/minio/minio/cmd/erasure-server-pool.go:1764 +0x528
created by github.com/minio/minio/cmd.(*erasureServerPools).Walk.func1
        github.com/minio/minio/cmd/erasure-server-pool.go:1697 +0x1b7
```
2022-08-18 17:49:08 -07:00
Harshavardhana d350b666ff feat: add idempotent delete marker support (#15521)
The bottom line is delete markers are a nuisance,
most applications are not version aware and this
has simply complicated the version management.

AWS S3 gave an unnecessary complication overhead
for customers, they need to now manage these
markers by applying ILM settings and clean
them up on a regular basis.

To make matters worse all these delete markers
get replicated as well in a replicated setup,
requiring two ILM settings on each site.

This PR is an attempt to address this inferior
implementation by deviating MinIO towards an
idempotent delete marker implementation i.e
MinIO will never create any more than single
consecutive delete markers.

This significantly reduces operational overhead
by making versioning more useful for real data.

This is an S3 spec deviation for pragmatic reasons.
2022-08-18 16:41:59 -07:00
Aditya Manthramurthy 21831b3fe2 fix: cleanup config KV parsing using madmin helpers (#15552) 2022-08-18 15:55:17 -07:00
Harshavardhana 895357607a avoid using errors.As for 'errors.New' use errors.Is (#15549)
Bonus: ignore coredns CVE, for now, there is no fix yet

https://github.com/coredns/coredns/issues/5574
2022-08-18 11:10:49 -07:00
Minio Trusted ac240a8477 fix: CoreDNS vulnerability CVE-2022-2837 dependency 2022-08-18 10:10:06 -07:00
Harshavardhana bf38c0c0d1 fix: increase concurrency of DeleteObjects() to N/10th (#15546)
instead of keeping the value 10 and static, make
the concurrency a function of incoming number of
objects being deleted.
2022-08-18 09:33:56 -07:00
Pierre KELBERT 67cf15d036 mount export volume even if persistence is disabled (#15543) 2022-08-17 08:50:32 -07:00
Aditya Manthramurthy 701a82642b Ensure subnet+callhome configs are single sub-sys (#15542) 2022-08-16 17:46:46 -07:00
Poorna 21fe14201f replication: centralize healthcheck for remote targets (#15516)
This PR moves health check from minio-go client to being
managed on the server.

Additionally integrating health check into site replication
2022-08-16 17:46:22 -07:00
Harshavardhana 48640b1de2 fix: trim arn:aws:kms from incoming SSE aws-kms-key-id (#15540) 2022-08-16 11:28:30 -07:00
Anis Elleuch 5682685c80 Introduce disk io stats metrics (#15512) 2022-08-16 07:13:49 -07:00
Sébastien Rodriguez 9c025b8cce helm: fix correct deployment environment variable (#15534) 2022-08-14 18:13:56 -07:00
Minio Trusted eef9f13360 update helm v4.0.12 2022-08-13 22:50:43 -07:00
Minio Trusted fa9b361a3d Update yaml files to latest version RELEASE.2022-08-13T21-54-44Z 2022-08-14 05:33:08 +00:00
reyerdam 49862ba347 helm: (Bugfix) enable objectlocking (#15524) 2022-08-13 14:54:44 -07:00
Aditya Manthramurthy ee2afcf70b Move some config constants to madmin (#15529) 2022-08-13 14:50:16 -07:00
Harshavardhana c7d535c648 init console after IAM init() (#15531)
fixes #15527
2022-08-13 12:54:41 -07:00
Aditya Manthramurthy 9986e103cf Fix env var output in config get/export APIs (#15528)
Fix a bug where env vars are not output when the config for the
subsystem is specified solely via env vars.
2022-08-13 10:39:01 -07:00
Minio Trusted c5b3666089 Update yaml files to latest version RELEASE.2022-08-11T04-37-28Z 2022-08-11 22:36:34 +00:00
Minio Trusted d265fe7f9e update console v0.19.5 2022-08-10 21:37:28 -07:00
Krishnan Parthasarathi 91e6af4470 Add trace support for decommissioning (#15502)
* Add trace support for decommissioning
* Add support for tracing errors during decommission
2022-08-10 12:46:45 -07:00
Noah Gao b940fe8fca chore: bad title syntax of helm chart README (#15513) 2022-08-10 11:16:22 -07:00
reyerdam 73fe2e95fe helm: Support adding objectlocking for buckets (#15505)
implemented object locking during bucket creation in helm chart
2022-08-10 08:13:51 -07:00
Shireesh Anjal 316c492842 Upgrade madmin-go to latest version (v1.4.15) (#15510) 2022-08-10 07:36:13 -07:00
Harshavardhana 74418b542a fix: incorrect context timeout during listPath() (#15509)
This PR cleans up the listing code for single drive
to ensure that we do not add an incorrect context
timeout, while resuming the listing.

fixes #15508
2022-08-10 07:35:29 -07:00
Poorna 172e63dbb6 fix: site replication group updates to set status correctly (#15507)
Fixes: #15486
2022-08-09 15:17:43 -07:00
Poorna 21bf5b4db7 replication: heal proactively upon access (#15501)
Queue failed/pending replication for healing during listing and GET/HEAD
API calls. This includes healing of existing objects that were never
replicated or those in the middle of a resync operation.

This PR also fixes a bug in ListObjectVersions where lifecycle filtering
should be done.
2022-08-09 15:00:24 -07:00
Harshavardhana a406bb0288 restrict number of disks used for scanning buckets upto GOMAXPROCS (#15492)
control scanner parallelism to avoid higher CPU
usage on nodes that have more drives but an old CPU.
2022-08-08 16:16:44 -07:00
Harshavardhana 1823ab6808 LDAP/OpenID must be initialized IAM Init() (#15491)
This allows for LDAP/OpenID to be non-blocking,
allowing for unreachable Identity targets to be
initialized in IAM.
2022-08-08 16:16:27 -07:00
Harshavardhana 8eec49304d use logger.Info instead of logger.LogIf 2022-08-08 16:13:58 -07:00
Harshavardhana ecdc2f2f5f fix: maxConcurrent '0' is an invalid value (#15500)
log and continue with defaults instead of
crashing the service.
2022-08-08 15:18:45 -07:00
Minio Trusted 6a6c772ff2 Update yaml files to latest version RELEASE.2022-08-08T18-34-09Z 2022-08-08 21:30:40 +00:00
Harshavardhana e178c55bc3 remove non-working GetRawData() from FS mode (#15498) 2022-08-08 11:34:09 -07:00
Poorna 2c137c0d04 fix: handle invalid endpoint errors in site replication(#15499)
fixes #15497
2022-08-08 11:12:05 -07:00
Minio Trusted 1d35f2b58f update madmin-go to v1.4.13 2022-08-08 10:41:38 -07:00
Harshavardhana 638c57e466 revert changes in FS implementation for umask
fixes #15494
2022-08-08 09:48:24 -07:00
Harshavardhana 5e4213b3be fix: keep writing previous speedtest result (#15484)
when object speedtest is running keep writing
previous speedtest result back to client until
we have a new result - this avoids sending back
blank entries in between the speedtest when it
is running in 'autotune' mode.
2022-08-07 23:04:03 -07:00
Minio Trusted 102295f58a update helm v4.0.11 2022-08-06 22:41:47 -07:00
Jan Šafařík a0d14f8ff7 helm: add a new line to the end of the credentials file (#15485) 2022-08-06 15:01:01 -07:00
Harshavardhana e0b0a351c6 remove IAM old migration code (#15476)
```
commit 7bdaf9bc50
Author: Aditya Manthramurthy <donatello@users.noreply.github.com>
Date:   Wed Jul 24 17:34:23 2019 -0700

    Update on-disk storage format for users system (#7949)
```

Bonus: fixes a bug when etcd keys were being re-encrypted.
2022-08-05 17:53:23 -07:00
Minio Trusted fcd4b3ba9b Update yaml files to latest version RELEASE.2022-08-05T23-27-09Z 2022-08-06 00:08:21 +00:00
Anis Elleuch 1d2ff46a89 Ensure lock/versioning permissions when creating a bucket (#15432)
Currently, the code doesn't check if the user creating a bucket with
locking feature has bucket locking and versioning permissions enabled,
adding it in accordance with S3 spec.

https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html

Object Lock - If ObjectLockEnabledForBucket is set to true in your CreateBucket request,
s3:PutBucketObjectLockConfiguration and s3:PutBucketVersioning permissions are required.
2022-08-05 16:27:09 -07:00
Harshavardhana 8f7c739328 feat: add SpeedTest ResponseTimes and TTFB (#15479)
Capture average, p50, p99, p999 response times
and ttfb values. These are needed for latency
measurements and overall understanding of our
speedtest results.
2022-08-05 09:40:03 -07:00
Poorna 1beea3daba fix: import bucket metadata import to return a summary (#15462) 2022-08-05 01:52:50 -07:00
Minio Trusted 1ffd063939 update CREDITS for latest deps 2022-08-04 23:30:31 -07:00
Aditya Manthramurthy 3d94c38ec4 Add env variables to configuration APIs output (#15465)
Config export and config get APIs now include environment 
variables set on the server
2022-08-04 22:21:52 -07:00
Harshavardhana f4af2d3cdc fix: decodeDirObject() in single drive DeleteObjects() call (#15477)
Thanks to @bh4t for reproducing this issue.
2022-08-04 18:57:43 -07:00
ebozduman b57e7321e7 Replaces 'disk'=>'drive' visible to end user (#15464) 2022-08-04 16:10:08 -07:00
Anis Elleuch e93867488b actively cancel listIAMConfigItems to avoid goroutine leak (#15471)
listConfigItems creates a goroutine but sometimes callers will
exit without properly asking listAllIAMConfigItems() to stop sending
results, hence a goroutine leak.

Create a new context and cancel it for each listAllIAMConfigItems
call.
2022-08-04 13:20:43 -07:00
Minio Trusted c08790edd2 upgrade helm v4.0.10 2022-08-04 09:09:22 -07:00
Kourosh Tafreshi a46baddbc4 Add OIDC to the HelmChart (#15469) 2022-08-04 09:07:51 -07:00
Harshavardhana 3bd9615d0e fix: log if there is readDir() failure with ListBuckets (#15461)
This is actionable and must be logged.

Bonus: also honor umask by using 0o666 for all Open() syscalls.
2022-08-04 07:23:05 -07:00
Minio Trusted 2871cb5775 upgrade helm v4.0.9 2022-08-02 23:10:44 -07:00
Harshavardhana a6e0ec4e6f Add support converting non-inlined to inlined (#15444)
This is a feature to allow for inode compaction on
large clusters that use a lot of small files spread
across a large heirarchy.
2022-08-02 23:10:22 -07:00
Minio Trusted e956369c4e Update yaml files to latest version RELEASE.2022-08-02T23-59-16Z 2022-08-03 01:45:42 +00:00
Minio Trusted 76f950c663 upgrade to minio-go/v7 v7.0.34 2022-08-02 16:59:16 -07:00
Andreas Auernhammer d774a3309b kes: automatically reload KES client certificate (#15450)
This commit adds support for automatically reloading
the MinIO client certificate for authentication to KES.

The client certificate will now be reloaded:
 - when the private key / certificate file changes
 - when a SIGHUP signal is received
 - every 15 minutes

Fixes #14869

Signed-off-by: Andreas Auernhammer <hi@aead.dev>
2022-08-02 16:58:09 -07:00
Anis Elleuch b3edb25377 bloom: healObject to mark a path dirty only for dangling objects (#15458)
The path is marked dirty automatically when healObject() is called, which is
wrong. HealObject() is called during self-healing and this will lead to
an increase in the false positive result of the bloom filter.

Also move NSUpdated() from renameData() and call it directly in
CompleteMultipart and PutObject, this is not a functional change but
it will make it less prone to errors in the future.
2022-08-02 16:57:39 -07:00
Harshavardhana 026b87e39b do not crash for unwrapErrs return nil (#15456)
fixes #15454
2022-08-02 15:10:11 -07:00
Harshavardhana 53a816b17a fix: readdir fallback on root of the drive (#15457)
fixes #15452
2022-08-02 14:57:36 -07:00
Harshavardhana 043aaa792d fix: intrument os.OpenFile differently for Reads and Writes (#15449)
allows us to trace latency for READs or WRITEs
2022-08-01 13:22:43 -07:00
dorman aad9cb208a helm: modify user secret volumes mount path name (#15443) 2022-08-01 12:28:04 -07:00
Minio Trusted edf081c6a2 update minio-go v7.0.33 2022-08-01 10:44:35 -07:00
Harshavardhana fd349103e8 fix: allow P-384/P-512 constant time implementation (#15445)
since go1.18.x P-384/P-512 are now constant time
implementations, enable them.
2022-08-01 09:27:16 -07:00
Anis Elleuch 10b49eb4fb Fix resetting a config with a non default target name (#15448)
mc admin config reset <alias> notify_webhook:something was not working
properly.

The reason is that GetSubSys() was not calculating the target
name properly because it is quitting early when the number of config
inputs ('notify_webhook:something' in this case) is equal to 1.

This commit will make the code calculates always calculate the target
name if found.
2022-08-01 07:52:23 -07:00
Anis Elleuch 3856d078d2 fix: set 20000 as maximum parallel event calls (#15435)
This is needed to avoid consuming a lot of goroutines when a target is
very slow or there is a bug in a target library.
2022-07-30 12:12:33 -07:00
Minio Trusted 6b4cb35f4f Update yaml files to latest version RELEASE.2022-07-30T05-21-40Z 2022-07-30 05:50:06 +00:00
Shireesh Anjal e6eab2091f fix: Incorrect ServersCount in cluster.info (#15431)
The `ServersCount` field in cluster.info is expected to contain the
number of nodes, and not number of endpoints.
2022-07-29 22:21:40 -07:00
Harshavardhana 3cdb609cca allow root users to return appropriate policy in AccountInfo (#15437)
fixes #15436

This fixes a regression caused after the removal of "consoleAdmin"
policy usage for 'root users' in PR #15402
2022-07-29 20:58:03 -07:00
Minio Trusted d6a7f62ff5 update helm v4.0.8 2022-07-29 16:39:54 -07:00
Minio Trusted 72f170f5d2 update console to v0.19.4 2022-07-29 15:07:53 -07:00
Minio Trusted 824d52a82b Update yaml files to latest version RELEASE.2022-07-29T19-40-48Z 2022-07-29 22:06:57 +00:00
Minio Trusted 067ebab9d8 update object-locking docs and word them appropriately 2022-07-29 12:40:48 -07:00
Anis Elleuch 6be6c0d2e3 Update kafka library to v1.35.0 (#15434)
There is a known rare issue in the current version 1.30.0 described here
https://github.com/Shopify/sarama/issues/2241.

Update the library to 1.35.0

Bonus:  update shirou/gopsutil v3.22.5 to v3.22.6 to fix a compilation
error for OpenBSD
2022-07-29 11:34:45 -07:00
Harshavardhana aa874010e2 fix: regression in resolving the right versions (#15430)
fix: regression in resolving right versions

commit d480022711 caused a regression in real
resolver, by picking up incorrect versionID.
2022-07-29 10:03:53 -07:00
Cesar Celis Hernandez 8ec888d13d feat: update binary once and push it to other servers (#15407) 2022-07-29 08:34:30 -07:00
Harshavardhana 916f274c83 choose starting concurrency based on number of local disks (#15428)
smaller setups may have less drives per server choosing
the concurrency based on number of local drives, and let
the MinIO server change the overall concurrency as
necessary.
2022-07-29 00:00:06 -07:00
Aditya Manthramurthy 7ac53c07af fix: passing application configuration to console (#15409)
This is an update to MinIO server after swagger codegen related build
fixes added after issues introduced in 39fd7b0b3b
2022-07-28 18:30:24 -07:00
Harshavardhana bc72e4226e do not allow filesystem fallback in server download (#15429)
It is possible for anyone with admin access to relatively
to get any content of any random OS location by simply
providing the file with 'mc admin update alias/ /etc/passwd`.

Workaround is to disable 'admin:ServiceUpdate' action. Everyone
is advised to upgrade to this patch.

Thanks to @alevsk for finding this bug.
2022-07-28 17:44:21 -07:00
Poorna 5e0776e96a replication: Include replica object versions for resync (#15427) 2022-07-28 13:43:02 -07:00
Anis Elleuch 2f1ef02d35 Do not update directory access time (#15426)
Most setups will have relatime it only updates the access time 
following a change in the directory.
2022-07-28 12:40:48 -07:00
Minio Trusted db8442584e update helm chart v4.0.7 2022-07-27 20:54:38 -07:00
Naveen d46cf50760 chore(deps): Included dependency review (#14958)
> Dependency Review GitHub Action in your repository to enforce dependency
> reviews on your pull requests.
> The action scans for vulnerable versions of dependencies introduced by package version
> changes in pull requests,
> and warns you about the associated security vulnerabilities.
> This gives you better visibility of what's changing in a pull request,
> and helps prevent vulnerabilities from being added to your repository.

https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement

Signed-off-by: Naveen <172697+naveensrinivasan@users.noreply.github.com>
2022-07-27 20:53:26 -07:00
Andrew Hibbert 0357121d17 Optionally run a command when completing jobs (#15017) 2022-07-27 20:51:16 -07:00
Harshavardhana aff236e20e fix: cluster healthcheck for single drive setups (#15415)
single drive setups must return '200 OK' if
drive is accessible, current master returns '503'
2022-07-27 16:46:34 -07:00
Harshavardhana cbd70d26b5 optimize speedtest for smaller setups (#15414)
this has been observed in multiple environments
where the setups are small `speedtest` naturally
fails with default '10s' and the concurrency
of '32' is big for such clusters.

choose a smaller value i.e equal to number of
drives in such clusters and let 'autotune'
increase the concurrency instead.
2022-07-27 14:41:59 -07:00
Harshavardhana 5e763b71dc use logger.LogOnce to reduce printing disconnection logs (#15408)
fixes #15334

- re-use net/url parsed value for http.Request{}
- remove gosimple, structcheck and unusued due to https://github.com/golangci/golangci-lint/issues/2649
- unwrapErrs upto leafErr to ensure that we store exactly the correct errors
2022-07-27 09:44:59 -07:00
Aditya Manthramurthy 7e4e7a66af Remove internal usage of consoleAdmin (#15402)
"consoleAdmin" was used as the policy for root derived accounts, but this
lead to unexpected bugs when an administrator modified the consoleAdmin
policy

This change avoids evaluating a policy for root derived accounts as by
default no policy is mapped to the root user. If a session policy is
attached to a root derived account, it will be evaluated as expected.
2022-07-26 19:06:55 -07:00
Shireesh Anjal 906947a285 fix: typo in json key ClusterInfo DeploymentID (#15406)
deployement_id -> deployment_id
2022-07-26 19:05:33 -07:00
Minio Trusted bfc70bc74e Update yaml files to latest version RELEASE.2022-07-26T00-53-03Z 2022-07-26 06:56:37 +00:00
jiuker 6b4f833a12 convert repeated error checks into single function in logger (#15387) 2022-07-25 17:53:03 -07:00
Poorna 426c902b87 site replication: fix healing of bucket deletes. (#15377)
This PR changes the handling of bucket deletes for site 
replicated setups to hold on to deleted bucket state until 
it syncs to all the clusters participating in site replication.
2022-07-25 17:51:32 -07:00
Anis Elleuch e4b51235f8 upgrade: Split in two steps to ensure a stable retry (#15396)
Currently, if one server in a distributed setup fails to upgrade 
due to any reasons, it is not possible to upgrade again unless 
nodes are restarted.

To fix this, split the upgrade process into two steps :

- download the new binary on all servers
- If successful, overwrite the old binary with the new one
2022-07-25 17:49:47 -07:00
Harshavardhana 4c6498d726 move all CI/CD to go1.18 (#15401) 2022-07-25 15:27:20 -07:00
Eng Zer Jun 0a3b1ad4eb test: use T.TempDir to create temporary test directory (#15400)
This commit replaces `ioutil.TempDir` with `t.TempDir` in tests. The
directory created by `t.TempDir` is automatically removed when the test
and all its subtests complete.

Prior to this commit, temporary directory created using `ioutil.TempDir`
needs to be removed manually by calling `os.RemoveAll`, which is omitted
in some tests. The error handling boilerplate e.g.
	defer func() {
		if err := os.RemoveAll(dir); err != nil {
			t.Fatal(err)
		}
	}
is also tedious, but `t.TempDir` handles this for us nicely.

Reference: https://pkg.go.dev/testing#T.TempDir

Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-07-25 12:37:26 -07:00
Anis Elleuch f23f442d33 Add cluster info to inspect/profiling archive (#15360)
Add cluster info to inspect and profiling archive.

In addition to the existing data generation for both inspect and profiling,
cluster.info file is added. This latter contains some info of the cluster.
The generation of cluster.info is is done as the last step and it can fail
if it exceed 10 seconds.
2022-07-25 09:11:35 -07:00
Minio Trusted e465c3587b Update yaml files to latest version RELEASE.2022-07-24T17-09-31Z 2022-07-24 17:37:58 +00:00
Minio Trusted 7109b6d414 update console to v0.19.3 2022-07-24 10:09:31 -07:00
Minio Trusted 8c97f3e9bc update minio-go/v7 v7.0.32 2022-07-24 09:28:19 -07:00
Klaus Post 3795b2c8ba Add compression scheme to header (#15395)
For easier debugging. We still do not return compressed size for security reasons.
2022-07-24 07:15:49 -07:00
Harshavardhana 7725425e05 fix: fork os.MkdirAll to optimize cases where parent exists (#15379)
a/b/c/d/ where `a/b/c/` exists results in additional syscalls
such as an Lstat() call to verify if the `a/b/c/` exists
and its a directory.

We do not need to do this on MinIO since the parent prefixes
if exist, we can simply return success without spending
additional syscalls.

Also this implementation attempts to simply use Access() calls
to avoid os.Stat() calls since the latter does memory allocation
for things we do not need to use.

Access() is simpler since we have a predictable structure on
the backend and we know exactly how our path structures are.
2022-07-24 00:43:11 -07:00
Minio Trusted b2f4948bbe update helm v4.0.6 2022-07-23 20:34:14 -07:00
Minio Trusted f802d2ba83 Update yaml files to latest version RELEASE.2022-07-24T01-54-52Z 2022-07-24 02:31:24 +00:00
560 changed files with 30782 additions and 26916 deletions
+2 -1
View File
@@ -15,5 +15,6 @@
## Checklist:
- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
- [ ] Documentation updated
- [ ] Unit tests added/updated
- [ ] Internal documentation updated
- [ ] Create a documentation update request [here](https://github.com/minio/docs/issues/new?label=doc-change,title=Doc+Updated+Needed+For+PR+github.com%2fminio%2fminio%2fpull%2fNNNNN)
+14
View File
@@ -0,0 +1,14 @@
name: 'Dependency Review'
on: [pull_request]
permissions:
contents: read
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@v3
- name: 'Dependency Review'
uses: actions/dependency-review-action@v1
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.17.x, 1.18.x]
go-version: [1.18.x, 1.19.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.17.11b7, 1.18.3b7]
go-version: [1.18.5b7]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
+2 -9
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.17.x]
go-version: [1.18.x, 1.19.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
@@ -28,14 +28,6 @@ jobs:
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build on ${{ matrix.os }}
if: matrix.os == 'ubuntu-latest'
env:
@@ -49,3 +41,4 @@ jobs:
make verify-healing
make verify-healing-inconsistent-versions
make verify-healing-with-root-disks
make verify-healing-with-rewrite
+2 -22
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.17.x, 1.18.x]
go-version: [1.18.x, 1.19.x]
os: [ubuntu-latest, windows-latest]
steps:
- uses: actions/checkout@v2
@@ -28,30 +28,13 @@ jobs:
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- uses: actions/cache@v2
if: matrix.os == 'ubuntu-latest'
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- uses: actions/cache@v2
if: matrix.os == 'windows-latest'
with:
path: |
%LocalAppData%\go-build
~/go/pkg/mod
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build on ${{ matrix.os }}
if: matrix.os == 'windows-latest'
env:
CGO_ENABLED: 0
GO111MODULE: on
run: |
netsh int ipv4 set dynamicport tcp start=60000 num=61000
go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
go test -v --timeout 50m ./...
- name: Build on ${{ matrix.os }}
@@ -63,9 +46,6 @@ jobs:
sudo apt install jq -y
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
go list -deps -json ./... | jq -s 'unique_by(.Module.Path)|.[]|select(has("Module"))|.Module' | ./nancy sleuth
make
make test
make test-race
+2 -13
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.17.x, 1.18.x]
go-version: [1.18.x, 1.19.x]
os: [ubuntu-latest]
steps:
- uses: actions/checkout@v2
@@ -28,23 +28,12 @@ jobs:
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build on ${{ matrix.os }}
if: matrix.os == 'ubuntu-latest'
env:
CGO_ENABLED: 0
GO111MODULE: on
MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
MINIO_KMS_KES_KEY_NAME: "my-minio-key"
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
MINIO_KMS_AUTO_ENCRYPTION: on
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+1 -9
View File
@@ -61,7 +61,7 @@ jobs:
# are turned off - i.e. if ldap="", then ldap server is not enabled for
# the tests.
matrix:
go-version: [1.17.x]
go-version: [1.18.x]
ldap: ["", "localhost:389"]
etcd: ["", "http://localhost:2379"]
openid: ["", "http://127.0.0.1:5556/dex"]
@@ -80,14 +80,6 @@ jobs:
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Test LDAP/OpenID/Etcd combo
env:
LDAP_TEST_SERVER: ${{ matrix.ldap }}
+1 -9
View File
@@ -21,7 +21,7 @@ jobs:
strategy:
matrix:
go-version: [1.17.x]
go-version: [1.18.x, 1.19.x]
steps:
- uses: actions/checkout@v2
@@ -29,14 +29,6 @@ jobs:
with:
go-version: ${{ matrix.go-version }}
check-latest: true
- uses: actions/cache@v2
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Test Decom
run: |
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.17.x]
go-version: [1.18.x, 1.19.x]
os: [ubuntu-latest]
steps:
+26
View File
@@ -0,0 +1,26 @@
name: VulnCheck
on:
pull_request:
branches:
- master
push:
branches:
- master
jobs:
vulncheck:
name: Analysis
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.19.x
check-latest: true
- name: Get official govulncheck
run: go install golang.org/x/vuln/cmd/govulncheck@latest
shell: bash
- name: Run govulncheck
run: govulncheck ./...
shell: bash
+1 -1
View File
@@ -25,7 +25,7 @@ mc.*
s3-check-md5*
xl-meta*
healing-*
inspect*
inspect*.zip
200M*
hash-set
minio.RELEASE*
+3 -16
View File
@@ -1,6 +1,6 @@
linters-settings:
golint:
min-confidence: 0
gofumpt:
lang-version: "1.18"
misspell:
locale: US
@@ -14,28 +14,15 @@ linters:
- govet
- revive
- ineffassign
- gosimple
- deadcode
- structcheck
- gomodguard
- gofmt
- unused
- structcheck
- unconvert
- varcheck
- unused
- gocritic
- gofumpt
- tenv
- durationcheck
linters-settings:
gofumpt:
lang-version: "1.17"
# Choose whether or not to use the extra rules that are disabled
# by default
extra-rules: false
issues:
exclude-use-default: false
exclude:
-4
View File
@@ -1,4 +0,0 @@
CVE-2020-26160
CVE-2020-15136
CVE-2020-15115
CVE-2020-15114
+2262 -285
View File
File diff suppressed because it is too large Load Diff
+11 -5
View File
@@ -15,12 +15,12 @@ checks: ## check dependencies
@(env bash $(PWD)/buildscripts/checkdeps.sh)
help: ## print this help
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}'
getdeps: ## fetch necessary dependencies
@mkdir -p ${GOPATH}/bin
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.45.2
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@f3635b96e4838a6c773babb65ef35297fe5fe2f9
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
crosscompile: ## cross compile minio
@@ -89,12 +89,18 @@ verify-healing: ## verify healing and replacing disks with minio binary
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@(env bash $(PWD)/buildscripts/verify-healing.sh)
@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
verify-healing-with-root-disks:
@echo "Verify healing with root disks"
verify-healing-with-root-disks: ## verify healing root disks
@echo "Verify healing with root drives"
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
verify-healing-with-rewrite: ## verify healing to rewrite old xl.meta -> new xl.meta
@echo "Verify healing with rewrite"
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@(env bash $(PWD)/buildscripts/rewrite-old-new.sh)
verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
@echo "Verify resolving inconsistent versions build with race"
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+19 -21
View File
@@ -14,7 +14,7 @@ Use the following commands to run a standalone MinIO server as a container.
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
for more complete documentation.
### Stable
@@ -32,7 +32,7 @@ root credentials. You can use the Browser to create buckets, upload objects, and
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
> NOTE: To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.
@@ -40,7 +40,7 @@ see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to vi
Use the following commands to run a standalone MinIO server on macOS.
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
### Homebrew (recommended)
@@ -60,7 +60,7 @@ brew install minio/stable/minio
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html/> to view MinIO SDKs for supported languages.
### Binary Download
@@ -74,7 +74,7 @@ chmod +x minio
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
## GNU/Linux
@@ -99,9 +99,9 @@ The following table lists supported architectures. Replace the `wget` URL with t
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
## Microsoft Windows
@@ -119,23 +119,23 @@ minio.exe server D:\
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html#) for more complete documentation.
## Install from Source
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
```sh
GO111MODULE=on go install github.com/minio/minio@latest
go install github.com/minio/minio@latest
```
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://min.io/docs/minio/linux/developers/minio-drivers.html> to view MinIO SDKs for supported languages.
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html) for more complete documentation.
MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
@@ -222,7 +222,7 @@ For example: `export MINIO_SERVER_URL="https://minio.example.net"`
## Test using MinIO Client `mc`
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart) for further instructions.
## Upgrading MinIO
@@ -230,7 +230,7 @@ Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all tr
> NOTE: requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/minio/baremetal/reference/minio-mc-admin/mc-admin-update.html)
- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://min.io/docs/minio/linux/reference/minio-mc-admin/mc-admin-update.html)
```sh
mc admin update <minio alias, e.g., myminio>
@@ -250,12 +250,10 @@ mc admin update <minio alias, e.g., myminio>
## Explore Further
- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
- [Use `s3cmd` with MinIO Server](https://docs.min.io/docs/s3cmd-with-minio)
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
- [The MinIO documentation website](https://docs.min.io)
- [MinIO Erasure Code Overview](https://min.io/docs/minio/linux/operations/concepts/erasure-coding.html)
- [Use `mc` with MinIO Server](https://min.io/docs/minio/linux/reference/minio-mc.html)
- [Use `minio-go` SDK with MinIO Server](https://min.io/docs/minio/linux/developers/go/minio-go.html)
- [The MinIO documentation website](https://min.io/docs/minio/linux/index.html)
## Contribute to MinIO Project
+1 -1
View File
@@ -9,7 +9,7 @@ function _init() {
export CGO_ENABLED=0
## List of architectures and OS to test coss compilation.
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
}
function _build() {
+93
View File
@@ -0,0 +1,93 @@
#!/bin/bash -e
set -E
set -o pipefail
set -x
WORK_DIR="$PWD/.verify-$RANDOM"
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
if [ ! -x "$PWD/minio" ]; then
echo "minio executable binary not found in current directory"
exit 1
fi
if [ ! -x "$PWD/minio" ]; then
echo "minio executable binary not found in current directory"
exit 1
fi
function start_minio_4drive() {
start_port=$1
export MINIO_ROOT_USER=minio
export MINIO_ROOT_PASSWORD=minio123
export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
export MINIO_CI_CD=1
mkdir ${WORK_DIR}
C_PWD=${PWD}
if [ ! -x "$PWD/mc" ]; then
MC_BUILD_DIR="mc-$RANDOM"
if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
echo "failed to download https://github.com/minio/mc"
purge "${MC_BUILD_DIR}"
exit 1
fi
(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
# remove mc source.
purge "${MC_BUILD_DIR}"
fi
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 &
pid=$!
disown $pid
sleep 5
if ! ps -p ${pid} 1>&2 >/dev/null; then
echo "server1 log:"
cat "${WORK_DIR}/server1.log"
echo "FAILED"
purge "$WORK_DIR"
exit 1
fi
"${PWD}/mc" mb --with-versioning minio/bucket
for i in $(seq 1 4); do
"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
sudo chown -R root. "${WORK_DIR}/disk${i}"
"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
done
for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
done
pkill minio
sleep 3
}
function main() {
start_port=$(shuf -i 10000-65000 -n 1)
start_minio_4drive ${start_port}
}
function purge()
{
rm -rf "$1"
}
( main "$@" )
rv=$?
purge "$WORK_DIR"
exit "$rv"
+87
View File
@@ -0,0 +1,87 @@
//go:build ignore
// +build ignore
//
// MinIO Object Storage (c) 2022 MinIO, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"time"
"github.com/minio/madmin-go"
)
func main() {
// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
// dummy values, please replace them with original values.
// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
// New returns an MinIO Admin client object.
madmClnt, err := madmin.New(os.Args[1], os.Args[2], os.Args[3], false)
if err != nil {
log.Fatalln(err)
}
opts := madmin.HealOpts{
Recursive: true, // recursively heal all objects at 'prefix'
Remove: true, // remove content that has lost quorum and not recoverable
Recreate: true, // rewrite all old non-inlined xl.meta to new xl.meta
ScanMode: madmin.HealNormalScan, // by default do not do 'deep' scanning
}
start, _, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, "", false, false)
if err != nil {
log.Fatalln(err)
}
fmt.Println("Healstart sequence ===")
enc := json.NewEncoder(os.Stdout)
if err = enc.Encode(&start); err != nil {
log.Fatalln(err)
}
fmt.Println()
for {
_, status, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, start.ClientToken, false, false)
if status.Summary == "finished" {
fmt.Println("Healstatus on items ===")
for _, item := range status.Items {
if err = enc.Encode(&item); err != nil {
log.Fatalln(err)
}
}
break
}
if status.Summary == "stopped" {
fmt.Println("Healstatus on items ===")
fmt.Println("Heal failed with", status.FailureDetail)
break
}
for _, item := range status.Items {
if err = enc.Encode(&item); err != nil {
log.Fatalln(err)
}
}
time.Sleep(time.Second)
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ __init__() {
mc mb minio/minio-test/
mc cp ./minio minio/minio-test/to-read/
mc cp /etc/hosts minio/minio-test/to-read/hosts
mc policy set download minio/minio-test
mc anonymous set download minio/minio-test
verify_checksum_mc ./minio minio/minio-test/to-read/minio
+1 -1
View File
@@ -39,7 +39,7 @@ function start_minio_5drive() {
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
pid=$!
disown $pid
sleep 30
sleep 5
if ! ps -p ${pid} 1>&2 >/dev/null; then
echo "server1 log:"
+151
View File
@@ -0,0 +1,151 @@
#!/bin/bash -e
set -E
set -o pipefail
set -x
WORK_DIR="$PWD/.verify-$RANDOM"
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
if [ ! -x "$PWD/minio" ]; then
echo "minio executable binary not found in current directory"
exit 1
fi
function download_old_release() {
if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
fi
}
function verify_rewrite() {
start_port=$1
export MINIO_ACCESS_KEY=minio
export MINIO_SECRET_KEY=minio123
export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
export MINIO_CI_CD=1
MC_BUILD_DIR="mc-$RANDOM"
if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
echo "failed to download https://github.com/minio/mc"
purge "${MC_BUILD_DIR}"
exit 1
fi
(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
# remove mc source.
purge "${MC_BUILD_DIR}"
"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
pid=$!
disown $pid
sleep 10
if ! ps -p ${pid} 1>&2 >/dev/null; then
echo "server1 log:"
cat "${WORK_DIR}/server1.log"
echo "FAILED"
purge "$WORK_DIR"
exit 1
fi
"${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
"${WORK_DIR}/mc" cp \
buildscripts/verify-build.sh \
minio/healing-rewrite-bucket/ \
--disable-multipart --quiet
"${WORK_DIR}/mc" cp \
buildscripts/verify-build.sh \
minio/healing-rewrite-bucket/ \
--disable-multipart --quiet
"${WORK_DIR}/mc" cp \
buildscripts/verify-build.sh \
minio/healing-rewrite-bucket/ \
--disable-multipart --quiet
kill ${pid}
sleep 3
"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
pid=$!
disown $pid
sleep 10
if ! ps -p ${pid} 1>&2 >/dev/null; then
echo "server1 log:"
cat "${WORK_DIR}/server1.log"
echo "FAILED"
purge "$WORK_DIR"
exit 1
fi
go build ./docs/debugging/s3-check-md5/
if ! ./s3-check-md5 \
-debug \
-versions \
-access-key minio \
-secret-key minio123 \
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
echo "server1 log:"
cat "${WORK_DIR}/server1.log"
echo "FAILED"
mkdir -p inspects
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
"${WORK_DIR}/mc" mb play/inspects
"${WORK_DIR}/mc" mirror inspects play/inspects
purge "$WORK_DIR"
exit 1
fi
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
sleep 1
if ! ./s3-check-md5 \
-debug \
-versions \
-access-key minio \
-secret-key minio123 \
-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
echo "server1 log:"
cat "${WORK_DIR}/server1.log"
echo "FAILED"
mkdir -p inspects
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
"${WORK_DIR}/mc" mb play/inspects
"${WORK_DIR}/mc" mirror inspects play/inspects
purge "$WORK_DIR"
exit 1
fi
kill ${pid}
}
function main() {
download_old_release
start_port=$(shuf -i 10000-65000 -n 1)
verify_rewrite ${start_port}
}
function purge()
{
rm -rf "$1"
}
( main "$@" )
rv=$?
purge "$WORK_DIR"
exit "$rv"
+2 -2
View File
@@ -120,7 +120,7 @@ function start_minio_16drive() {
cat "${WORK_DIR}/server1.log"
echo "FAILED"
mkdir -p inspects
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
"${WORK_DIR}/mc" mb play/inspects
"${WORK_DIR}/mc" mirror inspects play/inspects
@@ -140,7 +140,7 @@ function start_minio_16drive() {
cat "${WORK_DIR}/server1.log"
echo "FAILED"
mkdir -p inspects
(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
"${WORK_DIR}/mc" mb play/inspects
"${WORK_DIR}/mc" mirror inspects play/inspects
+2 -2
View File
@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
}
// Before proceeding validate if bucket exists.
_, err := objAPI.GetBucketInfo(ctx, bucket)
_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
}
// Before proceeding validate if bucket exists.
_, err := objAPI.GetBucketInfo(ctx, bucket)
_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
+183 -196
View File
@@ -25,7 +25,6 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
"time"
@@ -69,12 +68,12 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
vars := mux.Vars(r)
bucket := pathClean(vars["bucket"])
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
data, err := ioutil.ReadAll(r.Body)
data, err := io.ReadAll(r.Body)
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
@@ -131,7 +130,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
vars := mux.Vars(r)
bucket := pathClean(vars["bucket"])
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -161,11 +160,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
bucket := pathClean(vars["bucket"])
update := r.Form.Get("update") == "true"
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Get current object layer instance.
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
if objectAPI == nil {
@@ -173,7 +167,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -249,7 +243,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
}
if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil {
switch err.(type) {
case BucketRemoteConnectionErr:
case RemoteTargetConnectionErr:
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
default:
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -290,10 +284,6 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
bucket := pathClean(vars["bucket"])
arnType := vars["type"]
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Get current object layer instance.
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
if objectAPI == nil {
@@ -301,7 +291,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
}
if bucket != "" {
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -329,10 +319,6 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
bucket := pathClean(vars["bucket"])
arn := vars["arn"]
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Get current object layer instance.
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
if objectAPI == nil {
@@ -340,7 +326,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -374,10 +360,6 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
bucket := pathClean(r.Form.Get("bucket"))
if !globalIsErasure {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Get current object layer instance.
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
if objectAPI == nil {
@@ -390,13 +372,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
)
if bucket != "" {
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
buckets = append(buckets, BucketInfo{Name: bucket})
} else {
buckets, err = objectAPI.ListBuckets(ctx)
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -619,6 +601,39 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
}
}
type importMetaReport struct {
madmin.BucketMetaImportErrs
}
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
st := i.Buckets[bucket]
var errMsg string
if err != nil {
errMsg = err.Error()
}
switch fname {
case bucketPolicyConfig:
st.Policy = madmin.MetaStatus{IsSet: true, Err: errMsg}
case bucketNotificationConfig:
st.Notification = madmin.MetaStatus{IsSet: true, Err: errMsg}
case bucketLifecycleConfig:
st.Lifecycle = madmin.MetaStatus{IsSet: true, Err: errMsg}
case bucketSSEConfig:
st.SSEConfig = madmin.MetaStatus{IsSet: true, Err: errMsg}
case bucketTaggingConfig:
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
case bucketQuotaConfigFile:
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
case objectLockConfig:
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
case bucketVersioningConfig:
st.Versioning = madmin.MetaStatus{IsSet: true, Err: errMsg}
default:
st.Err = errMsg
}
i.Buckets[bucket] = st
}
// ImportBucketMetadataHandler - imports all bucket metadata from a zipped file and overwrite bucket metadata config
// There are some caveats regarding the following:
// 1. object lock config - object lock should have been specified at time of bucket creation. Only default retention settings are imported here.
@@ -628,18 +643,13 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
ctx := newContext(r, w, "ImportBucketMetadata")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
bucket := pathClean(r.Form.Get("bucket"))
if !globalIsErasure {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Get current object layer instance.
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
if objectAPI == nil {
return
}
data, err := ioutil.ReadAll(r.Body)
data, err := io.ReadAll(r.Body)
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
@@ -651,47 +661,46 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
return
}
bucketMap := make(map[string]struct{}, 1)
rpt := importMetaReport{
madmin.BucketMetaImportErrs{
Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
},
}
// import object lock config if any - order of import matters here.
for _, file := range zr.File {
slc := strings.Split(file.Name, slashSeparator)
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
}
b, fileName := slc[0], slc[1]
if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
bucket = b
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
continue
}
bucket, fileName := slc[0], slc[1]
switch fileName {
case objectLockConfig:
reader, err := file.Open()
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
config, err := objectlock.ParseObjectLockConfig(reader)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
apiErr.Description = err.Error()
writeErrorResponse(ctx, w, apiErr, r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
continue
}
configData, err := xml.Marshal(config)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
if _, ok := bucketMap[bucket]; !ok {
opts := BucketOptions{
opts := MakeBucketOptions{
LockEnabled: config.ObjectLockEnabled == "Enabled",
}
err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
if err != nil {
if _, ok := err.(BucketExists); !ok {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
}
bucketMap[bucket] = struct{}{}
@@ -699,15 +708,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
// Deny object locking configuration settings on existing buckets without object lock enabled.
if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rpt.SetStatus(bucket, fileName, nil)
// Call site replication hook.
//
@@ -720,8 +730,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
ObjectLockConfig: &cfgStr,
UpdatedAt: updatedAt,
}); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
}
}
@@ -730,162 +740,145 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
for _, file := range zr.File {
slc := strings.Split(file.Name, slashSeparator)
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
}
b, fileName := slc[0], slc[1]
if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
bucket = b
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
continue
}
bucket, fileName := slc[0], slc[1]
switch fileName {
case bucketVersioningConfig:
reader, err := file.Open()
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
v, err := versioning.ParseConfig(io.LimitReader(reader, maxBucketVersioningConfigSize))
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
if _, ok := bucketMap[bucket]; !ok {
err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
if err != nil {
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{}); err != nil {
if _, ok := err.(BucketExists); !ok {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
}
bucketMap[bucket] = struct{}{}
}
if globalSiteReplicationSys.isEnabled() && v.Suspended() {
writeErrorResponse(ctx, w, APIError{
Code: "InvalidBucketState",
Description: "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
HTTPStatusCode: http.StatusConflict,
}, r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended."))
continue
}
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
writeErrorResponse(ctx, w, APIError{
Code: "InvalidBucketState",
Description: "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
HTTPStatusCode: http.StatusConflict,
}, r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
continue
}
if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
writeErrorResponse(ctx, w, APIError{
Code: "InvalidBucketState",
Description: "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
HTTPStatusCode: http.StatusConflict,
}, r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
continue
}
configData, err := xml.Marshal(v)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
continue
}
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rpt.SetStatus(bucket, fileName, nil)
}
}
for _, file := range zr.File {
reader, err := file.Open()
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, ""), r.URL)
return
rpt.SetStatus(file.Name, "", err)
continue
}
sz := file.FileInfo().Size()
slc := strings.Split(file.Name, slashSeparator)
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
}
b, fileName := slc[0], slc[1]
if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
bucket = b
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
continue
}
bucket, fileName := slc[0], slc[1]
// create bucket if it does not exist yet.
if _, ok := bucketMap[bucket]; !ok {
err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
if err != nil {
if _, ok := err.(BucketExists); !ok {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, "", err)
continue
}
}
bucketMap[bucket] = struct{}{}
}
if _, ok := bucketMap[bucket]; !ok {
continue
}
switch fileName {
case bucketNotificationConfig:
config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalNotificationSys.targetList)
config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalEventNotifier.targetList)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
if event.IsEventError(err) {
apiErr = importError(ctx, err, file.Name, bucket)
}
writeErrorResponse(ctx, w, apiErr, r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
continue
}
configData, err := xml.Marshal(config)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rulesMap := config.ToRulesMap()
globalNotificationSys.AddRulesMap(bucket, rulesMap)
globalEventNotifier.AddRulesMap(bucket, rulesMap)
rpt.SetStatus(bucket, fileName, nil)
case bucketPolicyConfig:
// Error out if Content-Length is beyond allowed size.
if sz > maxBucketPolicySize {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
continue
}
bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(reader, sz))
bucketPolicyBytes, err := io.ReadAll(io.LimitReader(reader, sz))
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
// Version in policy must not be empty
if bucketPolicy.Version == "" {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrMalformedPolicy.String()))
continue
}
configData, err := json.Marshal(bucketPolicy)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rpt.SetStatus(bucket, fileName, nil)
// Call site replication hook.
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
Type: madmin.SRBucketMetaTypePolicy,
@@ -893,56 +886,51 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
Policy: bucketPolicyBytes,
UpdatedAt: updatedAt,
}); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
case bucketLifecycleConfig:
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
// Validate the received bucket policy document
if err = bucketLifecycle.Validate(); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
// Validate the transition storage ARNs
if err = validateTransitionTier(bucketLifecycle); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
configData, err := xml.Marshal(bucketLifecycle)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rpt.SetStatus(bucket, fileName, nil)
case bucketSSEConfig:
// Parse bucket encryption xml
encConfig, err := validateBucketSSEConfig(io.LimitReader(reader, maxBucketSSEConfigSize))
if err != nil {
apiErr := APIError{
Code: "MalformedXML",
Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, err),
HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
}
writeErrorResponse(ctx, w, apiErr, r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
continue
}
// Return error if KMS is not initialized
if GlobalKMS == nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s", errorCodes[ErrKMSNotConfigured].Description))
continue
}
kmsKey := encConfig.KeyID()
if kmsKey != "" {
@@ -950,26 +938,27 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
if err != nil {
if errors.Is(err, kes.ErrKeyNotFound) {
writeErrorResponse(ctx, w, importError(ctx, errKMSKeyNotFound, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, errKMSKeyNotFound)
continue
}
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
}
configData, err := xml.Marshal(encConfig)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
// Store the bucket encryption configuration in the object layer
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rpt.SetStatus(bucket, fileName, nil)
// Call site replication hook.
//
@@ -982,29 +971,30 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
SSEConfig: &cfgStr,
UpdatedAt: updatedAt,
}); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
case bucketTaggingConfig:
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
if err != nil {
apiErr := errorCodes.ToAPIErrWithErr(ErrMalformedXML, fmt.Errorf("error importing %s with %w", file.Name, err))
writeErrorResponse(ctx, w, apiErr, r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
continue
}
configData, err := xml.Marshal(tags)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rpt.SetStatus(bucket, fileName, nil)
// Call site replication hook.
//
// We encode the xml bytes as base64 to ensure there are no encoding
@@ -1016,32 +1006,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
Tags: &cfgStr,
UpdatedAt: updatedAt,
}); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
case bucketQuotaConfigFile:
data, err := ioutil.ReadAll(reader)
data, err := io.ReadAll(reader)
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
quotaConfig, err := parseBucketQuota(bucket, data)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
if quotaConfig.Type == "fifo" {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
continue
}
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
if err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
rpt.SetStatus(bucket, fileName, nil)
bucketMeta := madmin.SRBucketMeta{
Type: madmin.SRBucketMetaTypeQuotaConfig,
@@ -1055,15 +1046,23 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
// Call site replication hook.
if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
return
rpt.SetStatus(bucket, fileName, err)
continue
}
}
}
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, rptData)
}
// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
// to the connected HTTP client. This is a MinIO only extension
// to the connected HTTP client.
func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ReplicationDiff")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -1071,25 +1070,13 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
vars := mux.Vars(r)
bucket := vars["bucket"]
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Get current object layer instance.
objectAPI := newObjectLayerFn()
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
if objectAPI == nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return
}
// check if user has permissions to perform this operation
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
+12
View File
@@ -124,6 +124,18 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case errors.Is(err, errDecommissionRebalanceAlreadyRunning):
apiErr = APIError{
Code: "XMinioDecommissionNotAllowed",
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case errors.Is(err, errRebalanceDecommissionAlreadyRunning):
apiErr = APIError{
Code: "XMinioRebalanceNotAllowed",
Description: err.Error(),
HTTPStatusCode: http.StatusBadRequest,
}
case errors.Is(err, errConfigNotFound):
apiErr = APIError{
Code: "XMinioConfigError",
+49 -33
View File
@@ -81,6 +81,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err = validateConfig(cfg, subSys); err != nil {
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
return
@@ -91,6 +92,12 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
return
}
// freshly retrieve the config so that default values are loaded for reset config
if cfg, err = getValidConfig(objectAPI); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
dynamic := config.SubSystemsDynamic.Contains(subSys)
if dynamic {
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
@@ -173,10 +180,16 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
if dynamic {
applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
}
writeSuccessResponseHeadersOnly(w)
}
// GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
//
// `key` can be one of three forms:
// 1. `subsys:target` -> request for config of a single subsystem and target pair.
// 2. `subsys:` -> request for config of a single subsystem and the default target.
// 3. `subsys` -> request for config of all targets for the given subsystem.
func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetConfigKV")
@@ -189,15 +202,34 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
cfg := globalServerConfig.Clone()
vars := mux.Vars(r)
buf := &bytes.Buffer{}
cw := config.NewConfigWriteTo(cfg, vars["key"])
if _, err := cw.WriteTo(buf); err != nil {
key := vars["key"]
var subSys, target string
{
ws := strings.SplitN(key, madmin.SubSystemSeparator, 2)
subSys = ws[0]
if len(ws) == 2 {
if ws[1] == "" {
target = madmin.Default
} else {
target = ws[1]
}
}
}
subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
var s strings.Builder
for _, subSysConfig := range subSysConfigs {
subSysConfig.WriteTo(&s, false)
}
password := cred.SecretKey
econfigData, err := madmin.EncryptData(password, buf.Bytes())
econfigData, err := madmin.EncryptData(password, []byte(s.String()))
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -422,47 +454,31 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
var s strings.Builder
hkvs := config.HelpSubSysMap[""]
var count int
for _, hkv := range hkvs {
count += len(cfg[hkv.Key])
}
for _, hkv := range hkvs {
v := cfg[hkv.Key]
for target, kv := range v {
off := kv.Get(config.Enable) == config.EnableOff
// We ignore the error below, as we cannot get one.
cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
for _, item := range cfgSubsysItems {
off := item.Config.Get(config.Enable) == config.EnableOff
switch hkv.Key {
case config.EtcdSubSys:
off = !etcd.Enabled(kv)
off = !etcd.Enabled(item.Config)
case config.CacheSubSys:
off = !cache.Enabled(kv)
off = !cache.Enabled(item.Config)
case config.StorageClassSubSys:
off = !storageclass.Enabled(kv)
off = !storageclass.Enabled(item.Config)
case config.PolicyPluginSubSys:
off = !polplugin.Enabled(kv)
off = !polplugin.Enabled(item.Config)
case config.IdentityOpenIDSubSys:
off = !openid.Enabled(kv)
off = !openid.Enabled(item.Config)
case config.IdentityLDAPSubSys:
off = !xldap.Enabled(kv)
off = !xldap.Enabled(item.Config)
case config.IdentityTLSSubSys:
off = !globalSTSTLSConfig.Enabled
case config.IdentityPluginSubSys:
off = !idplugin.Enabled(kv)
}
if off {
s.WriteString(config.KvComment)
s.WriteString(config.KvSpaceSeparator)
}
s.WriteString(hkv.Key)
if target != config.Default {
s.WriteString(config.SubSystemSeparator)
s.WriteString(target)
}
s.WriteString(config.KvSpaceSeparator)
s.WriteString(kv.String())
count--
if count > 0 {
s.WriteString(config.KvNewline)
off = !idplugin.Enabled(item.Config)
}
item.WriteTo(&s, off)
}
}
+245 -128
View File
@@ -20,6 +20,7 @@ package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -29,22 +30,14 @@ import (
"github.com/minio/madmin-go"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/config"
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
"github.com/minio/minio/internal/config/identity/openid"
"github.com/minio/minio/internal/logger"
iampolicy "github.com/minio/pkg/iam/policy"
"github.com/minio/pkg/ldap"
)
// List of implemented ID config types.
var idCfgTypes = set.CreateStringSet("openid")
// SetIdentityProviderCfg:
//
// PUT <admin-prefix>/id-cfg?type=openid&name=dex1
func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "SetIdentityCfg")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
if objectAPI == nil {
return
@@ -56,6 +49,14 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
return
}
// Ensure body content type is opaque to ensure that request body has not
// been interpreted as form data.
contentType := r.Header.Get("Content-Type")
if contentType != "application/octet-stream" {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
password := cred.SecretKey
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
if err != nil {
@@ -64,42 +65,48 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
return
}
cfgType := mux.Vars(r)["type"]
if !idCfgTypes.Contains(cfgType) {
// TODO: change this to invalid type error when implementation
// is complete.
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
idpCfgType := mux.Vars(r)["type"]
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
return
}
var cfgDataBuilder strings.Builder
switch cfgType {
case "openid":
fmt.Fprintf(&cfgDataBuilder, "identity_openid")
var subSys string
switch idpCfgType {
case madmin.OpenidIDPCfg:
subSys = madmin.IdentityOpenIDSubSys
case madmin.LDAPIDPCfg:
subSys = madmin.IdentityLDAPSubSys
}
// Ensure body content type is opaque.
contentType := r.Header.Get("Content-Type")
if contentType != "application/octet-stream" {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
// Subsystem configuration name could be empty.
cfgName := mux.Vars(r)["name"]
cfgTarget := madmin.Default
if cfgName != "" {
fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
cfgTarget = cfgName
if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
// LDAP does not support multiple configurations. So cfgName must be
// empty or `madmin.Default`.
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
return
}
}
fmt.Fprintf(&cfgDataBuilder, "%s%s", config.KvSpaceSeparator, string(reqBytes))
cfgData := cfgDataBuilder.String()
subSys, _, _, err := config.GetSubSys(cfgData)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
// Check that this is a valid Create vs Update API call.
s := globalServerConfig.Clone()
if apiErrCode := handleCreateUpdateValidation(s, subSys, cfgTarget, isUpdate); apiErrCode != ErrNone {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
return
}
cfgData := ""
{
tgtSuffix := ""
if cfgTarget != madmin.Default {
tgtSuffix = config.SubSystemSeparator + cfgTarget
}
cfgData = subSys + tgtSuffix + config.KvSpaceSeparator + string(reqBytes)
}
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -119,6 +126,16 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
}
if err = validateConfig(cfg, subSys); err != nil {
var validationErr ldap.Validation
if errors.As(err, &validationErr) {
// If we got an LDAP validation error, we need to send appropriate
// error message back to client (likely mc).
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
validationErr.FormatError(), r.URL)
return
}
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
return
}
@@ -138,86 +155,101 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
writeSuccessResponseHeadersOnly(w)
}
// GetIdentityProviderCfg:
//
// GET <admin-prefix>/id-cfg?type=openid&name=dex_test
//
// GetIdentityProviderCfg returns a list of configured IDPs on the server if
// name is empty. If name is non-empty, returns the configuration details for
// the IDP of the given type and configuration name. The configuration name for
// the default ("un-named") configuration target is `_`.
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetIdentityProviderCfg")
func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isUpdate bool) APIErrorCode {
if cfgTarget != madmin.Default {
// This cannot give an error at this point.
subSysTargets, _ := s.GetAvailableTargets(subSys)
subSysTargetsSet := set.CreateStringSet(subSysTargets...)
if isUpdate && !subSysTargetsSet.Contains(cfgTarget) {
return ErrAdminConfigIDPCfgNameDoesNotExist
}
if !isUpdate && subSysTargetsSet.Contains(cfgTarget) {
return ErrAdminConfigIDPCfgNameAlreadyExists
}
return ErrNone
}
// For the default configuration name, since it will always be an available
// target, we need to check if a configuration value has been set previously
// to figure out if this is a valid create or update API call.
// This cannot really error (FIXME: improve the type for GetConfigInfo)
var cfgInfos []madmin.IDPCfgInfo
switch subSys {
case madmin.IdentityOpenIDSubSys:
cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
case madmin.IdentityLDAPSubSys:
cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
}
if len(cfgInfos) > 0 && !isUpdate {
return ErrAdminConfigIDPCfgNameAlreadyExists
}
if len(cfgInfos) == 0 && isUpdate {
return ErrAdminConfigIDPCfgNameDoesNotExist
}
return ErrNone
}
// AddIdentityProviderCfg: adds a new IDP config for openid/ldap.
//
// PUT <admin-prefix>/idp-cfg/openid/dex1 -> create named config `dex1`
//
// PUT <admin-prefix>/idp-cfg/openid/_ -> create (default) named config `_`
func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "AddIdentityProviderCfg")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
a.addOrUpdateIDPHandler(ctx, w, r, false)
}
// UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
//
// PATCH <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
//
// PATCH <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "UpdateIdentityProviderCfg")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
a.addOrUpdateIDPHandler(ctx, w, r, true)
}
// ListIdentityProviderCfg:
//
// GET <admin-prefix>/idp-cfg/openid -> lists openid provider configs.
func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ListIdentityProviderCfg")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
if objectAPI == nil {
return
}
cfgType := mux.Vars(r)["type"]
cfgName := r.Form.Get("name")
password := cred.SecretKey
if !idCfgTypes.Contains(cfgType) {
// TODO: change this to invalid type error when implementation
// is complete.
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
idpCfgType := mux.Vars(r)["type"]
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
return
}
// If no cfgName is provided, we list.
if cfgName == "" {
a.listIdentityProviders(ctx, w, r, cfgType, password)
return
}
var cfgList []madmin.IDPListItem
var err error
switch idpCfgType {
case madmin.OpenidIDPCfg:
cfg := globalServerConfig.Clone()
cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
case madmin.LDAPIDPCfg:
cfg := globalServerConfig.Clone()
cfgList, err = globalLDAPConfig.GetConfigList(cfg)
cfg := globalServerConfig.Clone()
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
if err != nil {
if err == openid.ErrProviderConfigNotFound {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
return
}
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
res := madmin.IDPConfig{
Type: cfgType,
Name: cfgName,
Info: cfgInfos,
}
data, err := json.Marshal(res)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
econfigData, err := madmin.EncryptData(password, data)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, econfigData)
}
func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
// var subSys string
switch cfgType {
case "openid":
// subSys = config.IdentityOpenIDSubSys
default:
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
cfg := globalServerConfig.Clone()
cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -238,9 +270,69 @@ func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.Resp
writeSuccessResponseJSON(w, econfigData)
}
// GetIdentityProviderCfg:
//
// GET <admin-prefix>/idp-cfg/openid/dex_test
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "GetIdentityProviderCfg")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
if objectAPI == nil {
return
}
idpCfgType := mux.Vars(r)["type"]
cfgName := mux.Vars(r)["name"]
password := cred.SecretKey
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
return
}
cfg := globalServerConfig.Clone()
var cfgInfos []madmin.IDPCfgInfo
var err error
switch idpCfgType {
case madmin.OpenidIDPCfg:
cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
case madmin.LDAPIDPCfg:
cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
}
if err != nil {
if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
return
}
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
res := madmin.IDPConfig{
Type: idpCfgType,
Name: cfgName,
Info: cfgInfos,
}
data, err := json.Marshal(res)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
econfigData, err := madmin.EncryptData(password, data)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, econfigData)
}
// DeleteIdentityProviderCfg:
//
// DELETE <admin-prefix>/id-cfg?type=openid&name=dex_test
// DELETE <admin-prefix>/idp-cfg/openid/dex_test
func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "DeleteIdentityProviderCfg")
@@ -251,57 +343,82 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
return
}
cfgType := mux.Vars(r)["type"]
idpCfgType := mux.Vars(r)["type"]
cfgName := mux.Vars(r)["name"]
if !idCfgTypes.Contains(cfgType) {
// TODO: change this to invalid type error when implementation
// is complete.
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
return
}
cfg := globalServerConfig.Clone()
cfgCopy := globalServerConfig.Clone()
var subSys string
switch idpCfgType {
case madmin.OpenidIDPCfg:
subSys = config.IdentityOpenIDSubSys
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
if err != nil {
if errors.Is(err, openid.ErrProviderConfigNotFound) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
return
}
cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
if err != nil {
if err == openid.ErrProviderConfigNotFound {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
hasEnv := false
for _, ci := range cfgInfos {
if ci.IsCfg && ci.IsEnv {
hasEnv = true
break
hasEnv := false
for _, ci := range cfgInfos {
if ci.IsCfg && ci.IsEnv {
hasEnv = true
break
}
}
}
if hasEnv {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
return
}
if hasEnv {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
return
}
case madmin.LDAPIDPCfg:
subSys = config.IdentityLDAPSubSys
cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
if err != nil {
if errors.Is(err, openid.ErrProviderConfigNotFound) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
return
}
var subSys string
switch cfgType {
case "openid":
subSys = config.IdentityOpenIDSubSys
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
hasEnv := false
for _, ci := range cfgInfos {
if ci.IsCfg && ci.IsEnv {
hasEnv = true
break
}
}
if hasEnv {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
return
}
default:
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
cfg, err = readServerConfig(ctx, objectAPI)
cfg, err := readServerConfig(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
cfgKey := fmt.Sprintf("%s:%s", subSys, cfgName)
if cfgName == madmin.Default {
cfgKey = subSys
}
if err = cfg.DelKVS(cfgKey); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
+88
View File
@@ -0,0 +1,88 @@
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"encoding/json"
"net/http"
"github.com/minio/madmin-go"
"github.com/minio/minio/internal/logger"
iampolicy "github.com/minio/pkg/iam/policy"
)
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
//
// GET <admin-prefix>/idp/ldap/policy-entities?[query-params]
//
// Query params:
//
// user=... -> repeatable query parameter, specifying users to query for
// policy mapping
//
// group=... -> repeatable query parameter, specifying groups to query for
// policy mapping
//
// policy=... -> repeatable query parameter, specifying policy to query for
// user/group mapping
//
// When all query parameters are omitted, returns mappings for all policies.
func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ListLDAPPolicyMappingEntities")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
// Check authorization.
objectAPI, cred := validateAdminReq(ctx, w, r,
iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
if objectAPI == nil {
return
}
// Validate API arguments.
q := madmin.PolicyEntitiesQuery{
Users: r.Form["user"],
Groups: r.Form["group"],
Policy: r.Form["policy"],
}
// Query IAM
res, err := globalIAMSys.QueryLDAPPolicyEntities(r.Context(), q)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Encode result and send response.
data, err := json.Marshal(res)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
password := cred.SecretKey
econfigData, err := madmin.EncryptData(password, data)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, econfigData)
}
+145
View File
@@ -19,6 +19,7 @@ package cmd
import (
"encoding/json"
"errors"
"fmt"
"net/http"
@@ -27,6 +28,11 @@ import (
iampolicy "github.com/minio/pkg/iam/policy"
)
var (
errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is aleady in progress")
errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
)
func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "StartDecommission")
@@ -49,6 +55,11 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
return
}
if pools.IsRebalanceStarted() {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errDecommissionRebalanceAlreadyRunning), r.URL)
return
}
vars := mux.Vars(r)
v := vars["pool"]
@@ -200,3 +211,137 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
}
func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "RebalanceStart")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
if objectAPI == nil {
return
}
// NB rebalance-start admin API is always coordinated from first pool's
// first node. The following is required to serialize (the effects of)
// concurrent rebalance-start commands.
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
for nodeIdx, proxyEp := range globalProxyEndpoints {
if proxyEp.Endpoint.Host == ep.Host {
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
return
}
}
}
}
pools, ok := objectAPI.(*erasureServerPools)
if !ok || len(pools.serverPools) == 1 {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
if pools.IsDecommissionRunning() {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errRebalanceDecommissionAlreadyRunning), r.URL)
return
}
if pools.IsRebalanceStarted() {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL)
return
}
bucketInfos, err := objectAPI.ListBuckets(ctx, BucketOptions{})
if err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
buckets := make([]string, 0, len(bucketInfos))
for _, bInfo := range bucketInfos {
buckets = append(buckets, bInfo.Name)
}
var id string
if id, err = pools.initRebalanceMeta(ctx, buckets); err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
// Rebalance routine is run on the first node of any pool participating in rebalance.
pools.StartRebalance()
b, err := json.Marshal(struct {
ID string `json:"id"`
}{ID: id})
if err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, b)
// Notify peers to load rebalance.bin and start rebalance routine if they happen to be
// participating pool's leader node
globalNotificationSys.LoadRebalanceMeta(ctx, true)
}
func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "RebalanceStatus")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
if objectAPI == nil {
return
}
// Proxy rebalance-status to first pool first node, so that users see a
// consistent view of rebalance progress even though different rebalancing
// pools may temporarily have out of date info on the others.
if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
for nodeIdx, proxyEp := range globalProxyEndpoints {
if proxyEp.Endpoint.Host == ep.Host {
if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
return
}
}
}
}
pools, ok := objectAPI.(*erasureServerPools)
if !ok {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
rs, err := rebalanceStatus(ctx, pools)
if err != nil {
if errors.Is(err, errRebalanceNotStarted) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL)
return
}
logger.LogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
logger.LogIf(r.Context(), json.NewEncoder(w).Encode(rs))
}
func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "RebalanceStop")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
if objectAPI == nil {
return
}
pools, ok := objectAPI.(*erasureServerPools)
if !ok {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Cancel any ongoing rebalance operation
globalNotificationSys.StopRebalance(r.Context())
writeSuccessResponseHeadersOnly(w)
logger.LogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
}
+26 -5
View File
@@ -22,8 +22,9 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"strings"
"time"
"github.com/gorilla/mux"
"github.com/minio/madmin-go"
@@ -116,19 +117,38 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
_, isLockEnabled := r.Form["lockEnabled"]
_, isVersioningEnabled := r.Form["versioningEnabled"]
_, isForceCreate := r.Form["forceCreate"]
opts := BucketOptions{
createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
if cerr != nil {
createdAt = timeSentinel
}
opts := MakeBucketOptions{
Location: r.Form.Get("location"),
LockEnabled: isLockEnabled,
VersioningEnabled: isVersioningEnabled,
ForceCreate: isForceCreate,
CreatedAt: createdAt,
}
err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
case madmin.ConfigureReplBktOp:
err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
case madmin.DeleteBucketBktOp:
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
_, noRecreate := r.Form["noRecreate"]
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
Force: false,
NoRecreate: noRecreate,
SRDeleteOp: getSRBucketDeleteOp(true),
})
case madmin.ForceDeleteBucketBktOp:
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
_, noRecreate := r.Form["noRecreate"]
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
Force: true,
NoRecreate: noRecreate,
SRDeleteOp: getSRBucketDeleteOp(true),
})
case madmin.PurgeDeletedBucketOp:
globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
}
if err != nil {
logger.LogIf(ctx, err)
@@ -298,7 +318,7 @@ func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Re
}
func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
data, err := ioutil.ReadAll(body)
data, err := io.ReadAll(body)
if err != nil {
return SRError{
Cause: err,
@@ -436,6 +456,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
opts.Users = q.Get("users") == "true"
opts.Entity = madmin.GetSREntityType(q.Get("entity"))
opts.EntityValue = q.Get("entityvalue")
opts.ShowDeleted = q.Get("showDeleted") == "true"
return
}
+79 -34
View File
@@ -22,7 +22,6 @@ import (
"encoding/json"
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"sort"
@@ -136,7 +135,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
// Add ldap users which have mapped policies if in LDAP mode
// FIXME(vadmeste): move this to policy info in the future
ldapUsers, err := globalIAMSys.ListLDAPUsers()
ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx)
if err != nil && err != errIAMActionNotAllowed {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -229,7 +228,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
return
}
data, err := ioutil.ReadAll(r.Body)
data, err := io.ReadAll(r.Body)
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
@@ -658,7 +657,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
// In case of LDAP we need to resolve the targetUser to a DN and
// query their groups:
if globalLDAPConfig.Enabled {
if globalLDAPConfig.Enabled() {
opts.claims[ldapUserN] = targetUser // simple username
targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
if err != nil {
@@ -1150,15 +1149,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
return rd, wr
}
var dataUsageInfo DataUsageInfo
var err error
if !globalIsGateway {
// Load the latest calculated data usage
dataUsageInfo, _ = loadDataUsageFromBackend(ctx, objectAPI)
}
// Load the latest calculated data usage
dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
// If etcd, dns federation configured list buckets from etcd.
var buckets []BucketInfo
var err error
if globalDNSConfig != nil && globalBucketFederation {
dnsBuckets, err := globalDNSConfig.List()
if err != nil && !IsErrIgnored(err,
@@ -1177,7 +1173,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
return buckets[i].Name < buckets[j].Name
})
} else {
buckets, err = objectAPI.ListBuckets(ctx)
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -1189,14 +1185,40 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
// For derived credentials, check the parent user's permissions.
accountName = cred.ParentUser
}
policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
if err != nil {
logger.LogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
buf, err := json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
roleArn := iampolicy.Args{Claims: claims}.GetRoleArn()
var effectivePolicy iampolicy.Policy
var buf []byte
switch {
case accountName == globalActiveCred.AccessKey:
for _, policy := range iampolicy.DefaultPolicies {
if policy.Name == "consoleAdmin" {
effectivePolicy = policy.Definition
break
}
}
case roleArn != "":
_, policy, err := globalIAMSys.GetRolePolicy(roleArn)
if err != nil {
logger.LogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
policySlice := newMappedPolicy(policy).toSlice()
effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
default:
policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
if err != nil {
logger.LogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
}
buf, err = json.MarshalIndent(effectivePolicy, "", " ")
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -1449,7 +1471,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
return
}
iamPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
iamPolicyBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -1514,7 +1536,29 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
}
}
updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup)
// Validate that user or group exists.
if !isGroup {
if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
_, ok := globalIAMSys.GetUser(ctx, entityName)
if !ok {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
return
}
}
} else {
_, err := globalIAMSys.GetGroupDescription(entityName)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
userType := regUser
if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
userType = stsUser
}
updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, userType, isGroup)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -1524,6 +1568,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
Type: madmin.SRIAMItemPolicyMapping,
PolicyMapping: &madmin.SRPolicyMapping{
UserOrGroup: entityName,
UserType: int(userType),
IsGroup: isGroup,
Policy: policyName,
},
@@ -1810,7 +1855,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
return
}
data, err := ioutil.ReadAll(r.Body)
data, err := io.ReadAll(r.Body)
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
@@ -1833,7 +1878,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var allPolicies map[string]iampolicy.Policy
data, err = ioutil.ReadAll(f)
data, err = io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
return
@@ -1868,7 +1913,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var userAccts map[string]madmin.AddOrUpdateUserReq
data, err := ioutil.ReadAll(f)
data, err := io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allUsersFile, ""), r.URL)
return
@@ -1945,7 +1990,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var grpInfos map[string]GroupInfo
data, err := ioutil.ReadAll(f)
data, err := io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allGroupsFile, ""), r.URL)
return
@@ -1983,7 +2028,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var serviceAcctReqs map[string]madmin.SRSvcAccCreate
data, err := ioutil.ReadAll(f)
data, err := io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allSvcAcctsFile, ""), r.URL)
return
@@ -2049,7 +2094,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
// In case of LDAP we need to resolve the targetUser to a DN and
// query their groups:
if globalLDAPConfig.Enabled {
if globalLDAPConfig.Enabled() {
opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
if err != nil {
@@ -2079,7 +2124,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var userPolicyMap map[string]MappedPolicy
data, err := ioutil.ReadAll(f)
data, err := io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, userPolicyMappingsFile, ""), r.URL)
return
@@ -2099,7 +2144,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, userPolicyMappingsFile, u), r.URL)
return
}
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil {
writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
return
}
@@ -2118,7 +2163,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var grpPolicyMap map[string]MappedPolicy
data, err := ioutil.ReadAll(f)
data, err := io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, groupPolicyMappingsFile, ""), r.URL)
return
@@ -2128,7 +2173,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
return
}
for g, pm := range grpPolicyMap {
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
return
}
@@ -2147,7 +2192,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var userPolicyMap map[string]MappedPolicy
data, err := ioutil.ReadAll(f)
data, err := io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsUserPolicyMappingsFile, ""), r.URL)
return
@@ -2167,7 +2212,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL)
return
}
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil {
writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
return
}
@@ -2186,7 +2231,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
default:
defer f.Close()
var grpPolicyMap map[string]MappedPolicy
data, err := ioutil.ReadAll(f)
data, err := io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
return
@@ -2196,7 +2241,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
return
}
for g, pm := range grpPolicyMap {
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
return
}
+65 -3
View File
@@ -24,7 +24,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"net/url"
"os"
@@ -34,7 +34,7 @@ import (
"time"
"github.com/minio/madmin-go"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
cr "github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -388,7 +388,7 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
req.ContentLength = int64(len(buf))
sum := sha256.Sum256(buf)
req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:]))
req.Body = ioutil.NopCloser(bytes.NewReader(buf))
req.Body = io.NopCloser(bytes.NewReader(buf))
req = signer.SignV4(*req, accessKey, secretKey, "", "")
// 3.1 Execute the request.
@@ -1141,6 +1141,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
}
func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials {
c.Helper()
randUser := mustGetUUID()
randPass := mustGetUUID()
err := admClnt.AddUser(ctx, randUser, randPass)
@@ -1154,6 +1155,7 @@ func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClie
}
func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminClient, accessKey string) madmin.UserInfo {
c.Helper()
ui, err := admClnt.GetUserInfo(ctx, accessKey)
if err != nil {
c.Fatalf("should be able to get user info: %v", err)
@@ -1162,6 +1164,7 @@ func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminCli
}
func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) {
c.Helper()
randUser := mustGetUUID()
randPass := mustGetUUID()
err := admClnt.AddUser(ctx, randUser, randPass)
@@ -1171,6 +1174,7 @@ func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminC
}
func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) madmin.Credentials {
c.Helper()
cr, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
TargetUser: tgtUser,
})
@@ -1181,6 +1185,7 @@ func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admCln
}
func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) {
c.Helper()
_, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
TargetUser: tgtUser,
})
@@ -1190,6 +1195,7 @@ func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, adm
}
func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bucket string) {
c.Helper()
res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
v, ok := <-res
if !ok || v.Err == nil {
@@ -1197,7 +1203,49 @@ func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bu
}
}
func (c *check) mustPutObjectWithTags(ctx context.Context, client *minio.Client, bucket, object string) {
c.Helper()
_, err := client.PutObject(ctx, bucket, object, bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{
UserTags: map[string]string{
"security": "public",
"virus": "true",
},
})
if err != nil {
c.Fatalf("user was unable to upload the object: %v", err)
}
}
func (c *check) mustGetObject(ctx context.Context, client *minio.Client, bucket, object string) {
c.Helper()
r, err := client.GetObject(ctx, bucket, object, minio.GetObjectOptions{})
if err != nil {
c.Fatalf("user was unable to download the object: %v", err)
}
defer r.Close()
_, err = io.Copy(io.Discard, r)
if err != nil {
c.Fatalf("user was unable to download the object: %v", err)
}
}
func (c *check) mustHeadObject(ctx context.Context, client *minio.Client, bucket, object string, tagCount int) {
c.Helper()
oinfo, err := client.StatObject(ctx, bucket, object, minio.StatObjectOptions{})
if err != nil {
c.Fatalf("user was unable to download the object: %v", err)
}
if oinfo.UserTagCount != tagCount {
c.Fatalf("expected tagCount: %d, got %d", tagCount, oinfo.UserTagCount)
}
}
func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) {
c.Helper()
res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
v, ok := <-res
if ok && v.Err != nil {
@@ -1205,7 +1253,16 @@ func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucke
}
}
func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) {
c.Helper()
_, err := client.ListBuckets(ctx)
if err != nil {
c.Fatalf("user was unable to list buckets: %v", err)
}
}
func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
c.Helper()
_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
if e, ok := err.(minio.ErrorResponse); ok {
if e.Code == "AccessDenied" {
@@ -1221,6 +1278,7 @@ func (c *check) assertSvcAccS3Access(ctx context.Context, s *TestSuiteIAM, cr ma
}
func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *madmin.AdminClient, parentAK, svcAK string) {
c.Helper()
listResp, err := madmClient.ListServiceAccounts(ctx, parentAK)
if err != nil {
c.Fatalf("unable to list svc accounts: %v", err)
@@ -1246,6 +1304,7 @@ func (c *check) assertSvcAccInfoQueryable(ctx context.Context, madmClient *madmi
// bucket. It creates a session policy that restricts listing on the bucket and
// then enables it again in a session policy update call.
func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
c.Helper()
svcAK, svcSK := mustGenerateCredentials(c)
// This policy does not allow listing objects.
@@ -1301,6 +1360,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
}
func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
c.Helper()
svcAK, svcSK := mustGenerateCredentials(c)
cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
TargetUser: accessKey,
@@ -1337,6 +1397,7 @@ func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *Tes
}
func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
c.Helper()
svcAK, svcSK := mustGenerateCredentials(c)
cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
TargetUser: accessKey,
@@ -1357,6 +1418,7 @@ func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmC
}
func mustGenerateCredentials(c *check) (string, string) {
c.Helper()
ak, sk, err := auth.GenerateCredentials()
if err != nil {
c.Fatalf("unable to generate credentials: %v", err)
+318 -231
View File
@@ -21,13 +21,16 @@ import (
"bytes"
"context"
crand "crypto/rand"
"crypto/rsa"
"crypto/subtle"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"hash/crc32"
"io"
"io/ioutil"
"math"
"math/rand"
"net/http"
@@ -45,13 +48,13 @@ import (
"github.com/gorilla/mux"
"github.com/klauspost/compress/zip"
"github.com/minio/madmin-go"
"github.com/minio/madmin-go/estream"
"github.com/minio/minio/internal/dsync"
"github.com/minio/minio/internal/handlers"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/logger/message/log"
"github.com/minio/minio/internal/pubsub"
iampolicy "github.com/minio/pkg/iam/policy"
xnet "github.com/minio/pkg/net"
"github.com/secure-io/sio-go"
@@ -71,16 +74,6 @@ const (
mgmtForceStop = "forceStop"
)
func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string, mode string) (us madmin.ServerUpdateStatus, err error) {
if err = doUpdate(u, lrTime, sha256Sum, releaseInfo, mode); err != nil {
return us, err
}
us.CurrentVersion = Version
us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout)
return us, nil
}
// ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
// ----------
// updates all minio servers and restarts them gracefully.
@@ -152,7 +145,16 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
return
}
for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
// Download Binary Once
reader, err := downloadBinary(u, mode)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Push binary to other servers
for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader) {
if nerr.Err != nil {
err := AdminError{
Code: AdminUpdateApplyFailure,
@@ -166,13 +168,39 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
}
}
updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
err = verifyBinary(u, sha256Sum, releaseInfo, mode, reader)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
for _, nerr := range globalNotificationSys.CommitBinary(ctx) {
if nerr.Err != nil {
err := AdminError{
Code: AdminUpdateApplyFailure,
Message: nerr.Err.Error(),
StatusCode: http.StatusInternalServerError,
}
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
err = commitBinary()
if err != nil {
logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
updateStatus := madmin.ServerUpdateStatus{
CurrentVersion: Version,
UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
}
// Marshal API response
jsonBytes, err := json.Marshal(updateStatus)
if err != nil {
@@ -363,18 +391,34 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
if err != nil || interval < time.Second {
interval = defaultMetricsInterval
}
n, err := strconv.Atoi(r.Form.Get("n"))
if err != nil || n <= 0 {
n = math.MaxInt32
}
var types madmin.MetricType
if t, _ := strconv.ParseUint(r.Form.Get("types"), 10, 64); t != 0 {
types = madmin.MetricType(t)
} else {
types = madmin.MetricsAll
}
disks := strings.Split(r.Form.Get("disks"), ",")
byDisk := strings.EqualFold(r.Form.Get("by-disk"), "true")
var diskMap map[string]struct{}
if len(disks) > 0 && disks[0] != "" {
diskMap = make(map[string]struct{}, len(disks))
for _, k := range disks {
if k != "" {
diskMap[k] = struct{}{}
}
}
}
jobID := r.Form.Get("by-jobID")
hosts := strings.Split(r.Form.Get("hosts"), ",")
byhost := strings.EqualFold(r.Form.Get("by-host"), "true")
byHost := strings.EqualFold(r.Form.Get("by-host"), "true")
var hostMap map[string]struct{}
if len(hosts) > 0 && hosts[0] != "" {
hostMap = make(map[string]struct{}, len(hosts))
@@ -384,34 +428,42 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
}
}
}
done := ctx.Done()
ticker := time.NewTicker(interval)
defer ticker.Stop()
w.Header().Set(xhttp.ContentType, string(mimeJSON))
enc := json.NewEncoder(w)
for n > 0 {
var m madmin.RealtimeMetrics
mLocal := collectLocalMetrics(types, hostMap)
mLocal := collectLocalMetrics(types, collectMetricsOpts{
hosts: hostMap,
disks: diskMap,
jobID: jobID,
})
m.Merge(&mLocal)
// Allow half the interval for collecting remote...
cctx, cancel := context.WithTimeout(ctx, interval/2)
mRemote := collectRemoteMetrics(cctx, types, hostMap)
mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{
hosts: hostMap,
disks: diskMap,
jobID: jobID,
})
cancel()
m.Merge(&mRemote)
if !byhost {
if !byHost {
m.ByHost = nil
}
if !byDisk {
m.ByDisk = nil
}
m.Final = n <= 1
// Marshal API response
jsonBytes, err := json.Marshal(m)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
_, err = w.Write(jsonBytes)
if err != nil {
// Marshal API reesponse
if err := enc.Encode(&m); err != nil {
n = 0
}
@@ -718,7 +770,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
}
}
// read request body
io.CopyN(ioutil.Discard, r.Body, 1)
io.CopyN(io.Discard, r.Body, 1)
globalProfilerMu.Lock()
@@ -896,11 +948,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
return
}
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
return
}
hip, errCode := extractHealInitParams(mux.Vars(r), r.Form, r.Body)
if errCode != ErrNone {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
@@ -1087,12 +1134,6 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
return
}
// Check if this setup has an erasure coded backend.
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
return
}
aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1168,11 +1209,6 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
return
}
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
sizeStr := r.Form.Get(peerRESTSize)
durationStr := r.Form.Get(peerRESTDuration)
concurrentStr := r.Form.Get(peerRESTConcurrent)
@@ -1190,16 +1226,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
concurrent = 32
}
if runtime.GOMAXPROCS(0) < concurrent {
concurrent = runtime.GOMAXPROCS(0)
}
duration, err := time.ParseDuration(durationStr)
if err != nil {
duration = time.Second * 10
}
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
storageInfo, _ := objectAPI.StorageInfo(ctx)
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
if !sufficientCapacity {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
Code: "XMinioSpeedtestInsufficientCapacity",
@@ -1249,14 +1283,24 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
storageClass: storageClass,
bucketName: customBucket,
})
var prevResult madmin.SpeedTestResult
for {
select {
case <-ctx.Done():
return
case <-keepAliveTicker.C:
// Write a blank entry to prevent client from disconnecting
if err := enc.Encode(madmin.SpeedTestResult{}); err != nil {
return
// if previous result is set keep writing the
// previous result back to the client
if prevResult.Version != "" {
if err := enc.Encode(prevResult); err != nil {
return
}
} else {
// first result is not yet obtained, keep writing
// empty entry to prevent client from disconnecting.
if err := enc.Encode(madmin.SpeedTestResult{}); err != nil {
return
}
}
w.(http.Flusher).Flush()
case result, ok := <-ch:
@@ -1266,13 +1310,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
if err := enc.Encode(result); err != nil {
return
}
prevResult = result
w.(http.Flusher).Flush()
}
}
}
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{}); err != nil {
if _, ok := err.(BucketExists); !ok {
// Only BucketExists error can be ignored.
return false, err
@@ -1286,11 +1331,11 @@ func deleteObjectPerfBucket(objectAPI ObjectLayer) {
objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
Force: true,
NoRecreate: true,
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
})
}
func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
storageInfo, _ := objectAPI.StorageInfo(ctx)
func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo, concurrent int, size int, autotune bool) (bool, bool, string) {
capacityNeeded := uint64(concurrent * size)
capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)
@@ -1332,11 +1377,6 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
return
}
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1468,16 +1508,12 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
// Use buffered channel to take care of burst sends or slow w.Write()
traceCh := make(chan pubsub.Maskable, 4000)
traceCh := make(chan madmin.TraceInfo, 4000)
peers, _ := newPeerRestClients(globalEndpoints)
mask := pubsub.MaskFromMaskable(traceOpts.TraceTypes())
err = globalTrace.Subscribe(mask, traceCh, ctx.Done(), func(entry pubsub.Maskable) bool {
if e, ok := entry.(madmin.TraceInfo); ok {
return shouldTrace(e, traceOpts)
}
return false
err = globalTrace.Subscribe(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
return shouldTrace(entry, traceOpts)
})
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
@@ -1549,7 +1585,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
setEventStreamHeaders(w)
logCh := make(chan pubsub.Maskable, 4000)
logCh := make(chan log.Info, 4000)
peers, _ := newPeerRestClients(globalEndpoints)
@@ -1575,9 +1611,11 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
for {
select {
case entry := <-logCh:
log, ok := entry.(log.Info)
if ok && log.SendLog(node, logKind) {
case log, ok := <-logCh:
if !ok {
return
}
if log.SendLog(node, logKind) {
if err := enc.Encode(log); err != nil {
return
}
@@ -1741,8 +1779,8 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
kmsStat := fetchKMSStatus()
ldap := madmin.LDAP{}
if globalLDAPConfig.Enabled {
ldapConn, err := globalLDAPConfig.Connect()
if globalLDAPConfig.Enabled() {
ldapConn, err := globalLDAPConfig.LDAP.Connect()
//nolint:gocritic
if err != nil {
ldap.Status = string(madmin.ItemOffline)
@@ -1828,7 +1866,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
Mode: string(mode),
Domain: domain,
Region: globalSite.Region,
SQSARN: globalNotificationSys.GetARNList(false),
SQSARN: globalEventNotifier.GetARNList(false),
DeploymentID: globalDeploymentID,
Buckets: buckets,
Objects: objects,
@@ -1853,7 +1891,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
}
client := &http.Client{
Transport: NewGatewayHTTPTransport(),
Transport: NewHTTPTransport(),
Timeout: 10 * time.Second,
}
@@ -1911,11 +1949,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
}
deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
if query.Get(string(madmin.HealthDataTypePerfNet)) != "" ||
query.Get(string(madmin.HealthDataTypePerfDrive)) != "" ||
query.Get(string(madmin.HealthDataTypePerfObj)) != "" {
deadline = 1 * time.Hour
}
if dstr := r.Form.Get("deadline"); dstr != "" {
var err error
deadline, err = time.ParseDuration(dstr)
@@ -1936,12 +1969,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
defer healthCancel()
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
// unfreeze all incoming S3 API calls after speedtest.
defer globalNotificationSys.ServiceFreeze(ctx, false)
hostAnonymizer := createHostAnonymizer()
// anonAddr - Anonymizes hosts in given input string.
anonAddr := func(addr string) string {
@@ -2087,7 +2114,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
if !globalIsDistErasure {
// FS mode - single server - hard code to `server1`
anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
return strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
if len(globalMinioConsoleHost) > 0 {
anonCmdLine = strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
}
return anonCmdLine
}
// Server start command regex groups:
@@ -2177,89 +2207,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
}
}
getAndWriteDrivePerfInfo := func() {
if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" {
opts := madmin.DriveSpeedTestOpts{
Serial: false,
BlockSize: 4 * humanize.MiByte,
FileSize: 1 * humanize.GiByte,
}
localDPI := driveSpeedTest(ctx, opts)
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, localDPI)
perfCh := globalNotificationSys.DriveSpeedTest(ctx, opts)
for perfInfo := range perfCh {
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, perfInfo)
}
partialWrite(healthInfo)
}
}
getAndWriteObjPerfInfo := func() {
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
concurrent := 32
if runtime.GOMAXPROCS(0) < concurrent {
concurrent = runtime.GOMAXPROCS(0)
}
size := 64 * humanize.MiByte
autotune := true
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
if !sufficientCapacity {
healthInfo.Perf.Error = capacityErrMsg
partialWrite(healthInfo)
return
}
if !canAutotune {
autotune = false
}
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, globalObjectPerfBucket)
if err != nil {
healthInfo.Perf.Error = "Unable to create bucket: " + err.Error()
partialWrite(healthInfo)
return
}
if !bucketExists {
defer deleteObjectPerfBucket(objectAPI)
}
opts := speedTestOpts{
objectSize: size,
concurrencyStart: concurrent,
duration: 10 * time.Second,
autotune: autotune,
}
perfCh := objectSpeedTest(ctx, opts)
for perfInfo := range perfCh {
healthInfo.Perf.ObjPerf = append(healthInfo.Perf.ObjPerf, perfInfo)
}
partialWrite(healthInfo)
}
}
getAndWriteNetPerfInfo := func() {
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
if !globalIsDistErasure {
return
}
netPerf := globalNotificationSys.Netperf(ctx, time.Second*10)
for _, np := range netPerf {
np.Endpoint = anonAddr(np.Endpoint)
healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np)
}
partialWrite(healthInfo)
}
}
anonymizeNetwork := func(network map[string]string) map[string]string {
anonNetwork := map[string]string{}
for endpoint, status := range network {
@@ -2289,16 +2236,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
getAndWriteMemInfo()
getAndWriteProcInfo()
getAndWriteMinioConfig()
getAndWriteDrivePerfInfo()
getAndWriteObjPerfInfo()
getAndWriteNetPerfInfo()
getAndWriteSysErrors()
getAndWriteSysServices()
getAndWriteSysConfig()
if query.Get("minioinfo") == "true" {
infoMessage := getServerInfo(ctx, r)
servers := []madmin.ServerInfo{}
servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
for _, server := range infoMessage.Servers {
anonEndpoint := anonAddr(server.Endpoint)
servers = append(servers, madmin.ServerInfo{
@@ -2317,6 +2261,11 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
Frees: server.MemStats.Frees,
HeapAlloc: server.MemStats.HeapAlloc,
},
GoMaxProcs: server.GoMaxProcs,
NumCPU: server.NumCPU,
RuntimeVersion: server.RuntimeVersion,
GCStats: server.GCStats,
MinioEnvVars: server.MinioEnvVars,
})
}
@@ -2509,20 +2458,6 @@ func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
lambdaMap[targetID.Name] = list
}
for _, tgt := range globalEnvTargetList.Targets() {
targetIDStatus := make(map[string]madmin.Status)
active, _ := tgt.IsActive()
targetID := tgt.ID()
if active {
targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)}
} else {
targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)}
}
list := lambdaMap[targetID.Name]
list = append(list, targetIDStatus)
lambdaMap[targetID.Name] = list
}
notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
counter := 0
for key, value := range lambdaMap {
@@ -2636,6 +2571,94 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
return nil
}
func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
// Send profiling data to zip as file
header, zerr := zip.FileInfoHeader(dummyFileInfo{
name: name,
size: int64(len(data)),
mode: 0o600,
modTime: UTCNow(),
isDir: false,
sys: nil,
})
if zerr != nil {
return zerr
}
header.Method = zip.Deflate
zwriter, zerr := zipWriter.CreateHeader(header)
if zerr != nil {
return zerr
}
_, err := io.Copy(zwriter, bytes.NewReader(data))
return err
}
// getClusterMetaInfo gets information of the current cluster and
// returns it.
// This is not a critical function, and it is allowed
// to fail with a ten seconds timeout, returning nil.
func getClusterMetaInfo(ctx context.Context) []byte {
objectAPI := newObjectLayerFn()
if objectAPI == nil {
return nil
}
// Add a ten seconds timeout because getting profiling data
// is critical for debugging, in contrary to getting cluster info
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
resultCh := make(chan madmin.ClusterRegistrationInfo)
go func() {
ci := madmin.ClusterRegistrationInfo{}
ci.Info.NoOfServerPools = len(globalEndpoints)
ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
ci.Info.MinioVersion = Version
si, _ := objectAPI.StorageInfo(ctx)
ci.Info.NoOfDrives = len(si.Disks)
for _, disk := range si.Disks {
ci.Info.TotalDriveSpace += disk.TotalSpace
ci.Info.UsedDriveSpace += disk.UsedSpace
}
dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
ci.UsedCapacity = dataUsageInfo.ObjectsTotalSize
ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount
ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount
ci.DeploymentID = globalDeploymentID
ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
resultCh <- ci
}()
select {
case <-ctx.Done():
return nil
case ci := <-resultCh:
out, err := json.MarshalIndent(ci, "", " ")
if err != nil {
logger.LogIf(ctx, err)
return nil
}
return out
}
}
func bytesToPublicKey(pub []byte) (*rsa.PublicKey, error) {
block, _ := pem.Decode(pub)
if block != nil {
pub = block.Bytes
}
key, err := x509.ParsePKCS1PublicKey(pub)
if err != nil {
return nil, err
}
return key, nil
}
// getRawDataer provides an interface for getting raw FS files.
type getRawDataer interface {
GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
@@ -2655,18 +2678,24 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
}
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
o, ok := newObjectLayerFn().(getRawDataer)
objLayer := newObjectLayerFn()
o, ok := objLayer.(getRawDataer)
if !ok {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
if err := parseForm(r); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
volume := r.Form.Get("volume")
file := r.Form.Get("file")
if len(volume) == 0 {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
return
}
file := r.Form.Get("file")
if len(file) == 0 {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
return
@@ -2679,41 +2708,102 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
return
}
var key [32]byte
// MUST use crypto/rand
n, err := crand.Read(key[:])
if err != nil || n != len(key) {
logger.LogIf(ctx, err)
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
return
}
stream, err := sio.AES_256_GCM.Stream(key[:])
if err != nil {
logger.LogIf(ctx, err)
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
return
}
// Zero nonce, we only use each key once, and 32 bytes is plenty.
nonce := make([]byte, stream.NonceSize())
encw := stream.EncryptWriter(w, nonce, nil)
var publicKey *rsa.PublicKey
defer encw.Close()
publicKeyB64 := r.Form.Get("public-key")
if publicKeyB64 != "" {
publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyB64)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
publicKey, err = bytesToPublicKey(publicKeyBytes)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
// Write a version for making *incompatible* changes.
// The AdminClient will reject any version it does not know.
w.Write([]byte{1})
var inspectZipW *zip.Writer
if publicKey != nil {
w.WriteHeader(200)
stream := estream.NewWriter(w)
defer stream.Close()
// Write key first (without encryption)
_, err = w.Write(key[:])
if err != nil {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
return
clusterKey, err := bytesToPublicKey(subnetAdminPublicKey)
if err != nil {
logger.LogIf(ctx, stream.AddError(err.Error()))
return
}
err = stream.AddKeyEncrypted(clusterKey)
if err != nil {
logger.LogIf(ctx, stream.AddError(err.Error()))
return
}
if b := getClusterMetaInfo(ctx); len(b) > 0 {
w, err := stream.AddEncryptedStream("cluster.info", nil)
if err != nil {
logger.LogIf(ctx, err)
return
}
w.Write(b)
w.Close()
}
// Add new key for inspect data.
if err := stream.AddKeyEncrypted(publicKey); err != nil {
logger.LogIf(ctx, stream.AddError(err.Error()))
return
}
encStream, err := stream.AddEncryptedStream("inspect.zip", nil)
if err != nil {
logger.LogIf(ctx, stream.AddError(err.Error()))
return
}
defer encStream.Close()
inspectZipW = zip.NewWriter(encStream)
defer inspectZipW.Close()
} else {
// Legacy: Remove if we stop supporting inspection without public key.
var key [32]byte
// MUST use crypto/rand
n, err := crand.Read(key[:])
if err != nil || n != len(key) {
logger.LogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Write a version for making *incompatible* changes.
// The AdminClient will reject any version it does not know.
if publicKey == nil {
w.Write([]byte{1})
w.Write(key[:])
}
stream, err := sio.AES_256_GCM.Stream(key[:])
if err != nil {
logger.LogIf(ctx, err)
return
}
// Zero nonce, we only use each key once, and 32 bytes is plenty.
nonce := make([]byte, stream.NonceSize())
encw := stream.EncryptWriter(w, nonce, nil)
defer encw.Close()
// Initialize a zip writer which will provide a zipped content
// of profiling data of all nodes
inspectZipW = zip.NewWriter(encw)
defer inspectZipW.Close()
if b := getClusterMetaInfo(ctx); len(b) > 0 {
logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b))
}
}
// Initialize a zip writer which will provide a zipped content
// of profiling data of all nodes
zipWriter := zip.NewWriter(encw)
defer zipWriter.Close()
rawDataFn := func(r io.Reader, host, disk, filename string, si StatInfo) error {
// Prefix host+disk
filename = path.Join(host, disk, filename)
@@ -2742,17 +2832,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
return nil
}
header.Method = zip.Deflate
zwriter, zerr := zipWriter.CreateHeader(header)
zwriter, zerr := inspectZipW.CreateHeader(header)
if zerr != nil {
logger.LogIf(ctx, zerr)
return nil
}
if _, err = io.Copy(zwriter, r); err != nil {
if _, err := io.Copy(zwriter, r); err != nil {
logger.LogIf(ctx, err)
}
return nil
}
err = o.GetRawData(ctx, volume, file, rawDataFn)
err := o.GetRawData(ctx, volume, file, rawDataFn)
if !errors.Is(err, errFileNotFound) {
logger.LogIf(ctx, err)
}
@@ -2764,20 +2854,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
if !errors.Is(err, errFileNotFound) {
logger.LogIf(ctx, err)
}
// save args passed to inspect command
inspectArgs := []string{fmt.Sprintf(" Inspect path: %s%s%s\n", volume, slashSeparator, file)}
cmdLine := []string{"Server command line args: "}
var sb bytes.Buffer
fmt.Fprintf(&sb, "Inspect path: %s%s%s\n", volume, slashSeparator, file)
sb.WriteString("Server command line args:")
for _, pool := range globalEndpoints {
cmdLine = append(cmdLine, pool.CmdLine)
}
cmdLine = append(cmdLine, "\n")
inspectArgs = append(inspectArgs, cmdLine...)
inspectArgsBytes := []byte(strings.Join(inspectArgs, " "))
if err = rawDataFn(bytes.NewReader(inspectArgsBytes), "", "", "inspect-input.txt", StatInfo{
Size: int64(len(inspectArgsBytes)),
}); err != nil {
logger.LogIf(ctx, err)
sb.WriteString(" ")
sb.WriteString(pool.CmdLine)
}
sb.WriteString("\n")
logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
}
func createHostAnonymizerForFSMode() map[string]string {
+2 -3
View File
@@ -22,7 +22,6 @@ import (
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
@@ -74,7 +73,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
initAllSubsystems()
initAllSubsystems(ctx)
initConfigSubsystem(ctx, objLayer)
@@ -220,7 +219,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
adminTestBed.router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
resp, _ := ioutil.ReadAll(rec.Body)
resp, _ := io.ReadAll(rec.Body)
t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
http.StatusOK, rec.Code, string(resp))
}
+3 -3
View File
@@ -100,14 +100,14 @@ type allHealState struct {
}
// newHealState - initialize global heal state management
func newHealState(cleanup bool) *allHealState {
func newHealState(ctx context.Context, cleanup bool) *allHealState {
hstate := &allHealState{
healSeqMap: make(map[string]*healSequence),
healLocalDisks: make(map[Endpoint]bool),
healStatus: make(map[string]healingTracker),
}
if cleanup {
go hstate.periodicHealSeqsClean(GlobalContext)
go hstate.periodicHealSeqsClean(ctx)
}
return hstate
}
@@ -822,7 +822,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
return h.healBucket(objAPI, h.bucket, bucketsOnly)
}
buckets, err := objAPI.ListBuckets(h.ctx)
buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
if err != nil {
return errFnHealFromAPIErr(h.ctx, err)
}
+32 -14
View File
@@ -60,7 +60,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
// Info operations
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(gz(httpTraceAll(adminAPI.ServerInfoHandler)))
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/inspect-data").HandlerFunc(httpTraceHdrs(adminAPI.InspectDataHandler)).Queries("volume", "{volume:.*}", "file", "{file:.*}")
adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(httpTraceAll(adminAPI.InspectDataHandler))
// StorageInfo operations
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
@@ -84,6 +84,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
// Rebalance operations
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStart)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStatus)))
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStop)))
}
// Profiling operations - deprecated API
@@ -179,9 +184,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
// IDentity Provider configuration APIs
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg))).Queries("type", "{type:.*}")
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddIdentityProviderCfg)))
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateIdentityProviderCfg)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListIdentityProviderCfg)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg)))
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg)))
// LDAP IAM operations
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListLDAPPolicyMappingEntities)))
// -- END IAM APIs --
@@ -206,6 +216,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
// Batch job operations
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc(
gz(httpTraceHdrs(adminAPI.StartBatchJob)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
gz(httpTraceHdrs(adminAPI.ListBatchJobs)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
// Bucket migration operations
// ExportBucketMetaHandler
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
@@ -264,16 +284,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(gz(httpTraceAll(adminAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}")
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSKeyStatusHandler)))
if !globalIsGateway {
// Keep obdinfo for backward compatibility with mc
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
// -- Health API --
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
}
// Keep obdinfo for backward compatibility with mc
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
// -- Health API --
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
}
// If none of the routes match add default error handler routes
+59 -1
View File
@@ -20,10 +20,14 @@ package cmd
import (
"context"
"net/http"
"os"
"runtime"
"runtime/debug"
"strings"
"time"
"github.com/minio/madmin-go"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger"
)
@@ -67,6 +71,22 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
var memstats runtime.MemStats
runtime.ReadMemStats(&memstats)
gcStats := debug.GCStats{
// If stats.PauseQuantiles is non-empty, ReadGCStats fills
// it with quantiles summarizing the distribution of pause time.
// For example, if len(stats.PauseQuantiles) is 5, it will be
// filled with the minimum, 25%, 50%, 75%, and maximum pause times.
PauseQuantiles: make([]time.Duration, 5),
}
debug.ReadGCStats(&gcStats)
// Truncate GC stats to max 5 entries.
if len(gcStats.PauseEnd) > 5 {
gcStats.PauseEnd = gcStats.PauseEnd[len(gcStats.PauseEnd)-5:]
}
if len(gcStats.Pause) > 5 {
gcStats.Pause = gcStats.Pause[len(gcStats.Pause)-5:]
}
props := madmin.ServerProperties{
State: string(madmin.ItemInitializing),
Endpoint: addr,
@@ -81,10 +101,48 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
Frees: memstats.Frees,
HeapAlloc: memstats.HeapAlloc,
},
GoMaxProcs: runtime.GOMAXPROCS(0),
NumCPU: runtime.NumCPU(),
RuntimeVersion: runtime.Version(),
GCStats: &madmin.GCStats{
LastGC: gcStats.LastGC,
NumGC: gcStats.NumGC,
PauseTotal: gcStats.PauseTotal,
Pause: gcStats.Pause,
PauseEnd: gcStats.PauseEnd,
},
MinioEnvVars: make(map[string]string, 10),
}
sensitive := map[string]struct{}{
config.EnvAccessKey: {},
config.EnvSecretKey: {},
config.EnvRootUser: {},
config.EnvRootPassword: {},
config.EnvMinIOSubnetAPIKey: {},
config.EnvKMSSecretKey: {},
}
for _, v := range os.Environ() {
if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {
continue
}
split := strings.SplitN(v, "=", 2)
key := split[0]
value := ""
if len(split) > 1 {
value = split[1]
}
// Do not send sensitive creds.
if _, ok := sensitive[key]; ok || strings.Contains(strings.ToLower(key), "password") || strings.HasSuffix(strings.ToLower(key), "key") {
props.MinioEnvVars[key] = "*** EXISTS, REDACTED ***"
continue
}
props.MinioEnvVars[key] = value
}
objLayer := newObjectLayerFn()
if objLayer != nil && !globalIsGateway {
if objLayer != nil {
// only need Disks information in server mode.
storageInfo, _ := objLayer.LocalStorageInfo(GlobalContext)
props.State = string(madmin.ItemOnline)
+90 -9
View File
@@ -30,7 +30,8 @@ import (
"github.com/Azure/azure-storage-blob-go/azblob"
"google.golang.org/api/googleapi"
minio "github.com/minio/minio-go/v7"
"github.com/minio/madmin-go"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/bucket/lifecycle"
@@ -176,6 +177,7 @@ const (
ErrBucketAlreadyOwnedByYou
ErrInvalidDuration
ErrBucketAlreadyExists
ErrTooManyBuckets
ErrMetadataTooLarge
ErrUnsupportedMetadata
ErrMaximumExpires
@@ -196,8 +198,9 @@ const (
ErrInvalidTagDirective
// Add new error codes here.
// SSE-S3 related API errors
// SSE-S3/SSE-KMS related API errors
ErrInvalidEncryptionMethod
ErrInvalidEncryptionKeyID
// Server-Side-Encryption (with Customer provided key) related API errors.
ErrInsecureSSECustomerRequest
@@ -231,6 +234,7 @@ const (
// S3 extended errors.
ErrContentSHA256Mismatch
ErrContentChecksumMismatch
// Add new extended error codes here.
@@ -260,6 +264,7 @@ const (
ErrAdminNoSuchUser
ErrAdminNoSuchGroup
ErrAdminGroupNotEmpty
ErrAdminNoSuchJob
ErrAdminNoSuchPolicy
ErrAdminInvalidArgument
ErrAdminInvalidAccessKey
@@ -270,6 +275,10 @@ const (
ErrAdminNoSuchConfigTarget
ErrAdminConfigEnvOverridden
ErrAdminConfigDuplicateKeys
ErrAdminConfigInvalidIDPType
ErrAdminConfigLDAPValidation
ErrAdminConfigIDPCfgNameAlreadyExists
ErrAdminConfigIDPCfgNameDoesNotExist
ErrAdminCredentialsMismatch
ErrInsecureClientRequest
ErrObjectTampered
@@ -284,6 +293,10 @@ const (
ErrSiteReplicationIAMError
ErrSiteReplicationConfigMissing
// Pool rebalance errors
ErrAdminRebalanceAlreadyStarted
ErrAdminRebalanceNotStarted
// Bucket Quota error codes
ErrAdminBucketQuotaExceeded
ErrAdminNoSuchQuotaConfiguration
@@ -391,6 +404,8 @@ const (
ErrAccountNotEligible
ErrAdminServiceAccountNotFound
ErrPostPolicyConditionInvalidFormat
ErrInvalidChecksum
)
type errorCodeMap map[APIErrorCode]APIError
@@ -680,6 +695,11 @@ var errorCodes = errorCodeMap{
Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrTooManyBuckets: {
Code: "TooManyBuckets",
Description: "You have attempted to create more buckets than allowed",
HTTPStatusCode: http.StatusBadRequest,
},
ErrBucketNotEmpty: {
Code: "BucketNotEmpty",
Description: "The bucket you tried to delete is not empty",
@@ -887,7 +907,7 @@ var errorCodes = errorCodeMap{
},
ErrReplicationRemoteConnectionError: {
Code: "XMinioAdminReplicationRemoteConnectionError",
Description: "Remote service connection error - please check remote service credentials and target bucket",
Description: "Remote service connection error",
HTTPStatusCode: http.StatusNotFound,
},
ErrReplicationBandwidthLimitError: {
@@ -1072,6 +1092,11 @@ var errorCodes = errorCodeMap{
Description: "The encryption method specified is not supported",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInvalidEncryptionKeyID: {
Code: "InvalidRequest",
Description: "The specified KMS KeyID contains unsupported characters",
HTTPStatusCode: http.StatusBadRequest,
},
ErrInsecureSSECustomerRequest: {
Code: "InvalidRequest",
Description: "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
@@ -1154,11 +1179,16 @@ var errorCodes = errorCodeMap{
Description: "The provided 'x-amz-content-sha256' header does not match what was computed.",
HTTPStatusCode: http.StatusBadRequest,
},
ErrContentChecksumMismatch: {
Code: "XAmzContentChecksumMismatch",
Description: "The provided 'x-amz-checksum' header does not match what was computed.",
HTTPStatusCode: http.StatusBadRequest,
},
// MinIO extensions.
ErrStorageFull: {
Code: "XMinioStorageFull",
Description: "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
Description: "Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.",
HTTPStatusCode: http.StatusInsufficientStorage,
},
ErrRequestBodyParse: {
@@ -1206,6 +1236,11 @@ var errorCodes = errorCodeMap{
Description: "The specified group does not exist.",
HTTPStatusCode: http.StatusNotFound,
},
ErrAdminNoSuchJob: {
Code: "XMinioAdminNoSuchJob",
Description: "The specified job does not exist.",
HTTPStatusCode: http.StatusNotFound,
},
ErrAdminGroupNotEmpty: {
Code: "XMinioAdminGroupNotEmpty",
Description: "The specified group is not empty - cannot remove it.",
@@ -1262,6 +1297,26 @@ var errorCodes = errorCodeMap{
Description: "JSON configuration provided has objects with duplicate keys",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminConfigInvalidIDPType: {
Code: "XMinioAdminConfigInvalidIDPType",
Description: fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminConfigLDAPValidation: {
Code: "XMinioAdminConfigLDAPValidation",
Description: "LDAP Configuration validation failed",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminConfigIDPCfgNameAlreadyExists: {
Code: "XMinioAdminConfigIDPCfgNameAlreadyExists",
Description: "An IDP configuration with the given name aleady exists",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminConfigIDPCfgNameDoesNotExist: {
Code: "XMinioAdminConfigIDPCfgNameDoesNotExist",
Description: "No such IDP configuration exists",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminConfigNotificationTargetsFailed: {
Code: "XMinioAdminNotificationTargetsTestFailed",
Description: "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",
@@ -1358,6 +1413,16 @@ var errorCodes = errorCodeMap{
Description: "Site not found in site replication configuration",
HTTPStatusCode: http.StatusBadRequest,
},
ErrAdminRebalanceAlreadyStarted: {
Code: "XMinioAdminRebalanceAlreadyStarted",
Description: "Pool rebalance is already started",
HTTPStatusCode: http.StatusConflict,
},
ErrAdminRebalanceNotStarted: {
Code: "XMinioAdminRebalanceNotStarted",
Description: "Pool rebalance is not started",
HTTPStatusCode: http.StatusNotFound,
},
ErrMaximumExpires: {
Code: "AuthorizationQueryParametersError",
Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
@@ -1868,6 +1933,11 @@ var errorCodes = errorCodeMap{
Description: "Invalid according to Policy: Policy Condition failed",
HTTPStatusCode: http.StatusForbidden,
},
ErrInvalidChecksum: {
Code: "InvalidArgument",
Description: "Invalid checksum provided.",
HTTPStatusCode: http.StatusBadRequest,
},
// Add your error structure here.
}
@@ -1898,6 +1968,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrAdminNoSuchGroup
case errGroupNotEmpty:
apiErr = ErrAdminGroupNotEmpty
case errNoSuchJob:
apiErr = ErrAdminNoSuchJob
case errNoSuchPolicy:
apiErr = ErrAdminNoSuchPolicy
case errSignatureMismatch:
@@ -1921,6 +1993,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrInvalidEncryptionParameters
case crypto.ErrInvalidEncryptionMethod:
apiErr = ErrInvalidEncryptionMethod
case crypto.ErrInvalidEncryptionKeyID:
apiErr = ErrInvalidEncryptionKeyID
case crypto.ErrInvalidCustomerAlgorithm:
apiErr = ErrInvalidSSECustomerAlgorithm
case crypto.ErrMissingCustomerKey:
@@ -2038,6 +2112,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrSignatureDoesNotMatch
case hash.SHA256Mismatch:
apiErr = ErrContentSHA256Mismatch
case hash.ChecksumMismatch:
apiErr = ErrContentChecksumMismatch
case ObjectTooLarge:
apiErr = ErrEntityTooLarge
case ObjectTooSmall:
@@ -2066,7 +2142,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrRemoteDestinationNotFoundError
case BucketRemoteTargetNotFound:
apiErr = ErrRemoteTargetNotFoundError
case BucketRemoteConnectionErr:
case RemoteTargetConnectionErr:
apiErr = ErrReplicationRemoteConnectionError
case BucketRemoteAlreadyExists:
apiErr = ErrBucketRemoteAlreadyExists
@@ -2188,9 +2264,14 @@ func toAPIError(ctx context.Context, err error) APIError {
if apiErr.Code == "InternalError" {
// If we see an internal error try to interpret
// any underlying errors if possible depending on
// their internal error types. This code is only
// useful with gateway implementations.
// their internal error types.
switch e := err.(type) {
case batchReplicationJobError:
apiErr = APIError{
Code: e.Code,
Description: e.Description,
HTTPStatusCode: e.HTTPStatusCode,
}
case InvalidArgument:
apiErr = APIError{
Code: "InvalidArgument",
@@ -2253,7 +2334,7 @@ func toAPIError(ctx context.Context, err error) APIError {
Description: e.Message,
HTTPStatusCode: e.StatusCode,
}
if globalIsGateway && strings.Contains(e.Message, "KMS is not configured") {
if strings.Contains(e.Message, "KMS is not configured") {
apiErr = APIError{
Code: "NotImplemented",
Description: e.Message,
@@ -2277,7 +2358,7 @@ func toAPIError(ctx context.Context, err error) APIError {
Description: e.Error(),
HTTPStatusCode: e.Response().StatusCode,
}
// Add more Gateway SDKs here if any in future.
// Add more other SDK related errors here if any in future.
default:
//nolint:gocritic
if errors.Is(err, errMalformedEncoding) {
+16 -4
View File
@@ -20,7 +20,6 @@ package cmd
import (
"bytes"
"encoding/json"
"encoding/xml"
"fmt"
"net/http"
"net/url"
@@ -30,6 +29,8 @@ import (
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
xxml "github.com/minio/xxml"
)
// Returns a hexadecimal representation of time at the
@@ -64,9 +65,13 @@ func setCommonHeaders(w http.ResponseWriter) {
// Encodes the response headers into XML format.
func encodeResponse(response interface{}) []byte {
var bytesBuffer bytes.Buffer
bytesBuffer.WriteString(xml.Header)
e := xml.NewEncoder(&bytesBuffer)
e.Encode(response)
bytesBuffer.WriteString(xxml.Header)
buf, err := xxml.Marshal(response)
if err != nil {
logger.LogIf(GlobalContext, err)
return nil
}
bytesBuffer.Write(buf)
return bytesBuffer.Bytes()
}
@@ -199,5 +204,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
}
if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
if i := strings.LastIndexByte(v, '/'); i >= 0 {
v = v[i+1:]
}
w.Header()[xhttp.MinIOCompressed] = []string{v}
}
return nil
}
+101 -32
View File
@@ -31,8 +31,10 @@ import (
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/handlers"
"github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
xxml "github.com/minio/xxml"
)
const (
@@ -162,6 +164,12 @@ type Part struct {
LastModified string
ETag string
Size int64
// Checksum values
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
}
// ListPartsResponse - format for list parts response.
@@ -183,6 +191,7 @@ type ListPartsResponse struct {
MaxParts int
IsTruncated bool
ChecksumAlgorithm string
// List of parts.
Parts []Part `xml:"Part"`
}
@@ -252,45 +261,88 @@ type ObjectVersion struct {
}
// MarshalXML - marshal ObjectVersion
func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
func (o ObjectVersion) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
if o.isDeleteMarker {
start.Name.Local = "DeleteMarker"
} else {
start.Name.Local = "Version"
}
type objectVersionWrapper ObjectVersion
return e.EncodeElement(objectVersionWrapper(o), start)
}
// StringMap is a map[string]string
type StringMap map[string]string
// DeleteMarkerVersion container for delete marker metadata
type DeleteMarkerVersion struct {
Key string
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
// Owner of the object.
Owner Owner
IsLatest bool
VersionID string `xml:"VersionId"`
}
// Metadata metadata items implemented to ensure XML marshaling works.
type Metadata struct {
Items []struct {
Key string
Value string
}
}
// Set add items, duplicate items get replaced.
func (s *Metadata) Set(k, v string) {
for i, item := range s.Items {
if item.Key == k {
s.Items[i] = struct {
Key string
Value string
}{
Key: k,
Value: v,
}
return
}
}
s.Items = append(s.Items, struct {
Key string
Value string
}{
Key: k,
Value: v,
})
}
type xmlKeyEntry struct {
XMLName xml.Name
Value string `xml:",chardata"`
}
// MarshalXML - StringMap marshals into XML.
func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
tokens := []xml.Token{start}
for key, value := range s {
t := xml.StartElement{}
t.Name = xml.Name{
Space: "",
Local: key,
}
tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name})
func (s *Metadata) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if s == nil {
return nil
}
tokens = append(tokens, xml.EndElement{
Name: start.Name,
})
if len(s.Items) == 0 {
return nil
}
for _, t := range tokens {
if err := e.EncodeToken(t); err != nil {
if err := e.EncodeToken(start); err != nil {
return err
}
for _, item := range s.Items {
if err := e.Encode(xmlKeyEntry{
XMLName: xml.Name{Local: item.Key},
Value: item.Value,
}); err != nil {
return err
}
}
// flush to ensure tokens are written
return e.Flush()
return e.EncodeToken(start.End())
}
// Object container for object metadata
@@ -307,7 +359,7 @@ type Object struct {
StorageClass string
// UserMetadata user-defined metadata
UserMetadata StringMap `xml:"UserMetadata,omitempty"`
UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
}
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -350,6 +402,11 @@ type CompleteMultipartUploadResponse struct {
Bucket string
Key string
ETag string
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
}
// DeleteError structure.
@@ -438,6 +495,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
// generates an ListBucketVersions response for the said bucket with other enumerated options.
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
versions := make([]ObjectVersion, 0, len(resp.Objects))
owner := Owner{
ID: globalMinioDefaultOwnerID,
DisplayName: "minio",
@@ -445,10 +503,10 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
data := ListVersionsResponse{}
for _, object := range resp.Objects {
content := ObjectVersion{}
if object.Name == "" {
continue
}
content := ObjectVersion{}
content.Key = s3EncodeName(object.Name, encodingType)
content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
if object.ETag != "" {
@@ -569,16 +627,16 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
}
content.Owner = owner
if metadata {
content.UserMetadata = make(StringMap)
content.UserMetadata = &Metadata{}
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
case crypto.S3:
content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
case crypto.S3KMS:
content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
case crypto.SSEC:
content.UserMetadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
}
for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
// Do not need to send any internal metadata
// values to client.
@@ -588,7 +646,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
continue
}
content.UserMetadata[k] = v
content.UserMetadata.Set(k, v)
}
}
contents = append(contents, content)
@@ -642,14 +700,20 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
}
// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
return CompleteMultipartUploadResponse{
func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
cs := oi.decryptChecksums()
c := CompleteMultipartUploadResponse{
Location: location,
Bucket: bucket,
Key: key,
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
ETag: "\"" + etag + "\"",
ETag: "\"" + oi.ETag + "\"",
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
}
return c
}
// generates ListPartsResponse from ListPartsInfo.
@@ -674,6 +738,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
listPartsResponse.IsTruncated = partsInfo.IsTruncated
listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
for index, part := range partsInfo.Parts {
@@ -682,6 +747,10 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
newPart.ETag = "\"" + part.ETag + "\""
newPart.Size = part.Size
newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
newPart.ChecksumCRC32 = part.ChecksumCRC32
newPart.ChecksumCRC32C = part.ChecksumCRC32C
newPart.ChecksumSHA1 = part.ChecksumSHA1
newPart.ChecksumSHA256 = part.ChecksumSHA256
listPartsResponse.Parts[index] = newPart
}
return listPartsResponse
+2 -2
View File
@@ -35,8 +35,8 @@ func shouldEscape(c byte) bool {
// s3URLEncode is based on Golang's url.QueryEscape() code,
// while considering some S3 exceptions:
// - Avoid encoding '/' and '*'
// - Force encoding of '~'
// - Avoid encoding '/' and '*'
// - Force encoding of '~'
func s3URLEncode(s string) string {
spaceCount, hexCount := 0, 0
for i := 0; i < len(s); i++ {
+201 -190
View File
File diff suppressed because one or more lines are too long
+96 -52
View File
@@ -25,7 +25,6 @@ import (
"encoding/hex"
"errors"
"io"
"io/ioutil"
"net/http"
"net/url"
"strconv"
@@ -85,6 +84,8 @@ func isRequestSignStreamingV4(r *http.Request) bool {
}
// Authorization type.
//
//go:generate stringer -type=authType -trimprefix=authType $GOFILE
type authType int
// List of all supported auth types.
@@ -288,28 +289,33 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
}
// Check request auth type verifies the incoming http request
// - validates the request signature
// - validates the policy action if anonymous tests bucket policies if any,
// for authenticated requests validates IAM policies.
// - validates the request signature
// - validates the policy action if anonymous tests bucket policies if any,
// for authenticated requests validates IAM policies.
//
// returns APIErrorCode if any to be replied to the client.
func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action, bucketName, objectName)
logger.GetReqInfo(ctx).BucketName = bucketName
logger.GetReqInfo(ctx).ObjectName = objectName
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
return s3Err
}
// Check request auth type verifies the incoming http request
// - validates the request signature
// - validates the policy action if anonymous tests bucket policies if any,
// for authenticated requests validates IAM policies.
// returns APIErrorCode if any to be replied to the client.
// Additionally returns the accessKey used in the request, and if this request is by an admin.
func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
if logger.GetReqInfo(ctx) == nil {
logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
return ErrAccessDenied
}
var cred auth.Credentials
var owner bool
switch getRequestAuthType(r) {
case authTypeUnknown, authTypeStreamingSigned:
return cred, owner, ErrSignatureVersionNotSupported
return ErrSignatureVersionNotSupported
case authTypePresignedV2, authTypeSignedV2:
if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
return cred, owner, s3Err
return s3Err
}
cred, owner, s3Err = getReqAccessKeyV2(r)
case authTypeSigned, authTypePresigned:
@@ -319,52 +325,67 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
region = ""
}
if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
return cred, owner, s3Err
return s3Err
}
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
}
if s3Err != ErrNone {
return cred, owner, s3Err
return s3Err
}
// LocationConstraint is valid only for CreateBucketAction.
var locationConstraint string
logger.GetReqInfo(ctx).Cred = cred
logger.GetReqInfo(ctx).Owner = owner
// region is valid only for CreateBucketAction.
var region string
if action == policy.CreateBucketAction {
// To extract region from XML in request body, get copy of request body.
payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
if err != nil {
logger.LogIf(ctx, err, logger.Application)
return cred, owner, ErrMalformedXML
return ErrMalformedXML
}
// Populate payload to extract location constraint.
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
var s3Error APIErrorCode
locationConstraint, s3Error = parseLocationConstraint(r)
if s3Error != ErrNone {
return cred, owner, s3Error
r.Body = io.NopCloser(bytes.NewReader(payload))
region, s3Err = parseLocationConstraint(r)
if s3Err != ErrNone {
return s3Err
}
// Populate payload again to handle it in HTTP handler.
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
}
if cred.AccessKey != "" {
logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
r.Body = io.NopCloser(bytes.NewReader(payload))
}
logger.GetReqInfo(ctx).Region = region
return s3Err
}
func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return ErrAccessDenied
}
cred := reqInfo.Cred
owner := reqInfo.Owner
region := reqInfo.Region
bucket := reqInfo.BucketName
object := reqInfo.ObjectName
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
// Anonymous checks are not meant for ListBuckets action
// Anonymous checks are not meant for ListAllBuckets action
if globalPolicySys.IsAllowed(policy.Args{
AccountName: cred.AccessKey,
Action: action,
BucketName: bucketName,
ConditionValues: getConditionValues(r, locationConstraint, "", nil),
BucketName: bucket,
ConditionValues: getConditionValues(r, region, "", nil),
IsOwner: false,
ObjectName: objectName,
ObjectName: object,
}) {
// Request is allowed return the appropriate access key.
return cred, owner, ErrNone
return ErrNone
}
if action == policy.ListBucketVersionsAction {
@@ -373,31 +394,31 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
if globalPolicySys.IsAllowed(policy.Args{
AccountName: cred.AccessKey,
Action: policy.ListBucketAction,
BucketName: bucketName,
ConditionValues: getConditionValues(r, locationConstraint, "", nil),
BucketName: bucket,
ConditionValues: getConditionValues(r, region, "", nil),
IsOwner: false,
ObjectName: objectName,
ObjectName: object,
}) {
// Request is allowed return the appropriate access key.
return cred, owner, ErrNone
return ErrNone
}
}
return cred, owner, ErrAccessDenied
return ErrAccessDenied
}
if globalIAMSys.IsAllowed(iampolicy.Args{
AccountName: cred.AccessKey,
Groups: cred.Groups,
Action: iampolicy.Action(action),
BucketName: bucketName,
BucketName: bucket,
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
ObjectName: objectName,
ObjectName: object,
IsOwner: owner,
Claims: cred.Claims,
}) {
// Request is allowed return the appropriate access key.
return cred, owner, ErrNone
return ErrNone
}
if action == policy.ListBucketVersionsAction {
@@ -407,18 +428,41 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
AccountName: cred.AccessKey,
Groups: cred.Groups,
Action: iampolicy.ListBucketAction,
BucketName: bucketName,
BucketName: bucket,
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
ObjectName: objectName,
ObjectName: object,
IsOwner: owner,
Claims: cred.Claims,
}) {
// Request is allowed return the appropriate access key.
return cred, owner, ErrNone
return ErrNone
}
}
return cred, owner, ErrAccessDenied
return ErrAccessDenied
}
// Check request auth type verifies the incoming http request
// - validates the request signature
// - validates the policy action if anonymous tests bucket policies if any,
// for authenticated requests validates IAM policies.
//
// returns APIErrorCode if any to be replied to the client.
// Additionally returns the accessKey used in the request, and if this request is by an admin.
func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {
s3Err = authenticateRequest(ctx, r, action)
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return cred, owner, ErrAccessDenied
}
cred = reqInfo.Cred
owner = reqInfo.Owner
if s3Err != ErrNone {
return cred, owner, s3Err
}
return cred, owner, authorizeRequest(ctx, r, action)
}
// Verify if request has valid AWS Signature Version '2'.
@@ -582,7 +626,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
conditions["object-lock-mode"] = []string{string(retMode)}
conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
if retDays > 0 {
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
}
@@ -622,22 +666,22 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
var cred auth.Credentials
var owner bool
region := globalSite.Region
switch atype {
case authTypeUnknown:
return ErrSignatureVersionNotSupported
case authTypeSignedV2, authTypePresignedV2:
cred, owner, s3Err = getReqAccessKeyV2(r)
case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
region := globalSite.Region
cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
}
if s3Err != ErrNone {
return s3Err
}
if cred.AccessKey != "" {
logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
}
logger.GetReqInfo(ctx).Cred = cred
logger.GetReqInfo(ctx).Owner = owner
logger.GetReqInfo(ctx).Region = region
// Do not check for PutObjectRetentionAction permission,
// if mode and retain until date are not set.
+13 -13
View File
@@ -21,7 +21,6 @@ import (
"bytes"
"context"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
@@ -44,7 +43,7 @@ func TestGetRequestAuthType(t *testing.T) {
req *http.Request
authT authType
}
nopCloser := ioutil.NopCloser(io.LimitReader(&nullReader{}, 1024))
nopCloser := io.NopCloser(io.LimitReader(&nullReader{}, 1024))
testCases := []testCase{
// Test case - 1
// Check for generic signature v4 header.
@@ -360,7 +359,10 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
// Tests is requested authenticated function, tests replies for s3 errors.
func TestIsReqAuthenticated(t *testing.T) {
objLayer, fsDir, err := prepareFS()
ctx, cancel := context.WithCancel(GlobalContext)
defer cancel()
objLayer, fsDir, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
@@ -369,10 +371,7 @@ func TestIsReqAuthenticated(t *testing.T) {
t.Fatalf("unable initialize config file, %s", err)
}
initAllSubsystems()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
initAllSubsystems(ctx)
initConfigSubsystem(ctx, objLayer)
@@ -406,7 +405,7 @@ func TestIsReqAuthenticated(t *testing.T) {
for i, testCase := range testCases {
s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region, serviceS3)
if s3Error != testCase.s3Error {
if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
if _, err := io.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
}
}
@@ -414,7 +413,10 @@ func TestIsReqAuthenticated(t *testing.T) {
}
func TestCheckAdminRequestAuthType(t *testing.T) {
objLayer, fsDir, err := prepareFS()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
objLayer, fsDir, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
@@ -440,7 +442,6 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
{Request: mustNewPresignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
}
ctx := context.Background()
for i, testCase := range testCases {
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
@@ -452,7 +453,7 @@ func TestValidateAdminSignature(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
objLayer, fsDir, err := prepareFS()
objLayer, fsDir, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
@@ -462,8 +463,7 @@ func TestValidateAdminSignature(t *testing.T) {
t.Fatalf("unable initialize config file, %s", err)
}
initAllSubsystems()
initAllSubsystems(ctx)
initConfigSubsystem(ctx, objLayer)
globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+32
View File
@@ -0,0 +1,32 @@
// Code generated by "stringer -type=authType -trimprefix=authType auth-handler.go"; DO NOT EDIT.
package cmd
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[authTypeUnknown-0]
_ = x[authTypeAnonymous-1]
_ = x[authTypePresigned-2]
_ = x[authTypePresignedV2-3]
_ = x[authTypePostPolicy-4]
_ = x[authTypeStreamingSigned-5]
_ = x[authTypeSigned-6]
_ = x[authTypeSignedV2-7]
_ = x[authTypeJWT-8]
_ = x[authTypeSTS-9]
}
const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTS"
var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81}
func (i authType) String() string {
if i < 0 || i >= authType(len(_authType_index)-1) {
return "authType(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _authType_name[_authType_index[i]:_authType_index[i+1]]
}
+5 -5
View File
@@ -22,13 +22,13 @@ import (
"runtime"
"github.com/minio/madmin-go"
"github.com/minio/minio/internal/pubsub"
)
// healTask represents what to heal along with options
// path: '/' => Heal disk formats along with metadata
// path: 'bucket/' or '/bucket/' => Heal bucket
// path: 'bucket/object' => Heal object
//
// path: '/' => Heal disk formats along with metadata
// path: 'bucket/' or '/bucket/' => Heal bucket
// path: 'bucket/object' => Heal object
type healTask struct {
bucket string
object string
@@ -53,7 +53,7 @@ type healRoutine struct {
func activeListeners() int {
// Bucket notification and http trace are not costly, it is okay to ignore them
// while counting the number of concurrent connections
return int(globalHTTPListen.NumSubscribers(pubsub.MaskAll)) + int(globalTrace.NumSubscribers(pubsub.MaskAll))
return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
}
func waitForLowHTTPReq() {
+9 -10
View File
@@ -87,7 +87,7 @@ type healingTracker struct {
// The disk ID will be validated against the loaded one.
func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, error) {
if disk == nil {
return nil, errors.New("loadHealingTracker: nil disk given")
return nil, errors.New("loadHealingTracker: nil drive given")
}
diskID, err := disk.GetDiskID()
if err != nil {
@@ -104,7 +104,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
return nil, err
}
if h.ID != diskID && h.ID != "" {
return nil, fmt.Errorf("loadHealingTracker: disk id mismatch expected %s, got %s", h.ID, diskID)
return nil, fmt.Errorf("loadHealingTracker: drive id mismatch expected %s, got %s", h.ID, diskID)
}
h.disk = disk
h.ID = diskID
@@ -129,7 +129,7 @@ func newHealingTracker(disk StorageAPI) *healingTracker {
// If the tracker has been deleted an error is returned.
func (h *healingTracker) update(ctx context.Context) error {
if h.disk.Healing() == nil {
return fmt.Errorf("healingTracker: disk %q is not marked as healing", h.ID)
return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
}
if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
h.ID, _ = h.disk.GetDiskID()
@@ -310,7 +310,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
}
// Prevent parallel erasure set healing
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-disk-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
if err != nil {
return err
@@ -318,8 +318,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
ctx = lkctx.Context()
defer locker.Unlock(lkctx.Cancel)
buckets, _ := z.ListBuckets(ctx)
buckets, _ := z.ListBuckets(ctx, BucketOptions{})
// Buckets data are dispersed in multiple zones/sets, make
// sure to heal all bucket metadata configuration.
buckets = append(buckets, BucketInfo{
@@ -338,14 +337,14 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
})
if serverDebugLog {
logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
}
// Load healing tracker in this disk
tracker, err := loadHealingTracker(ctx, disk)
if err != nil {
// So someone changed the drives underneath, healing tracker missing.
logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
disk, humanize.Ordinal(poolIdx+1), err))
tracker = newHealingTracker(disk)
}
@@ -370,9 +369,9 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
}
if tracker.ItemsFailed > 0 {
logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
} else {
logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
}
if serverDebugLog {
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+23
View File
@@ -0,0 +1,23 @@
// Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
package cmd
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[batchReplicationMetricObject-0]
}
const _batchReplicationMetric_name = "Object"
var _batchReplicationMetric_index = [...]uint8{0, 6}
func (i batchReplicationMetric) String() string {
if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
}
+7 -7
View File
@@ -25,7 +25,7 @@ import (
"strconv"
"testing"
humanize "github.com/dustin/go-humanize"
"github.com/dustin/go-humanize"
)
// Benchmark utility functions for ObjectLayer.PutObject().
@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
if err != nil {
b.Fatal(err)
}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
object := getRandomObjectName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
if err != nil {
b.Fatal(err)
}
@@ -85,12 +85,12 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
// PutObjectPart returns etag of the object inserted.
// etag variable is assigned with that value.
var etag, uploadID string
var etag string
// get text data generated for number of bytes equal to object size.
textData := generateBytesData(objSize)
// generate md5sum for the generated data.
// md5sum of the data to written is required as input for NewMultipartUpload.
uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
res, err := obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
if err != nil {
b.Fatal(err)
}
@@ -113,7 +113,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
}
md5hex := getMD5Hash(textPartData)
var partInfo PartInfo
partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, res.UploadID, j,
mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
if err != nil {
b.Fatal(err)
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
if err != nil {
b.Fatal(err)
}
+18 -5
View File
@@ -24,6 +24,7 @@ import (
"fmt"
"hash"
"io"
"strings"
"sync"
xhttp "github.com/minio/minio/internal/http"
@@ -146,16 +147,28 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
// Can never happen unless there are programmer bugs
return 0, errUnexpected
}
ignoredErrs := []error{
errDiskNotFound,
}
if strings.HasPrefix(b.volume, minioMetaBucket) {
ignoredErrs = append(ignoredErrs,
errFileNotFound,
errVolumeNotFound,
errFileVersionNotFound,
)
}
if b.rc == nil {
// For the first ReadAt() call we need to open the stream for reading.
b.currOffset = offset
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
if len(b.data) == 0 && b.tillOffset != streamOffset {
b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
if err != nil && err != errDiskNotFound {
logger.LogIf(GlobalContext,
fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
b.disk, b.volume, b.filePath, err))
if err != nil {
if !IsErr(err, ignoredErrs...) {
logger.LogIf(GlobalContext,
fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
b.disk, b.volume, b.filePath, err))
}
}
} else {
b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
@@ -180,7 +193,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
b.h.Write(buf)
if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s - content hash does not match - expected %s, got %s",
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s - content hash does not match - expected %s, got %s",
b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
return 0, errFileCorrupt
}
+4 -4
View File
@@ -38,12 +38,12 @@ type wholeBitrotWriter struct {
func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
if err != nil {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
return 0, err
}
_, err = b.Hash.Write(p)
if err != nil {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
return 0, err
}
return len(p), nil
@@ -72,12 +72,12 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
if b.buf == nil {
b.buf = make([]byte, b.tillOffset-offset)
if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
return 0, err
}
}
if len(b.buf) < len(buf) {
logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
return 0, errLessData
}
n = copy(buf, b.buf)
+7 -8
View File
@@ -20,17 +20,11 @@ package cmd
import (
"context"
"io"
"io/ioutil"
"os"
"testing"
)
func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
tmpDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
tmpDir := t.TempDir()
volume := "testvol"
filePath := "testfile"
@@ -60,7 +54,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
if err != nil {
t.Fatal(err)
}
writer.(io.Closer).Close()
if bw, ok := writer.(io.Closer); ok {
bw.Close()
}
reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
b := make([]byte, 10)
@@ -76,6 +72,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
if _, err = reader.ReadAt(b[:5], 30); err != nil {
t.Fatal(err)
}
if br, ok := reader.(io.Closer); ok {
br.Close()
}
}
func TestAllBitrotAlgorithms(t *testing.T) {
+4 -4
View File
@@ -65,7 +65,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
}
// Check if bucket exists.
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -155,7 +155,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
// Check if bucket exists
var err error
if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -198,13 +198,13 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
// Check if bucket exists
var err error
if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
// Delete bucket encryption config from object layer
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
-9
View File
@@ -34,15 +34,6 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
// Get - gets bucket encryption config for the given bucket.
func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) {
if globalIsGateway {
objAPI := newObjectLayerFn()
if objAPI == nil {
return nil, errServerNotInitialized
}
return nil, BucketSSEConfigNotFound{Bucket: bucket}
}
sseCfg, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
return sseCfg, err
}
+55 -26
View File
@@ -67,11 +67,16 @@ const (
// - Check if a bucket has an entry in etcd backend
// -- If no, make an entry
// -- If yes, check if the entry matches local IP check if we
// need to update the entry then proceed to update
//
// need to update the entry then proceed to update
//
// -- If yes, check if the IP of entry matches local IP.
// This means entry is for this instance.
//
// This means entry is for this instance.
//
// -- If IP of the entry doesn't match, this means entry is
// for another instance. Log an error to console.
//
// for another instance. Log an error to console.
func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
if len(buckets) == 0 {
return
@@ -201,7 +206,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
getBucketInfo := objectAPI.GetBucketInfo
if _, err := getBucketInfo(ctx, bucket); err != nil {
if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -227,7 +232,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
// using the Initiate Multipart Upload request, but has not yet been
// completed or aborted. This operation returns at most 1,000 multipart
// uploads in the response.
//
func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ListMultipartUploads")
@@ -296,7 +300,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
listBuckets := objectAPI.ListBuckets
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction, "", "")
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction)
if s3Error != ErrNone && s3Error != ErrAccessDenied {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
@@ -332,7 +336,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
} else {
// Invoke the list buckets.
var err error
bucketsInfo, err = listBuckets(ctx)
bucketsInfo, err = listBuckets(ctx, BucketOptions{})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -447,7 +451,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
// Before proceeding validate if bucket exists.
_, err := objectAPI.GetBucketInfo(ctx, bucket)
_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -727,11 +731,30 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
}
}
if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction)
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
if objectLockEnabled {
// Creating a bucket with locking requires the user having more permissions
for _, action := range []iampolicy.Action{iampolicy.PutBucketObjectLockConfigurationAction, iampolicy.PutBucketVersioningAction} {
if !globalIAMSys.IsAllowed(iampolicy.Args{
AccountName: cred.AccessKey,
Groups: cred.Groups,
Action: action,
ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
BucketName: bucket,
IsOwner: owner,
Claims: cred.Claims,
}) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
return
}
}
}
// Parse incoming location constraint.
location, s3Error := parseLocationConstraint(r)
if s3Error != ErrNone {
@@ -746,7 +769,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
return
}
opts := BucketOptions{
// check if client is attempting to create more buckets than allowed maximum.
if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
apiErr := errorCodes.ToAPIErr(ErrTooManyBuckets)
apiErr.Description = fmt.Sprintf("You have attempted to create %d buckets than allowed %d", currBuckets+1, maxBuckets)
writeErrorResponse(ctx, w, apiErr, r.URL)
return
}
opts := MakeBucketOptions{
Location: location,
LockEnabled: objectLockEnabled,
ForceCreate: forceCreate,
@@ -765,7 +796,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
}
if err = globalDNSConfig.Put(bucket); err != nil {
objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
Force: false,
NoRecreate: true,
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
})
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -823,10 +858,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
// Call site replication hook
if err := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
// Make sure to add Location information here only for bucket
if cp := pathClean(r.URL.Path); cp != "" {
@@ -1030,10 +1062,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
sseConfig.Apply(r.Header, sse.ApplyOptions{
AutoEncrypt: globalAutoEncryption,
Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
})
// get gateway encryption options
var opts ObjectOptions
opts, err = putOpts(ctx, r, bucket, object, metadata)
if err != nil {
@@ -1165,7 +1195,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -1228,7 +1258,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
getBucketInfo := objectAPI.GetBucketInfo
if _, err := getBucketInfo(ctx, bucket); err != nil {
if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
return
}
@@ -1305,7 +1335,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
deleteBucket := objectAPI.DeleteBucket
// Attempt to delete bucket.
if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
if err := deleteBucket(ctx, bucket, DeleteBucketOptions{
Force: forceDelete,
SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
}); err != nil {
apiErr := toAPIError(ctx, err)
if _, ok := err.(BucketNotEmpty); ok {
if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1360,10 +1393,6 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return
}
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
@@ -1472,7 +1501,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -1580,7 +1609,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
return
}
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"bytes"
"encoding/xml"
"fmt"
"io/ioutil"
"io"
"net/http"
"net/http/httptest"
"strconv"
@@ -885,7 +885,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
}
// read the response body.
actualContent, err = ioutil.ReadAll(rec.Body)
actualContent, err = io.ReadAll(rec.Body)
if err != nil {
t.Fatalf("Test %d : MinIO %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
}
+5 -5
View File
@@ -62,12 +62,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
}
// Check if bucket exists.
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(io.LimitReader(r.Body, r.ContentLength))
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
}
// Check if bucket exists.
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -163,12 +163,12 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
}
// Check if bucket exists.
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
+56 -34
View File
@@ -31,6 +31,7 @@ import (
"github.com/google/uuid"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/amztime"
sse "github.com/minio/minio/internal/bucket/encryption"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/event"
@@ -57,15 +58,6 @@ type LifecycleSys struct{}
// Get - gets lifecycle config associated to a given bucket name.
func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
if globalIsGateway {
objAPI := newObjectLayerFn()
if objAPI == nil {
return nil, errServerNotInitialized
}
return nil, BucketLifecycleNotFound{Bucket: bucketName}
}
return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
}
@@ -154,9 +146,14 @@ type newerNoncurrentTask struct {
versions []ObjectToDelete
}
type transitionTask struct {
tier string
objInfo ObjectInfo
}
type transitionState struct {
once sync.Once
transitionCh chan ObjectInfo
transitionCh chan transitionTask
ctx context.Context
objAPI ObjectLayer
@@ -170,13 +167,13 @@ type transitionState struct {
lastDayStats map[string]*lastDayTierStats
}
func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
select {
case <-GlobalContext.Done():
t.once.Do(func() {
close(t.transitionCh)
})
case t.transitionCh <- oi:
case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
default:
}
}
@@ -185,7 +182,7 @@ var globalTransitionState *transitionState
func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
return &transitionState{
transitionCh: make(chan ObjectInfo, 10000),
transitionCh: make(chan transitionTask, 10000),
ctx: ctx,
objAPI: objAPI,
killCh: make(chan struct{}),
@@ -212,27 +209,25 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
return
case <-ctx.Done():
return
case oi, ok := <-t.transitionCh:
case task, ok := <-t.transitionCh:
if !ok {
return
}
atomic.AddInt32(&t.activeTasks, 1)
var tier string
var err error
if tier, err = transitionObject(ctx, objectAPI, oi); err != nil {
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
} else {
ts := tierStats{
TotalSize: uint64(oi.Size),
TotalSize: uint64(task.objInfo.Size),
NumVersions: 1,
}
if oi.IsLatest {
if task.objInfo.IsLatest {
ts.NumObjects = 1
}
t.addLastDayStats(tier, ts)
t.addLastDayStats(task.tier, ts)
}
atomic.AddInt32(&t.activeTasks, -1)
}
}
}
@@ -303,9 +298,10 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
// This is to be called after a successful upload of an object (version).
func enqueueTransitionImmediate(obj ObjectInfo) {
if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
switch lc.ComputeAction(obj.ToLifecycleOpts()) {
event := lc.Eval(obj.ToLifecycleOpts(), time.Now())
switch event.Action {
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
globalTransitionState.queueTransitionTask(obj)
globalTransitionState.queueTransitionTask(obj, event.StorageClass)
}
}
}
@@ -402,12 +398,7 @@ func genTransitionObjName(bucket string) (string, error) {
// storage specified by the transition ARN, the metadata is left behind on source cluster and original content
// is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
// to the transition tier without decrypting or re-encrypting.
func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) (string, error) {
lc, err := globalLifecycleSys.Get(oi.Bucket)
if err != nil {
return "", err
}
tier := lc.TransitionTier(oi.ToLifecycleOpts())
func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
opts := ObjectOptions{
Transition: TransitionOptions{
Status: lifecycle.TransitionPending,
@@ -419,7 +410,37 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
MTime: oi.ModTime,
}
return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
}
type auditTierOp struct {
Tier string `json:"tier"`
TimeToResponseNS int64 `json:"timeToResponseNS"`
OutputBytes int64 `json:"tx,omitempty"`
Error string `json:"error,omitempty"`
}
func auditTierActions(ctx context.Context, tier string, bytes int64) func(err error) {
startTime := time.Now()
return func(err error) {
// Record only when audit targets configured.
if len(logger.AuditTargets()) == 0 {
return
}
op := auditTierOp{
Tier: tier,
OutputBytes: bytes,
}
if err == nil {
op.TimeToResponseNS = time.Since(startTime).Nanoseconds()
} else {
op.Error = err.Error()
}
logger.GetReqInfo(ctx).AppendTags("tierStats", op)
}
}
// getTransitionedObjectReader returns a reader from the transitioned tier.
@@ -441,12 +462,13 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
gopts.length = length
}
timeTierAction := auditTierActions(ctx, oi.TransitionedObject.Tier, length)
reader, err := tgtClient.Get(ctx, oi.TransitionedObject.Name, remoteVersionID(oi.TransitionedObject.VersionID), gopts)
if err != nil {
return nil, err
}
closer := func() {
reader.Close()
timeTierAction(reader.Close())
}
return fn(reader, h, closer)
}
@@ -560,7 +582,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
}
// Check if bucket exists.
if !r.OutputLocation.IsEmpty() {
if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName, BucketOptions{}); err != nil {
return err
}
if r.OutputLocation.S3.Prefix == "" {
@@ -755,7 +777,7 @@ func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) {
if strings.TrimSpace(expiryTokens[0]) != "expiry-date" {
return restoreObjStatus{}, errRestoreHDRMalformed
}
expiry, err := time.Parse(http.TimeFormat, strings.Trim(expiryTokens[1], `"`))
expiry, err := amztime.ParseHeader(strings.Trim(expiryTokens[1], `"`))
if err != nil {
return restoreObjStatus{}, errRestoreHDRMalformed
}
+3 -4
View File
@@ -32,9 +32,9 @@ import (
// Validate all the ListObjects query arguments, returns an APIErrorCode
// if one of the args do not meet the required conditions.
// Special conditions required by MinIO server are as below
// - delimiter if set should be equal to '/', otherwise the request is rejected.
// - marker if set should have a common prefix with 'prefix' param, otherwise
// the request is rejected.
// - delimiter if set should be equal to '/', otherwise the request is rejected.
// - marker if set should have a common prefix with 'prefix' param, otherwise
// the request is rejected.
func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
// Max keys cannot be negative.
if maxKeys < 0 {
@@ -298,7 +298,6 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
// This implementation of the GET operation returns some or all (up to 1000)
// of the objects in a bucket. You can use the request parameters as selection
// criteria to return a subset of the objects in a bucket.
//
func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ListObjectsV1")
+24 -76
View File
@@ -18,7 +18,6 @@
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
@@ -45,11 +44,16 @@ type BucketMetadataSys struct {
metadataMap map[string]BucketMetadata
}
// Count returns number of bucket metadata map entries.
func (sys *BucketMetadataSys) Count() int {
sys.RLock()
defer sys.RUnlock()
return len(sys.metadataMap)
}
// Remove bucket metadata from memory.
func (sys *BucketMetadataSys) Remove(bucket string) {
if globalIsGateway {
return
}
sys.Lock()
delete(sys.metadataMap, bucket)
globalBucketMonitor.DeleteBucket(bucket)
@@ -62,10 +66,6 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
// so they should be replaced atomically and not appended to, etc.
// Data is not persisted to disk.
func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
if globalIsGateway {
return
}
if bucket != minioMetaBucket {
sys.Lock()
sys.metadataMap[bucket] = meta
@@ -73,33 +73,17 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
}
}
// Update update bucket metadata for the specified config file.
// The configData data should not be modified after being sent here.
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
objAPI := newObjectLayerFn()
if objAPI == nil {
return updatedAt, errServerNotInitialized
}
if globalIsGateway && globalGatewayName != NASBackendGateway {
if configFile == bucketPolicyConfig {
if configData == nil {
return updatedAt, objAPI.DeleteBucketPolicy(ctx, bucket)
}
config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
if err != nil {
return updatedAt, err
}
return updatedAt, objAPI.SetBucketPolicy(ctx, bucket, config)
}
return updatedAt, NotImplemented{}
}
if bucket == minioMetaBucket {
return updatedAt, errInvalidArgument
}
meta, err := loadBucketMetadata(ctx, objAPI, bucket)
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
if err != nil {
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
// Only single drive mode needs this fallback.
@@ -157,6 +141,18 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
return updatedAt, nil
}
// Delete delete the bucket metadata for the specified bucket.
// must be used by all callers instead of using Update() with nil configData.
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
}
// Update update bucket metadata for the specified bucket.
// The configData data should not be modified after being sent here.
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
return sys.updateAndParse(ctx, bucket, configFile, configData, true)
}
// Get metadata for a bucket.
// If no metadata exists errConfigNotFound is returned and a new metadata is returned.
// Only a shallow copy is returned, so referenced data should not be modified,
@@ -168,7 +164,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
// For all other bucket specific metadata, use the relevant
// calls implemented specifically for each of those features.
func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
if globalIsGateway || bucket == minioMetaBucket {
if bucket == minioMetaBucket {
return newBucketMetadata(bucket), errConfigNotFound
}
@@ -231,22 +227,6 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
// GetLifecycleConfig returns configured lifecycle config
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
if globalIsGateway && globalGatewayName == NASBackendGateway {
// Only needed in case of NAS gateway.
objAPI := newObjectLayerFn()
if objAPI == nil {
return nil, errServerNotInitialized
}
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
if err != nil {
return nil, err
}
if meta.lifecycleConfig == nil {
return nil, BucketLifecycleNotFound{Bucket: bucket}
}
return meta.lifecycleConfig, nil
}
meta, err := sys.GetConfig(GlobalContext, bucket)
if err != nil {
if errors.Is(err, errConfigNotFound) {
@@ -263,19 +243,6 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
// GetNotificationConfig returns configured notification config
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
if globalIsGateway && globalGatewayName == NASBackendGateway {
// Only needed in case of NAS gateway.
objAPI := newObjectLayerFn()
if objAPI == nil {
return nil, errServerNotInitialized
}
meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
if err != nil {
return nil, err
}
return meta.notificationConfig, nil
}
meta, err := sys.GetConfig(GlobalContext, bucket)
if err != nil {
return nil, err
@@ -311,15 +278,6 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
// GetPolicyConfig returns configured bucket policy
// The returned object may not be modified.
func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
if globalIsGateway {
objAPI := newObjectLayerFn()
if objAPI == nil {
return nil, time.Time{}, errServerNotInitialized
}
p, err := objAPI.GetBucketPolicy(GlobalContext, bucket)
return p, UTCNow(), err
}
meta, err := sys.GetConfig(GlobalContext, bucket)
if err != nil {
if errors.Is(err, errConfigNotFound) {
@@ -387,10 +345,6 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (Buc
return newBucketMetadata(bucket), errServerNotInitialized
}
if globalIsGateway {
return newBucketMetadata(bucket), NotImplemented{}
}
if bucket == minioMetaBucket {
return newBucketMetadata(bucket), errInvalidArgument
}
@@ -418,12 +372,6 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
return errServerNotInitialized
}
// In gateway mode, we don't need to load bucket metadata except
// NAS gateway backend.
if globalIsGateway && !objAPI.IsNotificationSupported() {
return nil
}
// Load bucket metadata sys in background
go sys.load(ctx, buckets, objAPI)
return nil
@@ -452,7 +400,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
sys.metadataMap[buckets[index].Name] = meta
sys.Unlock()
globalNotificationSys.set(buckets[index], meta) // set notification targets
globalEventNotifier.set(buckets[index], meta) // set notification targets
globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
+45 -33
View File
@@ -105,10 +105,8 @@ type BucketMetadata struct {
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
func newBucketMetadata(name string) BucketMetadata {
now := UTCNow()
return BucketMetadata{
Name: name,
Created: now,
Name: name,
notificationConfig: &event.Config{
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
},
@@ -121,6 +119,17 @@ func newBucketMetadata(name string) BucketMetadata {
}
}
// SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
// creation time of bucket on this cluster in all other cases.
func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
if b.Created.IsZero() {
b.Created = UTCNow()
}
if !createdAt.IsZero() {
b.Created = createdAt.UTC()
}
}
// Load - loads the metadata of bucket by name from ObjectLayer api.
// If an error is returned the returned metadata will be default initialized.
func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
@@ -153,27 +162,47 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
return err
}
// loadBucketMetadata loads and migrates to bucket metadata.
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
b := newBucketMetadata(bucket)
err := b.Load(ctx, objectAPI, b.Name)
if err != nil && !errors.Is(err, errConfigNotFound) {
return b, err
}
if err == nil {
b.defaultTimestamps()
}
// Old bucket without bucket metadata. Hence we migrate existing settings.
if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
if err != nil {
return b, err
}
if len(configs) == 0 {
if parse {
// nothing to update, parse and proceed.
err = b.parseAllConfigs(ctx, objectAPI)
}
} else {
// Old bucket without bucket metadata. Hence we migrate existing settings.
err = b.convertLegacyConfigs(ctx, objectAPI, configs)
}
if err != nil {
return b, err
}
// migrate unencrypted remote targets
if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
return b, err
}
b.defaultTimestamps()
return b, nil
}
// loadBucketMetadata loads and migrates to bucket metadata.
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
}
// parseAllConfigs will parse all configs and populate the private fields.
// The first error encountered is returned.
func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
@@ -266,7 +295,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
return nil
}
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) (map[string][]byte, error) {
legacyConfigs := []string{
legacyBucketObjectLockEnabledConfigFile,
bucketPolicyConfig,
@@ -280,7 +309,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
objectLockConfig,
}
configs := make(map[string][]byte)
configs := make(map[string][]byte, len(legacyConfigs))
// Handle migration from lockEnabled to newer format.
if b.LockEnabled {
@@ -305,16 +334,15 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
continue
}
return err
return nil, err
}
configs[legacyFile] = configData
}
if len(configs) == 0 {
// nothing to update, return right away.
return b.parseAllConfigs(ctx, objectAPI)
}
return configs, nil
}
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
for legacyFile, configData := range configs {
switch legacyFile {
case legacyBucketObjectLockEnabledConfigFile:
@@ -345,6 +373,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
b.BucketTargetsConfigJSON = configData
}
}
b.defaultTimestamps()
if err := b.Save(ctx, objectAPI); err != nil {
return err
@@ -413,23 +442,6 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
return saveConfig(ctx, api, configFile, data)
}
// deleteBucketMetadata deletes bucket metadata
// If config does not exist no error is returned.
func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string) error {
metadataFiles := []string{
dataUsageCacheName,
bucketMetadataFile,
path.Join(replicationDir, resyncFileName),
}
for _, metaFile := range metadataFiles {
configFile := path.Join(bucketMetaPrefix, bucket, metaFile)
if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
return err
}
}
return nil
}
// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
var err error
+5 -5
View File
@@ -60,7 +60,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
return
}
_, err := objAPI.GetBucketInfo(ctx, bucketName)
_, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -72,7 +72,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
return
}
config.SetRegion(globalSite.Region)
if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
if err = config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
arnErr, ok := err.(*event.ErrARNNotFound)
if ok {
for i, queue := range config.QueueList {
@@ -132,7 +132,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
return
}
_, err := objectAPI.GetBucketInfo(ctx, bucketName)
_, err := objectAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -144,7 +144,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
return
}
config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalEventNotifier.targetList)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
if event.IsEventError(err) {
@@ -166,7 +166,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
}
rulesMap := config.ToRulesMap()
globalNotificationSys.AddRulesMap(bucketName, rulesMap)
globalEventNotifier.AddRulesMap(bucketName, rulesMap)
writeSuccessResponseHeadersOnly(w)
}
+6 -11
View File
@@ -19,6 +19,7 @@ package cmd
import (
"context"
"errors"
"math"
"net/http"
@@ -35,22 +36,16 @@ type BucketObjectLockSys struct{}
// Get - Get retention configuration.
func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
if globalIsGateway {
objAPI := newObjectLayerFn()
if objAPI == nil {
return r, errServerNotInitialized
}
return r, nil
}
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
if err != nil {
if _, ok := err.(BucketObjectLockConfigNotFound); ok {
if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucketName}) {
return r, nil
}
if errors.Is(err, errInvalidArgument) {
return r, err
}
logger.CriticalIf(context.Background(), err)
return r, err
}
return config.ToRetention(), nil
}
+5 -6
View File
@@ -21,7 +21,6 @@ import (
"bytes"
"encoding/json"
"io"
"io/ioutil"
"net/http"
humanize "github.com/dustin/go-humanize"
@@ -61,7 +60,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
}
// Check if bucket exists.
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -79,7 +78,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
return
}
bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
bucketPolicyBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -145,12 +144,12 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
}
// Check if bucket exists.
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -191,7 +190,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
}
// Check if bucket exists.
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
+4 -5
View File
@@ -21,7 +21,6 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"reflect"
@@ -128,7 +127,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
defer wg.Done()
// Sync start.
<-start
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
if _, ok := err.(BucketExists); !ok {
t.Logf("unexpected error: %T: %v", err, err)
return
@@ -163,7 +162,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
credentials auth.Credentials, t *testing.T,
) {
bucketName1 := fmt.Sprintf("%s-1", bucketName)
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
@@ -487,7 +486,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV4.Code)
}
// read the response body.
bucketPolicyReadBuf, err := ioutil.ReadAll(recV4.Body)
bucketPolicyReadBuf, err := io.ReadAll(recV4.Body)
if err != nil {
t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
}
@@ -525,7 +524,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV2.Code)
}
// read the response body.
bucketPolicyReadBuf, err = ioutil.ReadAll(recV2.Body)
bucketPolicyReadBuf, err = io.ReadAll(recV2.Body)
if err != nil {
t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
}
+35
View File
@@ -27,6 +27,7 @@ import (
jsoniter "github.com/json-iterator/go"
miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/handlers"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
@@ -125,6 +126,20 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
cloneHeader := r.Header.Clone()
if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
tag, _ := tags.ParseObjectTags(userTags)
if tag != nil {
tagMap := tag.ToMap()
keys := make([]string, 0, len(tagMap))
for k, v := range tagMap {
args[pathJoin("ExistingObjectTag", k)] = []string{v}
args[pathJoin("RequestObjectTag", k)] = []string{v}
keys = append(keys, k)
}
args["RequestObjectTagKeys"] = keys
}
}
for _, objLock := range []string{
xhttp.AmzObjectLockMode,
xhttp.AmzObjectLockLegalHold,
@@ -137,6 +152,9 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
}
for key, values := range cloneHeader {
if strings.EqualFold(key, xhttp.AmzObjectTagging) {
continue
}
if existingValues, found := args[key]; found {
args[key] = append(existingValues, values...)
} else {
@@ -169,6 +187,8 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
}
// JWT specific values
//
// Add all string claims
for k, v := range claims {
vStr, ok := v.(string)
if ok {
@@ -183,6 +203,21 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
}
}
}
// Add groups claim which could be a list. This will ensure that the claim
// `jwt:groups` works.
if grpsVal, ok := claims["groups"]; ok {
if grpsIs, ok := grpsVal.([]interface{}); ok {
grps := []string{}
for _, gI := range grpsIs {
if g, ok := gI.(string); ok {
grps = append(grps, g)
}
}
if len(grps) > 0 {
args["groups"] = grps
}
}
}
return args
}
-7
View File
@@ -35,13 +35,6 @@ type BucketQuotaSys struct {
// Get - Get quota configuration.
func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
if globalIsGateway {
objAPI := newObjectLayerFn()
if objAPI == nil {
return nil, errServerNotInitialized
}
return &madmin.BucketQuota{}, nil
}
qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
return qCfg, err
}
+20 -15
View File
@@ -46,20 +46,19 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
return
}
if globalIsGateway {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if globalSiteReplicationSys.isEnabled() && logger.GetReqInfo(ctx).Cred.AccessKey != globalActiveCred.AccessKey {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
return
}
if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
return
@@ -118,7 +117,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
return
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -157,7 +156,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
return
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -165,7 +164,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
return
}
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketReplicationConfig); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -198,7 +197,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -217,7 +216,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
w.Header().Set(xhttp.ContentType, string(mimeJSON))
enc := json.NewEncoder(w)
if err = enc.Encode(getLatestReplicationStats(bucket, usageInfo)); err != nil {
if err = enc.Encode(globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
@@ -267,7 +266,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -277,7 +276,13 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if !config.HasExistingObjectReplication(arn) {
hasARN, hasExistingObjEnabled := config.HasExistingObjectReplication(arn)
if !hasARN {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrRemoteTargetNotFoundError), r.URL)
return
}
if !hasExistingObjEnabled {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoExistingObjects), r.URL)
return
}
@@ -310,7 +315,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
switch err.(type) {
case BucketRemoteConnectionErr:
case RemoteTargetConnectionErr:
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
default:
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -356,7 +361,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
+175 -14
View File
@@ -19,6 +19,8 @@ package cmd
import (
"context"
"encoding/binary"
"math"
"sync"
"time"
@@ -36,10 +38,12 @@ func (b *BucketReplicationStats) hasReplicationUsage() bool {
// ReplicationStats holds the global in-memory replication stats
type ReplicationStats struct {
Cache map[string]*BucketReplicationStats
UsageCache map[string]*BucketReplicationStats
sync.RWMutex
ulock sync.RWMutex
Cache map[string]*BucketReplicationStats
UsageCache map[string]*BucketReplicationStats
mostRecentStats BucketStatsMap
sync.RWMutex // mutex for Cache
ulock sync.RWMutex // mutex for UsageCache
mostRecentStatsMu sync.Mutex // mutex for mostRecentStats
}
// Delete deletes in-memory replication statistics for a bucket.
@@ -92,14 +96,23 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
bs.Stats[arn] = b
}
switch status {
case replication.Pending:
if opType.IsDataReplication() && prevStatus != status {
b.PendingSize += n
b.PendingCount++
}
case replication.Completed:
switch prevStatus { // adjust counters based on previous state
case replication.Pending:
b.PendingCount--
case replication.Failed:
b.FailedCount--
}
if opType == replication.ObjectReplicationType {
if opType.IsDataReplication() {
b.ReplicatedSize += n
switch prevStatus {
case replication.Pending:
b.PendingSize -= n
case replication.Failed:
b.FailedSize -= n
}
@@ -108,10 +121,12 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
}
}
case replication.Failed:
if opType == replication.ObjectReplicationType {
if opType.IsDataReplication() {
if prevStatus == replication.Pending {
b.FailedSize += n
b.FailedCount++
b.PendingSize -= n
b.PendingCount--
}
}
case replication.Replica:
@@ -176,9 +191,20 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
}
}
// load replication metrics at cluster start from initial data usage
// load replication metrics at cluster start from latest replication stats saved in .minio.sys/buckets/replication/node-name.stats
// fallback to replication stats in data usage to be backward compatible
func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
rTimer := time.NewTimer(time.Minute)
m := make(map[string]*BucketReplicationStats)
if stats, err := globalReplicationPool.loadStatsFromDisk(); err == nil {
for b, st := range stats {
m[b] = &st
}
r.ulock.Lock()
r.UsageCache = m
r.ulock.Unlock()
return
}
rTimer := time.NewTimer(time.Second * 5)
defer rTimer.Stop()
var (
dui DataUsageInfo
@@ -192,15 +218,12 @@ outer:
case <-rTimer.C:
dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
// If LastUpdate is set, data usage is available.
if err == nil && !dui.LastUpdate.IsZero() {
if err == nil {
break outer
}
rTimer.Reset(time.Minute)
rTimer.Reset(time.Second * 5)
}
}
m := make(map[string]*BucketReplicationStats)
for bucket, usage := range dui.BucketsUsage {
b := &BucketReplicationStats{
Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
@@ -219,6 +242,144 @@ outer:
}
}
r.ulock.Lock()
defer r.ulock.Unlock()
r.UsageCache = m
r.ulock.Unlock()
}
// serializeStats will serialize the current stats.
// Will return (nil, nil) if no data.
func (r *ReplicationStats) serializeStats() ([]byte, error) {
if r == nil {
return nil, nil
}
r.mostRecentStatsMu.Lock()
defer r.mostRecentStatsMu.Unlock()
bsm := r.mostRecentStats
if len(bsm.Stats) == 0 {
return nil, nil
}
data := make([]byte, 4, 4+bsm.Msgsize())
// Add the replication stats meta header.
binary.LittleEndian.PutUint16(data[0:2], replStatsMetaFormat)
binary.LittleEndian.PutUint16(data[2:4], replStatsVersion)
// Add data
return r.mostRecentStats.MarshalMsg(data)
}
func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
bucketsReplicationStats = make(map[string]BucketReplicationStats, len(bucketsUsage))
for bucket, u := range bucketsUsage {
bucketStats := make([]BucketStats, len(peerBucketStatsList))
for i, peerBucketStats := range peerBucketStatsList {
bucketStat, ok := peerBucketStats.Stats[bucket]
if !ok {
continue
}
bucketStats[i] = bucketStat
}
bucketsReplicationStats[bucket] = r.calculateBucketReplicationStats(bucket, u, bucketStats)
}
return bucketsReplicationStats
}
func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u BucketUsageInfo, bucketStats []BucketStats) (s BucketReplicationStats) {
if r == nil {
s = BucketReplicationStats{
Stats: make(map[string]*BucketReplicationStat),
}
return s
}
// accumulate cluster bucket stats
stats := make(map[string]*BucketReplicationStat)
var totReplicaSize int64
for _, bucketStat := range bucketStats {
totReplicaSize += bucketStat.ReplicationStats.ReplicaSize
for arn, stat := range bucketStat.ReplicationStats.Stats {
oldst := stats[arn]
if oldst == nil {
oldst = &BucketReplicationStat{}
}
stats[arn] = &BucketReplicationStat{
FailedCount: stat.FailedCount + oldst.FailedCount,
FailedSize: stat.FailedSize + oldst.FailedSize,
ReplicatedSize: stat.ReplicatedSize + oldst.ReplicatedSize,
Latency: stat.Latency.merge(oldst.Latency),
PendingCount: stat.PendingCount + oldst.PendingCount,
PendingSize: stat.PendingSize + oldst.PendingSize,
}
}
}
// add initial usage stat to cluster stats
usageStat := globalReplicationStats.GetInitialUsage(bucket)
totReplicaSize += usageStat.ReplicaSize
for arn, stat := range usageStat.Stats {
st, ok := stats[arn]
if !ok {
st = &BucketReplicationStat{}
stats[arn] = st
}
st.ReplicatedSize += stat.ReplicatedSize
st.FailedSize += stat.FailedSize
st.FailedCount += stat.FailedCount
st.PendingSize += stat.PendingSize
st.PendingCount += stat.PendingCount
}
s = BucketReplicationStats{
Stats: make(map[string]*BucketReplicationStat, len(stats)),
}
var latestTotReplicatedSize int64
for _, st := range u.ReplicationInfo {
latestTotReplicatedSize += int64(st.ReplicatedSize)
}
// normalize computed real time stats with latest usage stat
for arn, tgtstat := range stats {
st := BucketReplicationStat{}
bu, ok := u.ReplicationInfo[arn]
if !ok {
bu = BucketTargetUsageInfo{}
}
// use in memory replication stats if it is ahead of usage info.
st.ReplicatedSize = int64(bu.ReplicatedSize)
if tgtstat.ReplicatedSize >= int64(bu.ReplicatedSize) {
st.ReplicatedSize = tgtstat.ReplicatedSize
}
s.ReplicatedSize += st.ReplicatedSize
// Reset FailedSize and FailedCount to 0 for negative overflows which can
// happen since data usage picture can lag behind actual usage state at the time of cluster start
st.FailedSize = int64(math.Max(float64(tgtstat.FailedSize), 0))
st.FailedCount = int64(math.Max(float64(tgtstat.FailedCount), 0))
st.PendingSize = int64(math.Max(float64(tgtstat.PendingSize), 0))
st.PendingCount = int64(math.Max(float64(tgtstat.PendingCount), 0))
st.Latency = tgtstat.Latency
s.Stats[arn] = &st
s.FailedSize += st.FailedSize
s.FailedCount += st.FailedCount
s.PendingCount += st.PendingCount
s.PendingSize += st.PendingSize
}
// normalize overall stats
s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
r.mostRecentStatsMu.Lock()
if len(r.mostRecentStats.Stats) == 0 {
r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
}
r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
r.mostRecentStats.Timestamp = UTCNow()
r.mostRecentStatsMu.Unlock()
return s
}
// get the most current of in-memory replication stats and data usage info from crawler.
func (r *ReplicationStats) getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
return r.calculateBucketReplicationStats(bucket, u, bucketStats)
}
+41
View File
@@ -737,3 +737,44 @@ func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) {
opts.Prefix = q.Get("prefix")
return
}
const (
replicationMRFDir = bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "mrf"
mrfMetaFormat = 1
mrfMetaVersionV1 = 1
mrfMetaVersion = mrfMetaVersionV1
)
// MRFReplicateEntry mrf entry to save to disk
type MRFReplicateEntry struct {
Bucket string `json:"bucket" msg:"b"`
Object string `json:"object" msg:"o"`
versionID string `json:"-"`
}
// MRFReplicateEntries has the map of MRF entries to save to disk
type MRFReplicateEntries struct {
Entries map[string]MRFReplicateEntry `json:"entries" msg:"e"`
Version int `json:"version" msg:"v"`
}
// ToMRFEntry returns the relevant info needed by MRF
func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
return MRFReplicateEntry{
Bucket: ri.Bucket,
Object: ri.Name,
versionID: ri.VersionID,
}
}
func getReplicationStatsPath(nodeName string) string {
nodeStr := strings.ReplaceAll(nodeName, ":", "_")
return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + nodeStr + ".stats"
}
const (
replStatsMetaFormat = 1
replStatsVersionV1 = 1
replStatsVersion = replStatsVersionV1
replStatsSaveInterval = time.Minute * 5
)
+407
View File
@@ -260,6 +260,413 @@ func (z *BucketReplicationResyncStatus) Msgsize() (s int) {
return
}
// DecodeMsg implements msgp.Decodable
func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "e":
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Entries")
return
}
if z.Entries == nil {
z.Entries = make(map[string]MRFReplicateEntry, zb0002)
} else if len(z.Entries) > 0 {
for key := range z.Entries {
delete(z.Entries, key)
}
}
for zb0002 > 0 {
zb0002--
var za0001 string
var za0002 MRFReplicateEntry
za0001, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Entries")
return
}
var zb0003 uint32
zb0003, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Entries", za0001)
return
}
for zb0003 > 0 {
zb0003--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "Entries", za0001)
return
}
switch msgp.UnsafeString(field) {
case "b":
za0002.Bucket, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "Bucket")
return
}
case "o":
za0002.Object, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "Object")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err, "Entries", za0001)
return
}
}
}
z.Entries[za0001] = za0002
}
case "v":
z.Version, err = dc.ReadInt()
if err != nil {
err = msgp.WrapError(err, "Version")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "e"
err = en.Append(0x82, 0xa1, 0x65)
if err != nil {
return
}
err = en.WriteMapHeader(uint32(len(z.Entries)))
if err != nil {
err = msgp.WrapError(err, "Entries")
return
}
for za0001, za0002 := range z.Entries {
err = en.WriteString(za0001)
if err != nil {
err = msgp.WrapError(err, "Entries")
return
}
// map header, size 2
// write "b"
err = en.Append(0x82, 0xa1, 0x62)
if err != nil {
return
}
err = en.WriteString(za0002.Bucket)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "Bucket")
return
}
// write "o"
err = en.Append(0xa1, 0x6f)
if err != nil {
return
}
err = en.WriteString(za0002.Object)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "Object")
return
}
}
// write "v"
err = en.Append(0xa1, 0x76)
if err != nil {
return
}
err = en.WriteInt(z.Version)
if err != nil {
err = msgp.WrapError(err, "Version")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
// string "e"
o = append(o, 0x82, 0xa1, 0x65)
o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
for za0001, za0002 := range z.Entries {
o = msgp.AppendString(o, za0001)
// map header, size 2
// string "b"
o = append(o, 0x82, 0xa1, 0x62)
o = msgp.AppendString(o, za0002.Bucket)
// string "o"
o = append(o, 0xa1, 0x6f)
o = msgp.AppendString(o, za0002.Object)
}
// string "v"
o = append(o, 0xa1, 0x76)
o = msgp.AppendInt(o, z.Version)
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "e":
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Entries")
return
}
if z.Entries == nil {
z.Entries = make(map[string]MRFReplicateEntry, zb0002)
} else if len(z.Entries) > 0 {
for key := range z.Entries {
delete(z.Entries, key)
}
}
for zb0002 > 0 {
var za0001 string
var za0002 MRFReplicateEntry
zb0002--
za0001, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Entries")
return
}
var zb0003 uint32
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001)
return
}
for zb0003 > 0 {
zb0003--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001)
return
}
switch msgp.UnsafeString(field) {
case "b":
za0002.Bucket, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "Bucket")
return
}
case "o":
za0002.Object, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001, "Object")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err, "Entries", za0001)
return
}
}
}
z.Entries[za0001] = za0002
}
case "v":
z.Version, bts, err = msgp.ReadIntBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Version")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *MRFReplicateEntries) Msgsize() (s int) {
s = 1 + 2 + msgp.MapHeaderSize
if z.Entries != nil {
for za0001, za0002 := range z.Entries {
_ = za0002
s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object)
}
}
s += 2 + msgp.IntSize
return
}
// DecodeMsg implements msgp.Decodable
func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "b":
z.Bucket, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Bucket")
return
}
case "o":
z.Object, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Object")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
return
}
// EncodeMsg implements msgp.Encodable
func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "b"
err = en.Append(0x82, 0xa1, 0x62)
if err != nil {
return
}
err = en.WriteString(z.Bucket)
if err != nil {
err = msgp.WrapError(err, "Bucket")
return
}
// write "o"
err = en.Append(0xa1, 0x6f)
if err != nil {
return
}
err = en.WriteString(z.Object)
if err != nil {
err = msgp.WrapError(err, "Object")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// map header, size 2
// string "b"
o = append(o, 0x82, 0xa1, 0x62)
o = msgp.AppendString(o, z.Bucket)
// string "o"
o = append(o, 0xa1, 0x6f)
o = msgp.AppendString(o, z.Object)
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
switch msgp.UnsafeString(field) {
case "b":
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Bucket")
return
}
case "o":
z.Object, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Object")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z MRFReplicateEntry) Msgsize() (s int) {
s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object)
return
}
// DecodeMsg implements msgp.Decodable
func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
var field []byte
+226
View File
@@ -122,6 +122,232 @@ func BenchmarkDecodeBucketReplicationResyncStatus(b *testing.B) {
}
}
func TestMarshalUnmarshalMRFReplicateEntries(t *testing.T) {
v := MRFReplicateEntries{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgMRFReplicateEntries(b *testing.B) {
v := MRFReplicateEntries{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgMRFReplicateEntries(b *testing.B) {
v := MRFReplicateEntries{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalMRFReplicateEntries(b *testing.B) {
v := MRFReplicateEntries{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeMRFReplicateEntries(t *testing.T) {
v := MRFReplicateEntries{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeMRFReplicateEntries Msgsize() is inaccurate")
}
vn := MRFReplicateEntries{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeMRFReplicateEntries(b *testing.B) {
v := MRFReplicateEntries{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeMRFReplicateEntries(b *testing.B) {
v := MRFReplicateEntries{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func TestMarshalUnmarshalMRFReplicateEntry(t *testing.T) {
v := MRFReplicateEntry{}
bts, err := v.MarshalMsg(nil)
if err != nil {
t.Fatal(err)
}
left, err := v.UnmarshalMsg(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
}
left, err = msgp.Skip(bts)
if err != nil {
t.Fatal(err)
}
if len(left) > 0 {
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
}
}
func BenchmarkMarshalMsgMRFReplicateEntry(b *testing.B) {
v := MRFReplicateEntry{}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.MarshalMsg(nil)
}
}
func BenchmarkAppendMsgMRFReplicateEntry(b *testing.B) {
v := MRFReplicateEntry{}
bts := make([]byte, 0, v.Msgsize())
bts, _ = v.MarshalMsg(bts[0:0])
b.SetBytes(int64(len(bts)))
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
bts, _ = v.MarshalMsg(bts[0:0])
}
}
func BenchmarkUnmarshalMRFReplicateEntry(b *testing.B) {
v := MRFReplicateEntry{}
bts, _ := v.MarshalMsg(nil)
b.ReportAllocs()
b.SetBytes(int64(len(bts)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := v.UnmarshalMsg(bts)
if err != nil {
b.Fatal(err)
}
}
}
func TestEncodeDecodeMRFReplicateEntry(t *testing.T) {
v := MRFReplicateEntry{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
m := v.Msgsize()
if buf.Len() > m {
t.Log("WARNING: TestEncodeDecodeMRFReplicateEntry Msgsize() is inaccurate")
}
vn := MRFReplicateEntry{}
err := msgp.Decode(&buf, &vn)
if err != nil {
t.Error(err)
}
buf.Reset()
msgp.Encode(&buf, &v)
err = msgp.NewReader(&buf).Skip()
if err != nil {
t.Error(err)
}
}
func BenchmarkEncodeMRFReplicateEntry(b *testing.B) {
v := MRFReplicateEntry{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
en := msgp.NewWriter(msgp.Nowhere)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
v.EncodeMsg(en)
}
en.Flush()
}
func BenchmarkDecodeMRFReplicateEntry(b *testing.B) {
v := MRFReplicateEntry{}
var buf bytes.Buffer
msgp.Encode(&buf, &v)
b.SetBytes(int64(buf.Len()))
rd := msgp.NewEndlessReader(buf.Bytes(), b)
dc := msgp.NewReader(rd)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
err := v.DecodeMsg(dc)
if err != nil {
b.Fatal(err)
}
}
}
func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
v := ReplicateDecision{}
bts, err := v.MarshalMsg(nil)
+779 -208
View File
File diff suppressed because it is too large Load Diff
+14 -2
View File
@@ -18,6 +18,7 @@
package cmd
import (
"fmt"
"time"
)
@@ -52,7 +53,10 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
}
// BucketStatsMap captures bucket statistics for all buckets
type BucketStatsMap map[string]BucketStats
type BucketStatsMap struct {
Stats map[string]BucketStats
Timestamp time.Time
}
// BucketStats bucket statistics
type BucketStats struct {
@@ -62,7 +66,7 @@ type BucketStats struct {
// BucketReplicationStats represents inline replication statistics
// such as pending, failed and completed bytes in total for a bucket
type BucketReplicationStats struct {
Stats map[string]*BucketReplicationStat
Stats map[string]*BucketReplicationStat `json:",omitempty"`
// Pending size in bytes
PendingSize int64 `json:"pendingReplicationSize"`
// Completed size in bytes
@@ -126,3 +130,11 @@ func (bs *BucketReplicationStat) hasReplicationUsage() bool {
bs.PendingCount > 0 ||
bs.PendingSize > 0
}
func (brs BucketReplicationStats) String() string {
s := "ReplicatedSize=" + fmt.Sprintf("%d", brs.ReplicatedSize) + "+\n ReplicaSize=" + fmt.Sprintf("%d", brs.ReplicaSize)
for arn, st := range brs.Stats {
s += "\n arn: " + arn + " ReplicatedSize=" + fmt.Sprintf("%d", st.ReplicatedSize) + " +::ReplicaSize=" + fmt.Sprintf("%d", st.ReplicaSize)
}
return s
}
+179 -98
View File
@@ -794,74 +794,109 @@ func (z *BucketStats) Msgsize() (s int) {
// DecodeMsg implements msgp.Decodable
func (z *BucketStatsMap) DecodeMsg(dc *msgp.Reader) (err error) {
var zb0003 uint32
zb0003, err = dc.ReadMapHeader()
var field []byte
_ = field
var zb0001 uint32
zb0001, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err)
return
}
if (*z) == nil {
(*z) = make(BucketStatsMap, zb0003)
} else if len((*z)) > 0 {
for key := range *z {
delete((*z), key)
}
}
for zb0003 > 0 {
zb0003--
var zb0001 string
var zb0002 BucketStats
zb0001, err = dc.ReadString()
for zb0001 > 0 {
zb0001--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err)
return
}
var field []byte
_ = field
var zb0004 uint32
zb0004, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, zb0001)
return
}
for zb0004 > 0 {
zb0004--
field, err = dc.ReadMapKeyPtr()
switch msgp.UnsafeString(field) {
case "Stats":
var zb0002 uint32
zb0002, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, zb0001)
err = msgp.WrapError(err, "Stats")
return
}
switch msgp.UnsafeString(field) {
case "ReplicationStats":
err = zb0002.ReplicationStats.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, zb0001, "ReplicationStats")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err, zb0001)
return
if z.Stats == nil {
z.Stats = make(map[string]BucketStats, zb0002)
} else if len(z.Stats) > 0 {
for key := range z.Stats {
delete(z.Stats, key)
}
}
for zb0002 > 0 {
zb0002--
var za0001 string
var za0002 BucketStats
za0001, err = dc.ReadString()
if err != nil {
err = msgp.WrapError(err, "Stats")
return
}
var zb0003 uint32
zb0003, err = dc.ReadMapHeader()
if err != nil {
err = msgp.WrapError(err, "Stats", za0001)
return
}
for zb0003 > 0 {
zb0003--
field, err = dc.ReadMapKeyPtr()
if err != nil {
err = msgp.WrapError(err, "Stats", za0001)
return
}
switch msgp.UnsafeString(field) {
case "ReplicationStats":
err = za0002.ReplicationStats.DecodeMsg(dc)
if err != nil {
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err, "Stats", za0001)
return
}
}
}
z.Stats[za0001] = za0002
}
case "Timestamp":
z.Timestamp, err = dc.ReadTime()
if err != nil {
err = msgp.WrapError(err, "Timestamp")
return
}
default:
err = dc.Skip()
if err != nil {
err = msgp.WrapError(err)
return
}
}
(*z)[zb0001] = zb0002
}
return
}
// EncodeMsg implements msgp.Encodable
func (z BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
err = en.WriteMapHeader(uint32(len(z)))
func (z *BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
// map header, size 2
// write "Stats"
err = en.Append(0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
if err != nil {
err = msgp.WrapError(err)
return
}
for zb0005, zb0006 := range z {
err = en.WriteString(zb0005)
err = en.WriteMapHeader(uint32(len(z.Stats)))
if err != nil {
err = msgp.WrapError(err, "Stats")
return
}
for za0001, za0002 := range z.Stats {
err = en.WriteString(za0001)
if err != nil {
err = msgp.WrapError(err)
err = msgp.WrapError(err, "Stats")
return
}
// map header, size 1
@@ -870,102 +905,148 @@ func (z BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
if err != nil {
return
}
err = zb0006.ReplicationStats.EncodeMsg(en)
err = za0002.ReplicationStats.EncodeMsg(en)
if err != nil {
err = msgp.WrapError(err, zb0005, "ReplicationStats")
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
return
}
}
// write "Timestamp"
err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
if err != nil {
return
}
err = en.WriteTime(z.Timestamp)
if err != nil {
err = msgp.WrapError(err, "Timestamp")
return
}
return
}
// MarshalMsg implements msgp.Marshaler
func (z BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
func (z *BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
o = msgp.AppendMapHeader(o, uint32(len(z)))
for zb0005, zb0006 := range z {
o = msgp.AppendString(o, zb0005)
// map header, size 2
// string "Stats"
o = append(o, 0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
o = msgp.AppendMapHeader(o, uint32(len(z.Stats)))
for za0001, za0002 := range z.Stats {
o = msgp.AppendString(o, za0001)
// map header, size 1
// string "ReplicationStats"
o = append(o, 0x81, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
o, err = zb0006.ReplicationStats.MarshalMsg(o)
o, err = za0002.ReplicationStats.MarshalMsg(o)
if err != nil {
err = msgp.WrapError(err, zb0005, "ReplicationStats")
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
return
}
}
// string "Timestamp"
o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
o = msgp.AppendTime(o, z.Timestamp)
return
}
// UnmarshalMsg implements msgp.Unmarshaler
func (z *BucketStatsMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
var zb0003 uint32
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
var field []byte
_ = field
var zb0001 uint32
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
if (*z) == nil {
(*z) = make(BucketStatsMap, zb0003)
} else if len((*z)) > 0 {
for key := range *z {
delete((*z), key)
}
}
for zb0003 > 0 {
var zb0001 string
var zb0002 BucketStats
zb0003--
zb0001, bts, err = msgp.ReadStringBytes(bts)
for zb0001 > 0 {
zb0001--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
var field []byte
_ = field
var zb0004 uint32
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, zb0001)
return
}
for zb0004 > 0 {
zb0004--
field, bts, err = msgp.ReadMapKeyZC(bts)
switch msgp.UnsafeString(field) {
case "Stats":
var zb0002 uint32
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, zb0001)
err = msgp.WrapError(err, "Stats")
return
}
switch msgp.UnsafeString(field) {
case "ReplicationStats":
bts, err = zb0002.ReplicationStats.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, zb0001, "ReplicationStats")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err, zb0001)
return
if z.Stats == nil {
z.Stats = make(map[string]BucketStats, zb0002)
} else if len(z.Stats) > 0 {
for key := range z.Stats {
delete(z.Stats, key)
}
}
for zb0002 > 0 {
var za0001 string
var za0002 BucketStats
zb0002--
za0001, bts, err = msgp.ReadStringBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Stats")
return
}
var zb0003 uint32
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Stats", za0001)
return
}
for zb0003 > 0 {
zb0003--
field, bts, err = msgp.ReadMapKeyZC(bts)
if err != nil {
err = msgp.WrapError(err, "Stats", za0001)
return
}
switch msgp.UnsafeString(field) {
case "ReplicationStats":
bts, err = za0002.ReplicationStats.UnmarshalMsg(bts)
if err != nil {
err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err, "Stats", za0001)
return
}
}
}
z.Stats[za0001] = za0002
}
case "Timestamp":
z.Timestamp, bts, err = msgp.ReadTimeBytes(bts)
if err != nil {
err = msgp.WrapError(err, "Timestamp")
return
}
default:
bts, err = msgp.Skip(bts)
if err != nil {
err = msgp.WrapError(err)
return
}
}
(*z)[zb0001] = zb0002
}
o = bts
return
}
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z BucketStatsMap) Msgsize() (s int) {
s = msgp.MapHeaderSize
if z != nil {
for zb0005, zb0006 := range z {
_ = zb0006
s += msgp.StringPrefixSize + len(zb0005) + 1 + 17 + zb0006.ReplicationStats.Msgsize()
func (z *BucketStatsMap) Msgsize() (s int) {
s = 1 + 6 + msgp.MapHeaderSize
if z.Stats != nil {
for za0001, za0002 := range z.Stats {
_ = za0002
s += msgp.StringPrefixSize + len(za0001) + 1 + 17 + za0002.ReplicationStats.Msgsize()
}
}
s += 10 + msgp.TimeSize
return
}
+146 -38
View File
@@ -19,6 +19,8 @@ package cmd
import (
"context"
"fmt"
"net/url"
"sync"
"time"
@@ -28,14 +30,15 @@ import (
miniogo "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/bucket/versioning"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger"
)
const (
defaultHealthCheckDuration = 30 * time.Second
defaultHealthCheckDuration = 5 * time.Second
// default interval for reload of all remote target endpoints
defaultHealthCheckReloadDuration = 30 * time.Minute
)
// BucketTargetSys represents bucket targets subsystem
@@ -43,6 +46,113 @@ type BucketTargetSys struct {
sync.RWMutex
arnRemotesMap map[string]*TargetClient
targetsMap map[string][]madmin.BucketTarget
hMutex sync.RWMutex
hc map[string]epHealth
hcClient *madmin.AnonymousClient
}
// epHealth struct represents health of a replication target endpoint.
type epHealth struct {
Endpoint string
Scheme string
Online bool
}
// isOffline returns current liveness result of remote target. Add endpoint to
// healthcheck map if missing and default to online status
func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
sys.hMutex.RLock()
defer sys.hMutex.RUnlock()
if h, ok := sys.hc[ep.Host]; ok {
return !h.Online
}
go sys.initHC(ep)
return false
}
func (sys *BucketTargetSys) initHC(ep *url.URL) {
sys.hMutex.Lock()
sys.hc[ep.Host] = epHealth{
Endpoint: ep.Host,
Scheme: ep.Scheme,
Online: true,
}
sys.hMutex.Unlock()
}
// newHCClient initializes an anonymous client for performing health check on the remote endpoints
func newHCClient() *madmin.AnonymousClient {
clnt, e := madmin.NewAnonymousClientNoEndpoint()
if e != nil {
logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to initialize health check client"), string(replicationSubsystem))
return nil
}
clnt.SetCustomTransport(globalRemoteTargetTransport)
return clnt
}
// heartBeat performs liveness check on remote endpoints.
func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
hcTimer := time.NewTimer(defaultHealthCheckDuration)
defer hcTimer.Stop()
for {
select {
case <-hcTimer.C:
sys.hMutex.RLock()
var eps []madmin.ServerProperties
for _, ep := range sys.hc {
eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme})
}
sys.hMutex.RUnlock()
if len(eps) > 0 {
cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
m := map[string]epHealth{}
for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
var online bool
if result.Error == nil {
online = result.Online
}
m[result.Endpoint.Host] = epHealth{
Endpoint: result.Endpoint.Host,
Scheme: result.Endpoint.Scheme,
Online: online,
}
}
cancel()
sys.hMutex.Lock()
sys.hc = m
sys.hMutex.Unlock()
}
hcTimer.Reset(defaultHealthCheckDuration)
case <-ctx.Done():
return
}
}
}
// periodically rebuild the healthcheck map from list of targets to clear
// out stale endpoints
func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) {
m := make(map[string]epHealth)
tgts := sys.ListTargets(ctx, "", "")
for _, t := range tgts {
if _, ok := m[t.Endpoint]; !ok {
scheme := "http"
if t.Secure {
scheme = "https"
}
m[t.Endpoint] = epHealth{
Online: true,
Endpoint: t.Endpoint,
Scheme: scheme,
}
}
}
sys.hMutex.Lock()
// swap out the map
sys.hc = m
sys.hMutex.Unlock()
}
// ListTargets lists bucket targets across tenant or for individual bucket, and returns
@@ -91,9 +201,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
return
}
for _, t := range tgts {
if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
tgt.healthCancelFn()
}
delete(sys.arnRemotesMap, t.Arn)
}
delete(sys.targetsMap, bucket)
@@ -101,9 +208,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
// SetTarget - sets a new minio-go client target for this bucket.
func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
if globalIsGateway {
return nil
}
if !tgt.Type.IsValid() && !update {
return BucketRemoteArnTypeInvalid{Bucket: bucket}
}
@@ -112,11 +216,15 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
}
// validate if target credentials are ok
if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)
if err != nil {
if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
}
return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
}
if !exists {
return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
}
if tgt.Type == madmin.ReplicationService {
if !globalBucketVersioningSys.Enabled(bucket) {
@@ -124,9 +232,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
}
vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
if err != nil {
return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
}
if vcfg.Status != string(versioning.Enabled) {
if !vcfg.Enabled() {
return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
}
}
@@ -152,10 +260,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
if !found && !update {
newtgts = append(newtgts, *tgt)
}
// cancel health check for previous target client to avoid leak.
if prevClnt, ok := sys.arnRemotesMap[tgt.Arn]; ok && prevClnt.healthCancelFn != nil {
prevClnt.healthCancelFn()
}
sys.targetsMap[bucket] = newtgts
sys.arnRemotesMap[tgt.Arn] = clnt
sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
@@ -163,9 +268,6 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
}
func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
if globalIsGateway {
return
}
if limit == 0 {
globalBucketMonitor.DeleteBucket(bucket)
return
@@ -177,10 +279,6 @@ func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
// RemoveTarget - removes a remote bucket target for this source bucket.
func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
if globalIsGateway {
return nil
}
if arnStr == "" {
return BucketRemoteArnInvalid{Bucket: bucket}
}
@@ -227,9 +325,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
return BucketRemoteTargetNotFound{Bucket: bucket}
}
sys.targetsMap[bucket] = targets
if tgt, ok := sys.arnRemotesMap[arnStr]; ok && tgt.healthCancelFn != nil {
tgt.healthCancelFn()
}
delete(sys.arnRemotesMap, arnStr)
sys.updateBandwidthLimit(bucket, 0)
return nil
@@ -258,11 +353,29 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
}
// NewBucketTargetSys - creates new replication system.
func NewBucketTargetSys() *BucketTargetSys {
return &BucketTargetSys{
func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
sys := &BucketTargetSys{
arnRemotesMap: make(map[string]*TargetClient),
targetsMap: make(map[string][]madmin.BucketTarget),
hc: make(map[string]epHealth),
hcClient: newHCClient(),
}
// reload healthcheck endpoints map periodically to remove stale endpoints from the map.
go func() {
rTimer := time.NewTimer(defaultHealthCheckReloadDuration)
defer rTimer.Stop()
for {
select {
case <-rTimer.C:
sys.reloadHealthCheckers(ctx)
rTimer.Reset(defaultHealthCheckReloadDuration)
case <-ctx.Done():
return
}
}
}()
go sys.heartBeat(ctx)
return sys
}
// UpdateAllTargets updates target to reflect metadata updates
@@ -276,9 +389,6 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
// Remove existingtarget and arn association
if tgts, ok := sys.targetsMap[bucket]; ok {
for _, t := range tgts {
if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
tgt.healthCancelFn()
}
delete(sys.arnRemotesMap, t.Arn)
}
delete(sys.targetsMap, bucket)
@@ -345,10 +455,6 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
hcDuration = tcfg.HealthCheckDuration
}
cancelFn, err := api.HealthCheck(hcDuration)
if err != nil {
return nil, err
}
tc := &TargetClient{
Client: api,
healthCheckDuration: hcDuration,
@@ -356,9 +462,10 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
Bucket: tcfg.TargetBucket,
StorageClass: tcfg.StorageClass,
disableProxy: tcfg.DisableProxy,
healthCancelFn: cancelFn,
ARN: tcfg.Arn,
ResetID: tcfg.ResetID,
Endpoint: tcfg.Endpoint,
Secure: tcfg.Secure,
}
return tc, nil
}
@@ -432,7 +539,8 @@ type TargetClient struct {
replicateSync bool
StorageClass string // storage class on remote
disableProxy bool
healthCancelFn context.CancelFunc // cancellation function for client healthcheck
ARN string // ARN to uniquely identify remote target
ARN string // ARN to uniquely identify remote target
ResetID string
Endpoint string
Secure bool
}
+1 -1
View File
@@ -143,7 +143,7 @@ func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r
}
// Check if bucket exists.
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
-5
View File
@@ -69,11 +69,6 @@ func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
// Get returns stored bucket policy
func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
if globalIsGateway {
// Gateway does not implement versioning.
return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
}
if bucket == minioMetaBucket || strings.HasPrefix(bucket, minioMetaBucket) {
return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
}
+28 -35
View File
@@ -25,7 +25,6 @@ import (
"github.com/minio/madmin-go"
"github.com/minio/minio/internal/logger"
uatomic "go.uber.org/atomic"
)
const (
@@ -34,9 +33,6 @@ const (
// callhomeSchemaVersion is current callhome schema version.
callhomeSchemaVersion = callhomeSchemaVersion1
// callhomeCycleDefault is the default interval between two callhome cycles (24hrs)
callhomeCycleDefault = 24 * time.Hour
)
// CallhomeInfo - Contains callhome information
@@ -45,26 +41,14 @@ type CallhomeInfo struct {
AdminInfo madmin.InfoMessage `json:"admin_info"`
}
var (
enableCallhome = uatomic.NewBool(false)
callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
callhomeFreq = uatomic.NewDuration(callhomeCycleDefault)
)
func updateCallhomeParams(ctx context.Context, objAPI ObjectLayer) {
alreadyEnabled := enableCallhome.Load()
enableCallhome.Store(globalCallhomeConfig.Enable)
callhomeFreq.Store(globalCallhomeConfig.Frequency)
// If callhome was disabled earlier and has now been enabled,
// initialize the callhome process again.
if !alreadyEnabled && enableCallhome.Load() {
initCallhome(ctx, objAPI)
}
}
var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
// initCallhome will start the callhome task in the background.
func initCallhome(ctx context.Context, objAPI ObjectLayer) {
if !globalCallhomeConfig.Enabled() {
return
}
go func() {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
// Leader node (that successfully acquires the lock inside runCallhome)
@@ -72,54 +56,63 @@ func initCallhome(ctx context.Context, objAPI ObjectLayer) {
// the lock will be released and another node will acquire it and take over
// because of this loop.
for {
runCallhome(ctx, objAPI)
if !enableCallhome.Load() {
if !globalCallhomeConfig.Enabled() {
return
}
if !runCallhome(ctx, objAPI) {
// callhome was disabled or context was canceled
return
}
// callhome running on a different node.
// sleep for some time and try again.
duration := time.Duration(r.Float64() * float64(callhomeFreq.Load()))
duration := time.Duration(r.Float64() * float64(globalCallhomeConfig.FrequencyDur()))
if duration < time.Second {
// Make sure to sleep atleast a second to avoid high CPU ticks.
duration = time.Second
}
time.Sleep(duration)
if !enableCallhome.Load() {
return
}
}
}()
}
func runCallhome(ctx context.Context, objAPI ObjectLayer) {
func runCallhome(ctx context.Context, objAPI ObjectLayer) bool {
// Make sure only 1 callhome is running on the cluster.
locker := objAPI.NewNSLock(minioMetaBucket, "callhome/runCallhome.lock")
lkctx, err := locker.GetLock(ctx, callhomeLeaderLockTimeout)
if err != nil {
return
// lock timedout means some other node is the leader,
// cycle back return 'true'
return true
}
ctx = lkctx.Context()
defer locker.Unlock(lkctx.Cancel)
callhomeTimer := time.NewTimer(callhomeFreq.Load())
callhomeTimer := time.NewTimer(globalCallhomeConfig.FrequencyDur())
defer callhomeTimer.Stop()
for {
if !globalCallhomeConfig.Enabled() {
// Stop the processing as callhome got disabled
return false
}
select {
case <-ctx.Done():
return
// indicates that we do not need to run callhome anymore
return false
case <-callhomeTimer.C:
if !enableCallhome.Load() {
if !globalCallhomeConfig.Enabled() {
// Stop the processing as callhome got disabled
return
return false
}
performCallhome(ctx)
// Reset the timer for next cycle.
callhomeTimer.Reset(callhomeFreq.Load())
callhomeTimer.Reset(globalCallhomeConfig.FrequencyDur())
}
}
}
+64 -67
View File
@@ -27,7 +27,6 @@ import (
"encoding/pem"
"errors"
"fmt"
"io/ioutil"
"math/rand"
"net"
"net/http"
@@ -135,7 +134,7 @@ func init() {
globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
PassHost: true,
RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
RoundTripper: newHTTPTransport(1 * time.Hour),
Logger: func(err error) {
if err != nil && !errors.Is(err, context.Canceled) {
logger.LogIf(GlobalContext, err)
@@ -152,7 +151,7 @@ func init() {
defaultAWSCredProvider = []credentials.Provider{
&credentials.IAM{
Client: &http.Client{
Transport: NewGatewayHTTPTransport(),
Transport: NewHTTPTransport(),
},
},
}
@@ -176,7 +175,9 @@ func minioConfigToConsoleFeatures() {
if globalMinioEndpoint != "" {
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
} else {
os.Setenv("CONSOLE_MINIO_SERVER", getAPIEndpoints()[0])
// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
}
if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
@@ -203,20 +204,13 @@ func minioConfigToConsoleFeatures() {
}
}
// Enable if LDAP is enabled.
if globalLDAPConfig.Enabled {
if globalLDAPConfig.Enabled() {
os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
}
os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
if globalSubnetConfig.License != "" {
os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
}
if globalSubnetConfig.APIKey != "" {
os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
}
if globalSubnetConfig.ProxyURL != nil {
os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.ProxyURL.String())
}
globalSubnetConfig.ApplyEnv()
}
func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
@@ -237,6 +231,7 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
Userinfo: cfg.ClaimUserinfo,
RedirectCallbackDynamic: cfg.RedirectURIDynamic,
RedirectCallback: callback,
RoleArn: cfg.GetRoleArn(),
}
}
return m
@@ -264,7 +259,7 @@ func initConsoleServer() (*restapi.Server, error) {
return nil, err
}
api := operations.NewConsoleAPI(swaggerSpec, buildOpenIDConsoleConfig())
api := operations.NewConsoleAPI(swaggerSpec)
if !serverDebugLog {
// Disable console logging if server debug log is not enabled
@@ -275,6 +270,12 @@ func initConsoleServer() (*restapi.Server, error) {
api.Logger = noLog
}
// Pass in console application config. This needs to happen before the
// ConfigureAPI() call.
restapi.GlobalMinIOConfig = restapi.MinIOConfig{
OpenIDProviders: buildOpenIDConsoleConfig(),
}
server := restapi.NewServer(api)
// register all APIs
server.ConfigureAPI()
@@ -300,22 +301,6 @@ func initConsoleServer() (*restapi.Server, error) {
return server, nil
}
func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
if strings.HasPrefix(name, "gateway") {
if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
logger.Fatal(uiErr, "Unable to start gateway with SSE")
}
}
globalCompressConfigMu.Lock()
if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
logger.Fatal(errInvalidArgument,
"Compression support is requested but '%s' does not support compression", name)
}
globalCompressConfigMu.Unlock()
}
// Check for updates and print a notification message
func checkUpdate(mode string) {
updateURL := minioReleaseInfoURL
@@ -502,8 +487,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
Skip: true,
}, nil
}
const envSeparator = "="
envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
if len(envTokens) != 2 {
return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
}
@@ -529,7 +513,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
// the environment values from a file, in the form "key, value".
// in a structured form.
func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
f, err := os.Open(envConfigFile)
f, err := Open(envConfigFile)
if err != nil {
return nil, err
}
@@ -559,7 +543,7 @@ func readFromSecret(sp string) (string, error) {
if isFile(pathJoin("/run/secrets/", sp)) {
sp = pathJoin("/run/secrets/", sp)
}
credBuf, err := ioutil.ReadFile(sp)
credBuf, err := os.ReadFile(sp)
if err != nil {
if os.IsNotExist(err) { // ignore if file doesn't exist.
return "", nil
@@ -824,43 +808,56 @@ func handleKMSConfig() {
endpoints = append(endpoints, strings.Join(lbls, ""))
}
}
// Manually load the certificate and private key into memory.
// We need to check whether the private key is encrypted, and
// if so, decrypt it using the user-provided password.
certBytes, err := os.ReadFile(env.Get(config.EnvKESClientCert, ""))
if err != nil {
logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
}
keyBytes, err := os.ReadFile(env.Get(config.EnvKESClientKey, ""))
if err != nil {
logger.Fatal(err, "Unable to load KES client private key as specified by the shell environment")
}
privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
if len(rest) != 0 {
logger.Fatal(errors.New("private key contains additional data"), "Unable to load KES client private key as specified by the shell environment")
}
if x509.IsEncryptedPEMBlock(privateKeyPEM) {
keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
if err != nil {
logger.Fatal(err, "Unable to decrypt KES client private key as specified by the shell environment")
}
keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
}
certificate, err := tls.X509KeyPair(certBytes, keyBytes)
if err != nil {
logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
}
rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
if err != nil {
logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
}
loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
// Manually load the certificate and private key into memory.
// We need to check whether the private key is encrypted, and
// if so, decrypt it using the user-provided password.
certBytes, err := os.ReadFile(certFile)
if err != nil {
return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
}
keyBytes, err := os.ReadFile(keyFile)
if err != nil {
return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
}
privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
if len(rest) != 0 {
return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
}
if x509.IsEncryptedPEMBlock(privateKeyPEM) {
keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
if err != nil {
return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
}
keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
}
certificate, err := tls.X509KeyPair(certBytes, keyBytes)
if err != nil {
return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
}
return certificate, nil
}
reloadCertEvents := make(chan tls.Certificate, 1)
certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
if err != nil {
logger.Fatal(err, "Failed to load KES client certificate")
}
certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
certificate.Notify(reloadCertEvents)
defaultKeyID := env.Get(config.EnvKESKeyName, "")
KMS, err := kms.NewWithConfig(kms.Config{
Endpoints: endpoints,
DefaultKeyID: defaultKeyID,
Certificate: certificate,
RootCAs: rootCAs,
Endpoints: endpoints,
DefaultKeyID: defaultKeyID,
Certificate: certificate,
ReloadCertEvents: reloadCertEvents,
RootCAs: rootCAs,
})
if err != nil {
logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
@@ -910,7 +907,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
// Therefore, we read all filenames in the cert directory and check
// for each directory whether it contains a public.crt and private.key.
// If so, we try to add it to certificate manager.
root, err := os.Open(globalCertsDir.Get())
root, err := Open(globalCertsDir.Get())
if err != nil {
return nil, nil, false, err
}
@@ -929,7 +926,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
continue
}
if file.Mode()&os.ModeSymlink == os.ModeSymlink {
file, err = os.Stat(filepath.Join(root.Name(), file.Name()))
file, err = Stat(filepath.Join(root.Name(), file.Name()))
if err != nil {
// not accessible ignore
continue
+3 -3
View File
@@ -19,7 +19,7 @@ package cmd
import (
"errors"
"io/ioutil"
"os"
"reflect"
"testing"
)
@@ -45,7 +45,7 @@ func Test_readFromSecret(t *testing.T) {
for _, testCase := range testCases {
testCase := testCase
t.Run("", func(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "testfile")
tmpfile, err := os.CreateTemp("", "testfile")
if err != nil {
t.Error(err)
}
@@ -157,7 +157,7 @@ MINIO_ROOT_PASSWORD=minio123`,
for _, testCase := range testCases {
testCase := testCase
t.Run("", func(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "testfile")
tmpfile, err := os.CreateTemp("", "testfile")
if err != nil {
t.Error(err)
}
+16 -8
View File
@@ -21,7 +21,7 @@ import (
"bytes"
"context"
"errors"
"io/ioutil"
"io"
"net/http"
"github.com/minio/minio/internal/hash"
@@ -29,10 +29,14 @@ import (
var errConfigNotFound = errors.New("config file not found")
func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string) ([]byte, ObjectInfo, error) {
r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string, opts ObjectOptions) ([]byte, ObjectInfo, error) {
lockType := readLock
if opts.NoLock {
lockType = noLock // erasureObjects.GetObjectNInfo honors lockType argument but not opts.NoLock.
}
r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, lockType, opts)
if err != nil {
// Treat object not found as config not found.
if isErrObjectNotFound(err) {
return nil, ObjectInfo{}, errConfigNotFound
}
@@ -41,7 +45,7 @@ func readConfigWithMetadata(ctx context.Context, store objectIO, configFile stri
}
defer r.Close()
buf, err := ioutil.ReadAll(r)
buf, err := io.ReadAll(r)
if err != nil {
return nil, ObjectInfo{}, err
}
@@ -52,7 +56,7 @@ func readConfigWithMetadata(ctx context.Context, store objectIO, configFile stri
}
func readConfig(ctx context.Context, store objectIO, configFile string) ([]byte, error) {
buf, _, err := readConfigWithMetadata(ctx, store, configFile)
buf, _, err := readConfigWithMetadata(ctx, store, configFile, ObjectOptions{})
return buf, err
}
@@ -70,16 +74,20 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
return err
}
func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error {
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
if err != nil {
return err
}
_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{MaxParity: true})
_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), opts)
return err
}
func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
return saveConfigWithOpts(ctx, store, configFile, data, ObjectOptions{MaxParity: true})
}
func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
// Treat object not found as config not found.
+89 -162
View File
@@ -25,6 +25,7 @@ import (
"sync"
"github.com/minio/madmin-go"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/config/api"
"github.com/minio/minio/internal/config/cache"
@@ -68,7 +69,6 @@ func initHelp() {
config.LoggerWebhookSubSys: logger.DefaultLoggerWebhookKVS,
config.AuditWebhookSubSys: logger.DefaultAuditWebhookKVS,
config.AuditKafkaSubSys: logger.DefaultAuditKafkaKVS,
config.HealSubSys: heal.DefaultKVS,
config.ScannerSubSys: scanner.DefaultKVS,
config.SubnetSubSys: subnet.DefaultKVS,
config.CallhomeSubSys: callhome.DefaultKVS,
@@ -78,6 +78,7 @@ func initHelp() {
}
if globalIsErasure {
kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
kvs[config.HealSubSys] = heal.DefaultKVS
}
config.RegisterDefaultKVS(kvs)
@@ -124,10 +125,6 @@ func initHelp() {
Key: config.APISubSys,
Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
},
config.HelpKV{
Key: config.HealSubSys,
Description: "manage object healing frequency and bitrot verification checks",
},
config.HelpKV{
Key: config.ScannerSubSys,
Description: "manage namespace scanning for usage calculation, lifecycle, healing and more",
@@ -212,12 +209,13 @@ func initHelp() {
}
if globalIsErasure {
helpSubSys = append(helpSubSys, config.HelpKV{})
copy(helpSubSys[2:], helpSubSys[1:])
helpSubSys[1] = config.HelpKV{
helpSubSys = append(helpSubSys, config.HelpKV{
Key: config.StorageClassSubSys,
Description: "define object level redundancy",
}
}, config.HelpKV{
Key: config.HealSubSys,
Description: "manage object healing frequency and bitrot verification checks",
})
}
helpMap := map[string]config.HelpKVS{
@@ -279,10 +277,6 @@ var (
func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) error {
switch subSys {
case config.CredentialsSubSys:
if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil {
return err
}
case config.SiteSubSys:
if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
return err
@@ -292,14 +286,12 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
return err
}
case config.StorageClassSubSys:
if globalIsErasure {
if objAPI == nil {
return errServerNotInitialized
}
for _, setDriveCount := range objAPI.SetDriveCounts() {
if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
return err
}
if objAPI == nil {
return errServerNotInitialized
}
for _, setDriveCount := range objAPI.SetDriveCounts() {
if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
return err
}
}
case config.CacheSubSys:
@@ -339,16 +331,16 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
}
case config.IdentityOpenIDSubSys:
if _, err := openid.LookupConfig(s,
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
return err
}
case config.IdentityLDAPSubSys:
cfg, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default], globalRootCAs)
cfg, err := xldap.Lookup(s, globalRootCAs)
if err != nil {
return err
}
if cfg.Enabled {
conn, cerr := cfg.Connect()
if cfg.Enabled() {
conn, cerr := cfg.LDAP.Connect()
if cerr != nil {
return cerr
}
@@ -360,7 +352,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
}
case config.IdentityPluginSubSys:
if _, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
return err
}
case config.SubnetSubSys:
@@ -368,9 +360,14 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
return err
}
case config.CallhomeSubSys:
if _, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default]); err != nil {
cfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
if err != nil {
return err
}
// callhome cannot be enabled if license is not registered yet, throw an error.
if cfg.Enabled() && !globalSubnetConfig.Registered() {
return errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'")
}
case config.PolicyOPASubSys:
// In case legacy OPA config is being set, we treat it as if the
// AuthZPlugin is being set.
@@ -378,12 +375,12 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
fallthrough
case config.PolicyPluginSubSys:
if ppargs, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
NewHTTPTransport(), xhttp.DrainBody); err != nil {
return err
} else if ppargs.URL == nil {
// Check if legacy opa is configured.
if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
NewHTTPTransport(), xhttp.DrainBody); err != nil {
return err
}
}
@@ -396,7 +393,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
}
if config.NotifySubSystems.Contains(subSys) {
if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs(), subSys); err != nil {
if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
return err
}
}
@@ -442,51 +439,33 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
if err != nil {
if globalIsGateway {
logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
} else {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
}
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
}
if err == nil && dnsURL != "" {
globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
dns.Authentication(dnsUser, dnsPass),
dns.RootCAs(globalRootCAs))
if err != nil {
if globalIsGateway {
logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
} else {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
}
logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
}
}
etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
if err != nil {
if globalIsGateway {
logger.FatalIf(err, "Unable to initialize etcd config")
} else {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
}
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
}
if etcdCfg.Enabled {
if globalEtcdClient == nil {
globalEtcdClient, err = etcd.New(etcdCfg)
if err != nil {
if globalIsGateway {
logger.FatalIf(err, "Unable to initialize etcd config")
} else {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
}
}
globalEtcdClient, err = etcd.New(etcdCfg)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
}
if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
if globalDNSConfig != nil {
// if global DNS is already configured, indicate with a warning, incase
// users are confused.
logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
} else {
globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
dns.DomainNames(globalDomainNames),
@@ -495,12 +474,8 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
dns.CoreDNSPath(etcdCfg.CoreDNSPath),
)
if err != nil {
if globalIsGateway {
logger.FatalIf(err, "Unable to initialize DNS config")
} else {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
globalDomainNames, err))
}
logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
globalDomainNames, err))
}
}
}
@@ -518,23 +493,9 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
}
apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
}
// Initialize remote instance transport once.
getRemoteInstanceTransportOnce.Do(func() {
getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
})
globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
if err != nil {
if globalIsGateway {
logger.FatalIf(err, "Unable to setup cache")
} else {
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
}
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
}
if globalCacheConfig.Enabled {
@@ -560,92 +521,46 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
}
globalOpenIDConfig, err = openid.LookupConfig(s,
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
}
transport := NewHTTPTransport()
globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
globalRootCAs)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
}
authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
}
globalAuthNPlugin = idplugin.New(authNPluginCfg)
authZPluginCfg, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
NewGatewayHTTPTransport(), xhttp.DrainBody)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err))
}
if authZPluginCfg.URL == nil {
opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
NewGatewayHTTPTransport(), xhttp.DrainBody)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
} else {
authZPluginCfg.URL = opaCfg.URL
authZPluginCfg.AuthToken = opaCfg.AuthToken
authZPluginCfg.Transport = opaCfg.Transport
authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
}
}
setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
}
globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
}
globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), NewGatewayHTTPTransport(), true)
globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
}
// Apply dynamic config values
if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
if globalIsGateway {
logger.FatalIf(err, "Unable to initialize dynamic configuration")
} else {
logger.LogIf(ctx, err)
}
logger.LogIf(ctx, err)
}
}
func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s config.Config, subSys string) error {
if objAPI == nil {
return errServerNotInitialized
}
setDriveCounts := objAPI.SetDriveCounts()
switch subSys {
case config.APISubSys:
apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
}
var setDriveCounts []int
if objAPI != nil {
setDriveCounts = objAPI.SetDriveCounts()
}
globalAPIConfig.init(apiConfig, setDriveCounts)
// Initialize remote instance transport once.
getRemoteInstanceTransportOnce.Do(func() {
getRemoteInstanceTransport = newHTTPTransport(apiConfig.RemoteTransportDeadline)
})
case config.CompressionSubSys:
cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
if err != nil {
return fmt.Errorf("Unable to setup Compression: %w", err)
}
// Validate if the object layer supports compression.
if objAPI != nil {
if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
return fmt.Errorf("Backend does not support compression")
}
if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
return fmt.Errorf("Backend does not support compression")
}
globalCompressConfigMu.Lock()
globalCompressConfig = cmpCfg
@@ -674,12 +589,11 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
if l.Enabled {
l.LogOnce = logger.LogOnceConsoleIf
l.UserAgent = userAgent
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
loggerCfg.HTTP[n] = l
}
}
err = logger.UpdateSystemTargets(loggerCfg)
if err != nil {
if err = logger.UpdateSystemTargets(loggerCfg); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
}
case config.AuditWebhookSubSys:
@@ -692,13 +606,12 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
if l.Enabled {
l.LogOnce = logger.LogOnceConsoleIf
l.UserAgent = userAgent
l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
loggerCfg.AuditWebhook[n] = l
}
}
err = logger.UpdateAuditWebhookTargets(loggerCfg)
if err != nil {
if err = logger.UpdateAuditWebhookTargets(loggerCfg); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
}
case config.AuditKafkaSubSys:
@@ -712,33 +625,40 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
loggerCfg.AuditKafka[n] = l
}
}
err = logger.UpdateAuditKafkaTargets(loggerCfg)
if err != nil {
if err = logger.UpdateAuditKafkaTargets(loggerCfg); err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
}
case config.StorageClassSubSys:
if globalIsErasure && objAPI != nil {
setDriveCounts := objAPI.SetDriveCounts()
for i, setDriveCount := range setDriveCounts {
sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
break
}
// if we validated all setDriveCounts and it was successful
// proceed to store the correct storage class globally.
if i == len(setDriveCounts)-1 {
globalStorageClass.Update(sc)
}
for i, setDriveCount := range setDriveCounts {
sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
break
}
// if we validated all setDriveCounts and it was successful
// proceed to store the correct storage class globally.
if i == len(setDriveCounts)-1 {
globalStorageClass.Update(sc)
}
}
case config.SubnetSubSys:
subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
} else {
globalSubnetConfig.Update(subnetConfig)
globalSubnetConfig.ApplyEnv() // update environment settings for Console UI
}
case config.CallhomeSubSys:
callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
if err != nil {
logger.LogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
} else {
globalCallhomeConfig = callhomeCfg
updateCallhomeParams(ctx, objAPI)
enable := callhomeCfg.Enable && !globalCallhomeConfig.Enabled()
globalCallhomeConfig.Update(callhomeCfg)
if enable {
initCallhome(ctx, objAPI)
}
}
}
globalServerConfigMu.Lock()
@@ -846,13 +766,20 @@ func newSrvConfig(objAPI ObjectLayer) error {
// Initialize server config.
srvCfg := newServerConfig()
if globalActiveCred.IsValid() && !globalActiveCred.Equal(auth.DefaultCredentials) {
kvs := srvCfg[config.CredentialsSubSys][config.Default]
kvs.Set(config.AccessKey, globalActiveCred.AccessKey)
kvs.Set(config.SecretKey, globalActiveCred.SecretKey)
srvCfg[config.CredentialsSubSys][config.Default] = kvs
}
// hold the mutex lock before a new config is assigned.
globalServerConfigMu.Lock()
globalServerConfig = srvCfg
globalServerConfigMu.Unlock()
// Save config into file.
return saveServerConfig(GlobalContext, objAPI, globalServerConfig)
return saveServerConfig(GlobalContext, objAPI, srvCfg)
}
func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
+4 -1
View File
@@ -26,7 +26,10 @@ import (
)
func TestServerConfig(t *testing.T) {
objLayer, fsDir, err := prepareFS()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
objLayer, fsDir, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
+47 -26
View File
@@ -69,6 +69,12 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
return err
}
// If backend doesn't have this file means we have already
// attempted then migration
if !encrypted {
return nil
}
if encrypted && GlobalKMS != nil {
stat, err := GlobalKMS.Stat(ctx)
if err != nil {
@@ -109,6 +115,8 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
}
}
} else {
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
}
}
data = pdata
@@ -131,6 +139,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
if encrypted && GlobalKMS != nil {
logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
}
return deleteKeyEtcd(ctx, client, backendEncryptedFile)
}
@@ -146,44 +155,56 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
}
var marker string
for {
res, err := objAPI.ListObjects(GlobalContext, minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
results := make(chan ObjectInfo)
if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil {
return err
}
for obj := range results {
data, err := readConfig(GlobalContext, objAPI, obj.Name)
if err != nil {
return err
}
for _, obj := range res.Objects {
data, err := readConfig(GlobalContext, objAPI, obj.Name)
if !utf8.Valid(data) {
pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
if err != nil {
if GlobalKMS != nil {
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
minioMetaBucket: path.Join(minioMetaBucket, obj.Name),
})
if err != nil {
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
minioMetaBucket: obj.Name,
})
if err != nil {
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
}
}
} else {
return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
}
}
data = pdata
}
if GlobalKMS != nil {
data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
obj.Bucket: path.Join(obj.Bucket, obj.Name),
})
if err != nil {
return err
}
if !utf8.Valid(data) {
data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
if err != nil {
return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
}
}
if GlobalKMS != nil {
data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
obj.Bucket: path.Join(obj.Bucket, obj.Name),
})
if err != nil {
return err
}
}
if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
return err
}
}
if !res.IsTruncated {
break
if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
return err
}
marker = res.NextMarker
}
if encrypted && GlobalKMS != nil {
logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
}
return deleteConfig(GlobalContext, objAPI, backendEncryptedFile)
}
+21 -38
View File
@@ -18,8 +18,8 @@
package cmd
import (
"context"
"fmt"
"io/ioutil"
"os"
"testing"
@@ -28,7 +28,10 @@ import (
// Test if config v1 is purged
func TestServerConfigMigrateV1(t *testing.T) {
objLayer, fsDir, err := prepareFS()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
objLayer, fsDir, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
@@ -37,11 +40,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
if err != nil {
t.Fatalf("Init Test config failed")
}
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
rootPath := t.TempDir()
globalConfigDir = &ConfigDir{path: rootPath}
globalObjLayerMutex.Lock()
@@ -51,7 +50,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
// Create a V1 config json file and store it
configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
configPath := rootPath + "/fsUsers.json"
if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
t.Fatal("Unexpected error: ", err)
}
@@ -74,13 +73,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
// Test if all migrate code returns nil when config file does not
// exist
func TestServerConfigMigrateInexistentConfig(t *testing.T) {
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
globalConfigDir = &ConfigDir{path: rootPath}
globalConfigDir = &ConfigDir{path: t.TempDir()}
if err := migrateV2ToV3(); err != nil {
t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@@ -164,15 +157,13 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
// Test if a config migration from v2 to v33 is successfully done
func TestServerConfigMigrateV2toV33(t *testing.T) {
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
rootPath := t.TempDir()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
globalConfigDir = &ConfigDir{path: rootPath}
objLayer, fsDir, err := prepareFS()
objLayer, fsDir, err := prepareFS(ctx)
if err != nil {
t.Fatal(err)
}
@@ -181,7 +172,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
configPath := rootPath + SlashSeparator + minioConfigFile
// Create a corrupted config file
if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
t.Fatal("Unexpected error: ", err)
}
// Fire a migrateConfig()
@@ -194,7 +185,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
// Create a V2 config json file and store it
configJSON := "{ \"version\":\"2\", \"credentials\": {\"accessKeyId\":\"" + accessKey + "\", \"secretAccessKey\":\"" + secretKey + "\", \"region\":\"us-east-1\"}, \"mongoLogger\":{\"addr\":\"127.0.0.1:3543\", \"db\":\"foodb\", \"collection\":\"foo\"}, \"syslogLogger\":{\"network\":\"127.0.0.1:543\", \"addr\":\"addr\"}, \"fileLogger\":{\"filename\":\"log.out\"}}"
if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
t.Fatal("Unexpected error: ", err)
}
@@ -234,17 +225,13 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
// Test if all migrate code returns error with corrupted config files
func TestServerConfigMigrateFaultyConfig(t *testing.T) {
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
rootPath := t.TempDir()
globalConfigDir = &ConfigDir{path: rootPath}
configPath := rootPath + SlashSeparator + minioConfigFile
// Create a corrupted config file
if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
t.Fatal("Unexpected error: ", err)
}
@@ -331,35 +318,31 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
// Test if all migrate code returns error with corrupted config files
func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(rootPath)
rootPath := t.TempDir()
globalConfigDir = &ConfigDir{path: rootPath}
configPath := rootPath + SlashSeparator + minioConfigFile
for i := 3; i <= 17; i++ {
// Create a corrupted config file
if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
if err := os.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
0o644); err != nil {
t.Fatal("Unexpected error: ", err)
}
// Test different migrate versions and be sure they are returning an error
if err = migrateConfig(); err == nil {
if err := migrateConfig(); err == nil {
t.Fatal("migrateConfig() should fail with a corrupted json")
}
}
// Create a corrupted config file for version '2'.
if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
t.Fatal("Unexpected error: ", err)
}
// Test different migrate versions and be sure they are returning an error
if err = migrateConfig(); err == nil {
if err := migrateConfig(); err == nil {
t.Fatal("migrateConfig() should fail with a corrupted json")
}
}
+1 -1
View File
@@ -855,5 +855,5 @@ type serverConfigV33 struct {
// Add new external policy enforcements here.
} `json:"policy"`
LDAPServerConfig xldap.Config `json:"ldapserverconfig"`
LDAPServerConfig xldap.LegacyConfig `json:"ldapserverconfig"`
}
+5 -6
View File
@@ -37,7 +37,7 @@ const defaultLogBufferCount = 10000
// HTTPConsoleLoggerSys holds global console logger state
type HTTPConsoleLoggerSys struct {
sync.RWMutex
pubsub *pubsub.PubSub
pubsub *pubsub.PubSub[log.Info, madmin.LogMask]
console *console.Target
nodeName string
logBuf *ring.Ring
@@ -46,9 +46,8 @@ type HTTPConsoleLoggerSys struct {
// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
// the console logging pub sub system
func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
ps := pubsub.New(8)
return &HTTPConsoleLoggerSys{
pubsub: ps,
pubsub: pubsub.New[log.Info, madmin.LogMask](8),
console: console.New(),
logBuf: ring.New(defaultLogBufferCount),
}
@@ -72,11 +71,11 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
// HasLogListeners returns true if console log listeners are registered
// for this node or peers
func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
return sys != nil && sys.pubsub.NumSubscribers(madmin.LogMaskAll) > 0
return sys != nil && sys.pubsub.Subscribers() > 0
}
// Subscribe starts console logging for this node.
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry pubsub.Maskable) bool) error {
func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry log.Info) bool) error {
// Enable console logging for remote client.
if !sys.HasLogListeners() {
logger.AddSystemTarget(sys)
@@ -116,7 +115,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-
}
}
}
return sys.pubsub.Subscribe(pubsub.MaskFromMaskable(madmin.LogMaskAll), subCh, doneCh, filter)
return sys.pubsub.Subscribe(madmin.LogMaskAll, subCh, doneCh, filter)
}
// Init if HTTPConsoleLoggerSys is valid, always returns nil right now
-7
View File
@@ -119,13 +119,6 @@ func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
}
}
func (p *scannerMetrics) incNoTime(s scannerMetric) {
atomic.AddUint64(&p.operations[s], 1)
if s < scannerMetricLastRealtime {
p.latency[s].add(0)
}
}
// timeILM times an ILM action.
// lifecycle.NoneAction is ignored.
// Use for s < scannerMetricLastRealtime
+48 -89
View File
@@ -64,7 +64,11 @@ const (
var (
globalHealConfig heal.Config
dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
dataScannerLeaderLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
timeout: 30 * time.Second,
minimum: 10 * time.Second,
retryInterval: time.Second,
})
// Sleeper values are updated when config is loaded.
scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
scannerCycle = uatomic.NewDuration(dataScannerStartDelay)
@@ -114,6 +118,10 @@ type backgroundHealInfo struct {
}
func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundHealInfo {
if globalIsErasureSD {
return backgroundHealInfo{}
}
// Get last healing information
buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
if err != nil {
@@ -123,15 +131,17 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
return backgroundHealInfo{}
}
var info backgroundHealInfo
err = json.Unmarshal(buf, &info)
if err != nil {
if err = json.Unmarshal(buf, &info); err != nil {
logger.LogIf(ctx, err)
return backgroundHealInfo{}
}
return info
}
func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgroundHealInfo) {
if globalIsErasureSD {
return
}
b, err := json.Marshal(info)
if err != nil {
logger.LogIf(ctx, err)
@@ -934,7 +944,10 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
ScanMode: scanMode,
}
res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
if err != nil && !errors.Is(err, NotImplemented{}) {
if err != nil {
if errors.Is(err, NotImplemented{}) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
err = nil
}
logger.LogIf(ctx, err)
return 0
}
@@ -956,21 +969,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
versionID := oi.VersionID
rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
if i.debug {
if versionID != "" {
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
} else {
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
}
}
defer globalScannerMetrics.timeILM(action)
defer globalScannerMetrics.timeILM(lcEvt.Action)
switch action {
switch lcEvt.Action {
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
return applyLifecycleAction(action, oi), 0
return applyLifecycleAction(lcEvt.Action, oi, ""), 0
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
return applyLifecycleAction(action, oi), size
return applyLifecycleAction(lcEvt.Action, oi, lcEvt.StorageClass), size
default:
// No action.
return false, size
@@ -1102,42 +1115,42 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
return size
}
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo, debug bool) (action lifecycle.Action) {
action = lc.ComputeAction(obj.ToLifecycleOpts())
if debug {
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
event := lc.Eval(obj.ToLifecycleOpts(), time.Now().UTC())
if serverDebugLog {
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
}
if action == lifecycle.NoneAction {
return action
if event.Action == lifecycle.NoneAction {
return event
}
switch action {
switch event.Action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
// Defensive code, should never happen
if obj.VersionID == "" {
return lifecycle.NoneAction
return lifecycle.Event{Action: lifecycle.NoneAction}
}
if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
if debug {
if serverDebugLog {
if obj.VersionID != "" {
console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
} else {
console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
}
}
return lifecycle.NoneAction
return lifecycle.Event{Action: lifecycle.NoneAction}
}
}
return action
return event
}
func applyTransitionRule(obj ObjectInfo) bool {
func applyTransitionRule(obj ObjectInfo, storageClass string) bool {
if obj.DeleteMarker {
return false
}
globalTransitionState.queueTransitionTask(obj)
globalTransitionState.queueTransitionTask(obj, storageClass)
return true
}
@@ -1206,14 +1219,14 @@ func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
}
// Perform actions (removal or transitioning of objects), return true the action is successfully performed
func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo, storageClass string) (success bool) {
switch action {
case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
success = applyTransitionRule(obj)
success = applyTransitionRule(obj, storageClass)
}
return
}
@@ -1225,29 +1238,16 @@ func (i *scannerItem) objectPath() string {
// healReplication will heal a scanned item that has failed replication.
func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
roi := getHealReplicateObjectInfo(oi, i.replication)
if !roi.Dsc.ReplicateAny() {
if oi.VersionID == "" {
return
}
if i.replication.Config == nil {
return
}
roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication)
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
// heal delete marker replication failure or versioned delete replication failure
if oi.ReplicationStatus == replication.Pending ||
oi.ReplicationStatus == replication.Failed ||
oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
i.healReplicationDeletes(ctx, o, roi)
return
}
// if replication status is Complete on DeleteMarker and existing object resync required
if roi.ExistingObjResync.mustResync() && (oi.ReplicationStatus == replication.Completed || oi.ReplicationStatus.Empty()) {
i.healReplicationDeletes(ctx, o, roi)
return
}
return
}
if roi.ExistingObjResync.mustResync() {
roi.OpType = replication.ExistingObjectReplicationType
}
if sizeS.replTargetStats == nil && len(roi.TargetStatuses) > 0 {
sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
@@ -1277,52 +1277,9 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
}
switch oi.ReplicationStatus {
case replication.Pending, replication.Failed:
roi.EventType = ReplicateHeal
globalReplicationPool.queueReplicaTask(roi)
return
case replication.Replica:
sizeS.replicaSize += oi.Size
}
if roi.ExistingObjResync.mustResync() {
roi.EventType = ReplicateExisting
globalReplicationPool.queueReplicaTask(roi)
}
}
// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, roi ReplicateObjectInfo) {
// handle soft delete and permanent delete failures here.
if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
versionID := ""
dmVersionID := ""
if roi.VersionPurgeStatus.Empty() {
dmVersionID = roi.VersionID
} else {
versionID = roi.VersionID
}
doi := DeletedObjectReplicationInfo{
DeletedObject: DeletedObject{
ObjectName: roi.Name,
DeleteMarkerVersionID: dmVersionID,
VersionID: versionID,
ReplicationState: roi.getReplicationState(roi.Dsc.String(), versionID, true),
DeleteMarkerMTime: DeleteMarkerMTime{roi.ModTime},
DeleteMarker: roi.DeleteMarker,
},
Bucket: roi.Bucket,
OpType: replication.HealReplicationType,
EventType: ReplicateHealDelete,
}
if roi.ExistingObjResync.mustResync() {
doi.OpType = replication.ExistingObjectReplicationType
doi.EventType = ReplicateExistingDelete
queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
return
}
globalReplicationPool.queueReplicaDeleteTask(doi)
}
}
type dynamicSleeper struct {
@@ -1490,9 +1447,11 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
case ILMTransition:
apiName = "ILMTransition"
}
auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
auditLogInternal(ctx, AuditLogOptions{
Event: event,
APIName: apiName,
Bucket: oi.Bucket,
Object: oi.Name,
VersionID: oi.VersionID,
})
}
+2 -3
View File
@@ -24,7 +24,6 @@ import (
"encoding/binary"
"errors"
"io"
"io/ioutil"
"os"
"path"
"sort"
@@ -227,7 +226,7 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
for _, drive := range drives {
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
f, err := os.Open(cacheFormatPath)
f, err := OpenFile(cacheFormatPath, readMode, 0o666)
if err != nil {
if osIsNotExist(err) {
continue
@@ -296,7 +295,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
}
for _, drive := range drives {
cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
err := ioutil.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
err := os.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
if err != nil {
if osIsNotExist(err) {
continue
+6 -9
View File
@@ -21,7 +21,6 @@ import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/rand"
"os"
"path"
@@ -100,15 +99,11 @@ func TestDataUpdateTracker(t *testing.T) {
dut.Current.bf = dut.newBloomFilter()
tmpDir, err := ioutil.TempDir("", "TestDataUpdateTracker")
tmpDir := t.TempDir()
err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
if err != nil {
t.Fatal(err)
}
err = os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dut.start(ctx, tmpDir)
@@ -205,11 +200,13 @@ func TestDataUpdateTracker(t *testing.T) {
}
ctx, cancel = context.WithCancel(context.Background())
defer cancel()
// Reload...
dut = newDataUpdateTracker()
dut.start(ctx, tmpDir)
defer func() {
cancel()
<-dut.saveExited
}()
if dut.current() != 2 {
t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
-44
View File
@@ -392,13 +392,6 @@ func (e *dataUsageEntry) addChild(hash dataUsageHash) {
e.Children[hash.Key()] = struct{}{}
}
// removeChild will remove a child based on its hash.
func (e *dataUsageEntry) removeChild(hash dataUsageHash) {
if len(e.Children) > 0 {
delete(e.Children, hash.Key())
}
}
// Create a clone of the entry.
func (e dataUsageEntry) clone() dataUsageEntry {
// We operate on a copy from the receiver.
@@ -488,43 +481,6 @@ func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {
}
}
// keepBuckets will keep only the buckets specified specified by delete all others.
func (d *dataUsageCache) keepBuckets(b []BucketInfo) {
lu := make(map[dataUsageHash]struct{})
for _, v := range b {
lu[hashPath(v.Name)] = struct{}{}
}
d.keepRootChildren(lu)
}
// keepRootChildren will keep the root children specified by delete all others.
func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
root := d.root()
if root == nil {
return
}
rh := d.rootHash()
for k := range d.Cache {
h := dataUsageHash(k)
if h == rh {
continue
}
if _, ok := list[h]; !ok {
delete(d.Cache, k)
d.deleteRecursive(h)
root.removeChild(h)
}
}
// Clean up abandoned children.
for k := range root.Children {
h := dataUsageHash(k)
if _, ok := list[h]; !ok {
delete(root.Children, k)
}
}
d.Cache[rh.Key()] = *root
}
// dui converts the flattened version of the path to madmin.DataUsageInfo.
// As a side effect d will be flattened, use a clone if this is not ok.
func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
+6
View File
@@ -651,6 +651,7 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(5)
var zb0001Mask uint8 /* 5 bits */
_ = zb0001Mask
if z.BloomFilter == nil {
zb0001Len--
zb0001Mask |= 0x10
@@ -724,6 +725,7 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
// omitempty: check for empty values
zb0001Len := uint32(5)
var zb0001Mask uint8 /* 5 bits */
_ = zb0001Mask
if z.BloomFilter == nil {
zb0001Len--
zb0001Mask |= 0x10
@@ -1652,6 +1654,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(8)
var zb0001Mask uint8 /* 8 bits */
_ = zb0001Mask
if z.ReplicationStats == nil {
zb0001Len--
zb0001Mask |= 0x20
@@ -1782,6 +1785,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
// omitempty: check for empty values
zb0001Len := uint32(8)
var zb0001Mask uint8 /* 8 bits */
_ = zb0001Mask
if z.ReplicationStats == nil {
zb0001Len--
zb0001Mask |= 0x20
@@ -2774,6 +2778,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(2)
var zb0001Mask uint8 /* 2 bits */
_ = zb0001Mask
if z.Targets == nil {
zb0001Len--
zb0001Mask |= 0x1
@@ -2835,6 +2840,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
// omitempty: check for empty values
zb0001Len := uint32(2)
var zb0001Mask uint8 /* 2 bits */
_ = zb0001Mask
if z.Targets == nil {
zb0001Len--
zb0001Mask |= 0x1
+2 -1
View File
@@ -56,7 +56,8 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
}
// loadPrefixUsageFromBackend returns prefix usages found in passed buckets
// e.g.: /testbucket/prefix => 355601334
//
// e.g.: /testbucket/prefix => 355601334
func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) {
z, ok := objAPI.(*erasureServerPools)
if !ok {

Some files were not shown because too many files have changed in this diff Show More