astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-09-09 03:54:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/access-based-ilm

Browse Source
This commit is contained in:
Feng Ruohang
2026-09-08 13:04:25 +08:00
committed by GitHub
parent 7a060cab1e 3abe0d95a5
commit 89b75e7913
308 changed files with 25104 additions and 12023 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/bucket/notifications/README.md
+6 -8
View File
@@ -840,12 +840,11 @@ Received a message: {"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jp
> database (string) database name (used only if `connection_string` is empty)
> ```
>
> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure
> to migrate to only using _connection_string_ option. To migrate, once you have upgraded all the servers use the
> following command to update the existing notification targets.
> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to
> _connection_string_ before starting SILO. On the old server, use the following command to update the target.
>
> ```
> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 username=psqluser password=psqlpass database=bucketevents"
> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 user=psqluser password=psqlpass dbname=bucketevents"
> ```
>
> Please make sure this step is carried out, without this step PostgreSQL notification targets will not work,
@@ -973,9 +972,8 @@ key | value
> database (string) database name (used only if `dsn_string` is empty)
> ```
>
> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure
> to migrate to only using _dsn_string_ option. To migrate, once you have upgraded all the servers use the
> following command to update the existing notification targets.
> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to
> _dsn_string_ before starting SILO. On the old server, use the following command to update the target.
>
> ```
> mc admin config set mysilo/ notify_mysql[:name] dsn_string="mysqluser:mysqlpass@tcp(localhost:2832)/bucketevents"
@@ -1045,7 +1043,7 @@ Before updating the configuration, let's start with `mc admin config get` comman
```sh
$ mc admin config get mysilo/ notify_mysql
notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0
notify_mysql:myinstance enable=off format=namespace dsn_string= table= queue_dir= queue_limit=0
```
Use `mc admin config set` command to update MySQL notification configuration for the deployment with `dsn_string` parameter:
docs/bucket/replication/README.md
+10
View File
@@ -96,6 +96,12 @@ The access key provided for the replication *target* cluster should have these m
Please note that the permissions required by the admin user on the target cluster can be more fine grained to exclude permissions like "s3:ReplicateDelete", "s3:GetBucketObjectLockConfiguration" etc depending on whether delete replication rules are set up or if object locking is disabled on `destbucket`. The above policies assume that replication of objects, tags and delete marker replication are all enabled on object lock enabled buckets. A sample script to setup replication is provided [here](https://github.com/pgsty/silo/blob/main/docs/bucket/replication/setup_replication.sh)
The target replication credential continues to authorize replicated deletes with
`s3:DeleteObject` plus `s3:ReplicateDelete`; it does not need
`s3:DeleteObjectVersion`. This internal receiver contract is deliberately
separate from ordinary S3 requests: a client deleting an explicitly named
version, including `versionId=null`, must have `s3:DeleteObjectVersion`.
To set up replication from `srcbucket` on the `mysilo` cluster to `destbucket`
on a target Silo cluster at `https://replica-endpoint:9000`, use:
```
@@ -200,6 +206,10 @@ To add a replication rule allowing both delete marker replication, versioned del
Additional permission of "s3:ReplicateDelete" action would need to be specified on the access key configured for the target cluster if Delete Marker replication or versioned delete replication is enabled.
An explicit deny on `s3:DeleteObjectVersion` still blocks the corresponding
replicated version purge. An allow is not otherwise required for the target
replication credential.
```
mc replicate add mysilo/srcbucket/Tax --priority 1 --remote-bucket `remote-target` --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --replicate "delete,delete-marker"
Replication configuration applied successfully to mysilo/srcbucket.
docs/bucket/replication/delete-replication.sh
+72 -6
View File
@@ -88,11 +88,7 @@ echo "=== mysilo2"
versionId="$(./mc ls --json --versions mysilo1/testbucket/dir/ | tail -n1 | jq -r .versionId)"
export AWS_ACCESS_KEY_ID=minioadmin
export AWS_SECRET_ACCESS_KEY=minioadmin
export AWS_REGION=us-east-1
aws s3api --endpoint-url http://localhost:9001 delete-object --bucket testbucket --key dir/file --version-id "$versionId"
./mc rm --version-id "$versionId" mysilo1/testbucket/dir/file
./mc ls -r --versions mysilo1/testbucket >/tmp/mysilo1.txt
./mc ls -r --versions mysilo2/testbucket >/tmp/mysilo2.txt
@@ -117,6 +113,76 @@ if [ $ret -ne 0 ]; then
exit 1
fi
# Verify the documented least-privilege target policy. Explicit version
# deletion on the receiver is replication traffic, so the target credential
# needs DeleteObject + ReplicateDelete but not DeleteObjectVersion.
./mc mb mysilo1/leastpriv/ mysilo2/leastpriv/ --with-versioning
./mc admin user add mysilo2 repluser repluser123
cat >/tmp/xl/replpolicy.json <<'EOF'
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetReplicationConfiguration",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:GetBucketLocation",
"s3:GetBucketVersioning"
],
"Resource": ["arn:aws:s3:::leastpriv"]
},
{
"Effect": "Allow",
"Action": [
"s3:GetReplicationConfiguration",
"s3:ReplicateTags",
"s3:AbortMultipartUpload",
"s3:GetObject",
"s3:GetObjectVersion",
"s3:GetObjectVersionTagging",
"s3:PutObject",
"s3:DeleteObject",
"s3:ReplicateObject",
"s3:ReplicateDelete"
],
"Resource": ["arn:aws:s3:::leastpriv/*"]
}
]
}
EOF
./mc admin policy create mysilo2 replpolicy /tmp/xl/replpolicy.json
./mc admin policy attach mysilo2 replpolicy --user repluser
./mc replicate add mysilo1/leastpriv --remote-bucket http://repluser:repluser123@localhost:9002/leastpriv/ --priority 1 --replicate delete,delete-marker
./mc cp README.md mysilo1/leastpriv/dir/file
./mc cp README.md mysilo1/leastpriv/dir/file
sleep 1s
leastPrivVersionId="$(./mc ls --json --versions mysilo1/leastpriv/dir/ | tail -n1 | jq -r .versionId)"
./mc rm --version-id "$leastPrivVersionId" mysilo1/leastpriv/dir/file
sleep 1s
./mc ls -r --versions mysilo1/leastpriv >/tmp/leastpriv1.txt
./mc ls -r --versions mysilo2/leastpriv >/tmp/leastpriv2.txt
out=$(diff -qpruN /tmp/leastpriv1.txt /tmp/leastpriv2.txt)
ret=$?
if [ $ret -ne 0 ]; then
echo "BUG: least-privilege version delete did not replicate: $out"
exit 1
fi
./mc rm mysilo1/leastpriv/dir/file
sleep 1s
./mc ls -r --versions mysilo1/leastpriv >/tmp/leastpriv1.txt
./mc ls -r --versions mysilo2/leastpriv >/tmp/leastpriv2.txt
out=$(diff -qpruN /tmp/leastpriv1.txt /tmp/leastpriv2.txt)
ret=$?
if [ $ret -ne 0 ]; then
echo "BUG: least-privilege delete marker did not replicate: $out"
exit 1
fi
# Test listing of non replicated permanent deletes
set -x
@@ -129,7 +195,7 @@ versionId="$(./mc ls --json --versions mysilo1/foobucket/dir/ | jq -r .versionId
kill ${pid2} && wait ${pid2} || true
aws s3api --endpoint-url http://localhost:9001 delete-object --bucket foobucket --key dir/file --version-id "$versionId"
./mc rm --version-id "$versionId" mysilo1/foobucket/dir/file
out="$(./mc ls mysilo1/foobucket/dir/)"
if [ "$out" != "" ]; then
docs/compression/README.md
+7
View File
@@ -86,6 +86,13 @@ To enable compression+encryption use:
Or alternatively through the environment variable `MINIO_COMPRESSION_ALLOW_ENCRYPTION=on`.
SSE-C objects are excluded from compression even with `allow_encryption=on`.
Replication ships an SSE-C object as raw ciphertext, because the server never holds the
customer key, and the compression metadata is not carried over the wire. A compressed
SSE-C object would therefore replicate to a replica that decrypts to a compressed stream.
`allow_encryption` still applies to SSE-S3 and SSE-KMS, where the server owns the key and
decompresses before replicating.
### 4. Excluded Types
- Already compressed objects are not fit for compression since they do not have compressible patterns.
docs/debugging/inspect/export.go
+2 -2
View File
@@ -358,7 +358,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) {
e = msgp.ArrayError{Wanted: 5, Got: zb0001}
return o, e
}
bts, e = msgp.ReadExactBytes(bts, (z.VersionID)[:])
bts, e = msgp.ReadExactBytes(bts, z.VersionID[:])
if e != nil {
e = msgp.WrapError(e, "VersionID")
return o, e
@@ -368,7 +368,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) {
e = msgp.WrapError(e, "ModTime")
return o, e
}
bts, e = msgp.ReadExactBytes(bts, (z.Signature)[:])
bts, e = msgp.ReadExactBytes(bts, z.Signature[:])
if e != nil {
e = msgp.WrapError(e, "Signature")
return o, e
docs/debugging/reorder-disks/go.mod
+2 -4
View File
@@ -1,7 +1,5 @@
module github.com/minio/minio/docs/debugging/reorder-disks
go 1.21
go 1.26.0
toolchain go1.24.8
require github.com/minio/pkg/v3 v3.0.1
require github.com/pgsty/silo-pkg/v3 v3.13.2
docs/debugging/reorder-disks/go.sum
+2 -2
View File
@@ -1,2 +1,2 @@
github.com/minio/pkg/v3 v3.0.1 h1:qts6g9rYjAdeomRdwjnMc1IaQ6KbaJs3dwqBntXziaw=
github.com/minio/pkg/v3 v3.0.1/go.mod h1:53gkSUVHcfYoskOs5YAJ3D99nsd2SKru90rdE9whlXU=
github.com/pgsty/silo-pkg/v3 v3.13.2 h1:Clw11c/J54Tx6pijNCWtXiC7e0fwP/f5Tgeb6fsXg2w=
github.com/pgsty/silo-pkg/v3 v3.13.2/go.mod h1:0GmaDA0ArQ8bkAI/obiSNTBQzdgBu6W0e0olDCbyXXo=
docs/debugging/reorder-disks/main.go
+1 -1
View File
@@ -30,7 +30,7 @@ import (
"strings"
"syscall"
"github.com/minio/pkg/v3/ellipses"
"github.com/pgsty/silo-pkg/v3/ellipses"
)
type xl struct {
docs/debugging/xl-meta/main.go
+2 -2
View File
@@ -745,7 +745,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt
err = msgp.ArrayError{Wanted: want, Got: zb0001}
return o, err
}
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
bts, err = msgp.ReadExactBytes(bts, z.VersionID[:])
if err != nil {
err = msgp.WrapError(err, "VersionID")
return o, err
@@ -755,7 +755,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt
err = msgp.WrapError(err, "ModTime")
return o, err
}
bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
bts, err = msgp.ReadExactBytes(bts, z.Signature[:])
if err != nil {
err = msgp.WrapError(err, "Signature")
return o, err
docs/metrics/healthcheck/README.md
+7 -1
View File
@@ -50,7 +50,7 @@ livenessProbe:
## Readiness probe
This probe always responds with '200 OK'. Only fails if 'etcd' is configured and unreachable. When readiness probe fails, Kubernetes like platforms turn-off routing to the container.
This probe responds with '200 OK' once the server process is up: it fails only while the request queue is overloaded, or when the health operation against the configured KMS or 'etcd' fails. When readiness probe fails, Kubernetes like platforms turn-off routing to the container.
```
readinessProbe:
@@ -120,3 +120,9 @@ X-Xss-Protection: 1; mode=block
X-Minio-Write-Quorum: 3
Date: Tue, 21 Jul 2020 00:35:43 GMT
```
## Startup readiness window
None of the probes above, and no `admin info` view, proves that the node which received the request can already serve the data path after a restart. Each node connects its erasure drives to its peers in a monitor loop: a remote drive that could not be connected during startup stays uninstalled on that node until a later pass, and the monitor waits 15 seconds after each completed pass, so that interval is a floor between attempts, not a bound on recovery. While a drive is uninstalled, the node's own liveness and readiness probes answer '200 OK', the cluster probes can report healthy as well, because they aggregate every peer's report of its own local drives rather than the drives this node has installed, and `mcli ready` inherits the same blind spot. Yet a PUT through that node can fail with '503 SlowDownWrite' for lack of write quorum, and a GET through it of an object that another node just wrote can answer '404 NoSuchKey'. In the review runs that established this, sampled I/O began succeeding roughly 13 to 15 seconds after the administrative views became healthy on a four-node loopback cluster, consistent with the reconnect interval, and every object acknowledged by other nodes during the window was readable afterwards; these are observations, not guarantees, since reconnection can keep failing.
Automation that restarts a cluster and then immediately writes to it, such as an upgrade or failover runbook, should therefore gate on a bounded data-path check rather than on these probes: one small PUT through each node followed by a read of each object through every node, repeated until every request returns the correct bytes and the acknowledged version within one fixed deadline, with each request budgeted from the remaining deadline and SDK retries disabled, and with the acknowledged objects re-read afterwards. Record the time to first usable I/O separately from the probe result. Such a check proves sampled I/O at that moment for the erasure sets those keys hash to; it proves neither that every set is complete nor that there is headroom for a further node loss, since a set can admit writes with fewer than all of its drives installed. The probes remain the right signal for their stated purpose, process liveness and quorum membership, and are unchanged.
docs/security/advisories.md
+22 -4
View File
@@ -1,9 +1,17 @@
# pgsty/minio Security Advisories
# pgsty/silo Security Advisories
This document summarizes fork-specific security fixes and closely related upgrade-impacting security notes in `pgsty/minio`. It is intentionally narrower than a full changelog and focuses on release-impacting security behavior.
This document summarizes fork-specific security fixes and closely related upgrade-impacting security notes in `pgsty/silo`. It is intentionally narrower than a full changelog and focuses on release-impacting security behavior.
Entries carry a CVE identifier where one exists. Where none does, they carry a fork-local `SN-<year>-<sequence>` identifier so that a finding without a CVE can still be referenced stably from release notes, commits and issues. An `SN-` identifier is **not** a CVE and is not registered in any vulnerability database; it is deliberately not written in CVE form so that scanners do not mistake it for one. Upstream `minio/minio` is archived, so for findings in inherited code there is no upstream maintainer to coordinate a CVE assignment with. `SN-2026-001` is the streaming-flush regression in `trackingResponseWriter`, which is a reliability defect rather than a security one and is tracked in the release notes rather than here.
## Inherited upstream advisory baseline
The first Silo community release was cut from upstream history that already contained the following security fix. Upstream and Silo links are both recorded even when the fork preserves the same commit object and SHA; that identity is the inheritance evidence, not a claim that Silo independently reimplemented the patch.
| ID | Upstream remediation | Silo inheritance | Regression evidence | Release / operator note |
| :-- | :-- | :-- | :-- | :-- |
| [CVE-2025-62506](https://github.com/advisories/GHSA-jjjj-jwhf-8rgr) | [minio/minio#21642](https://github.com/minio/minio/pull/21642), merged as [`c1a49490`](https://github.com/minio/minio/commit/c1a49490c78e9c3ebcad86ba0662319138ace190) | The same commit object is present as [`pgsty/silo@c1a49490`](https://github.com/pgsty/silo/commit/c1a49490c78e9c3ebcad86ba0662319138ace190) | The inherited [service-account](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/admin-handlers-users_test.go#L211-L212) and [STS](https://github.com/pgsty/silo/blob/c1a49490c78e9c3ebcad86ba0662319138ace190/cmd/sts-handlers_test.go#L45-L46) regression groups run for root and non-root parents through `go test ./cmd` | Resets `DenyOnly` while evaluating a restricted session policy so service or STS accounts cannot mint an unrestricted child service account. Upstream first fixed this in [`RELEASE.2025-10-15T17-29-55Z`](https://github.com/minio/minio/releases/tag/RELEASE.2025-10-15T17-29-55Z); every Silo community release, beginning with [`RELEASE.2025-12-03T12-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2025-12-03T12-00-00Z), contains it. Operators migrating from an older upstream build should upgrade and audit service accounts created by restricted service or STS identities. |
## Advisories since `RELEASE.2026-03-21T00-00-00Z`
| ID | Fixed by | Affected area | Remote exploitability | Summary | Upgrade / workaround notes |
@@ -16,9 +24,15 @@ Entries carry a CVE identifier where one exists. Where none does, they carry a f
| [CVE-2026-40344](https://github.com/advisories/GHSA-9c4q-hq6p-c237) | `efb6e5b00` | Snowball auto-extract authentication | Yes | Verifies request authentication before tar extraction in Snowball unsigned-trailer flows | Upgrade if you use `PutObjectExtract` or Snowball uploads. |
| [CVE-2026-42600](https://github.com/advisories/GHSA-xh8f-g2qw-gcm7) | `73ac52472` | Internode `ReadMultiple` storage-REST endpoint | Yes (cluster-root JWT required) | Removes the unused endpoint that allowed path traversal outside configured drive roots | Upgrade distributed-erasure deployments. Single-node deployments do not register this route. |
| `SN-2026-002` | `ca7baa670` and follow-ups | Internode storage-REST and Grid RPC payloads | Yes (cluster-root / internode JWT required) | Completes CVE-2026-42600. Its fix removed one endpoint that exercised the gap; the gap itself -- request bodies and grid frames never reaching the validity middleware, and no containment in the storage layer -- remained across three further protocol surfaces. Closes path traversal on both the volume and path axes (including the peer-S3 bucket RPCs, which bypass the storage-REST wrapper entirely), an unrecoverable divide-by-zero that killed a node per RPC frame, metadata that reported truncated shards as intact, and three allocations sized from caller-declared values. | Upgrade distributed-erasure deployments. Single-node deployments register none of these routes. No S3 API behaviour changes; object keys containing `.` or `..` path segments were already refused at the S3 boundary. |
| `SN-2026-003` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`2f55347f7`](https://github.com/pgsty/minio/commit/2f55347f78352aed8e08866d370c9426c73362cf) | S3/IAM bucket-policy condition values | Yes (policy-dependent) | Prevents raw request entries that spell condition-key names from shadowing or synthesizing internal condition values; confines `s3:signatureAge` to verified SigV4 presigned requests; separates query-only list fields from header-backed `x-amz-*` fields; and stops client request tags from impersonating stored existing-object tags. | The compatible query form remains for storage class and upload tagging on handlers that consume it; an explicitly present header wins, including an empty header. The historical `X-Amz-Tagging` Header mapping remains a client-supplied `RequestObjectTag` source, so use request-tag conditions only on operations that consume tags. Header-only `x-amz-*` policy keys no longer accept query substitutes. `aws:SourceIp` was left following the existing forwarding-header trust model; that model is addressed separately in the next row. See [Condition value sources and precedence](https://silo.pgsty.com/administration/identity-access-management/policy-based-access-control/#condition-value-sources). |
| `SN-2026-003` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`2f55347f7`](https://github.com/pgsty/silo/commit/2f55347f78352aed8e08866d370c9426c73362cf) | S3/IAM bucket-policy condition values | Yes (policy-dependent) | Prevents raw request entries that spell condition-key names from shadowing or synthesizing internal condition values; confines `s3:signatureAge` to verified SigV4 presigned requests; separates query-only list fields from header-backed `x-amz-*` fields; and stops client request tags from impersonating stored existing-object tags. | The compatible query form remains for storage class and upload tagging on handlers that consume it; an explicitly present header wins, including an empty header. The historical `X-Amz-Tagging` Header mapping remains a client-supplied `RequestObjectTag` source, so use request-tag conditions only on operations that consume tags. Header-only `x-amz-*` policy keys no longer accept query substitutes. `aws:SourceIp` was left following the existing forwarding-header trust model; that model is addressed separately in the next row. See [Condition value sources and precedence](https://silo.pgsty.com/administration/identity-access-management/policy-based-access-control/#condition-value-sources). |
| Not a vulnerability | `fe6dc4780` | Client source address (`aws:SourceIp`, audit `remotehost`, event notification `Host`) | N/A -- opt-in hardening | Adds an enforceable forwarded-header trust boundary, `MINIO_API_TRUSTED_PROXIES`. Set to a list of addresses or CIDR blocks, forwarded headers are believed only from those peers and forwarding chains are read right-to-left past listed hops -- which also stops the client-supplied left-most entry that an appending proxy (the stock nginx `$proxy_add_x_forwarded_for` recipe, or HAProxy's added second header line) leaves in place. Set to `none`, no forwarded header is believed at all. This is the guarantee `_MINIO_API_XFF_HEADER=off` never provided: it suppresses `X-Forwarded-For` alone, so `X-Real-IP` and RFC 7239 `Forwarded` remain one-line substitutions for anyone that setting was meant to stop. | **No behaviour change for any existing deployment**, so there is nothing to do on upgrade unless you want the new boundary. Not assigned a CVE: the default matches upstream, and upstream's own position (maintainer response in [discussion #17878](https://github.com/minio/minio/discussions/17878), Aug 2023) is that IP-based restrictions are impractical without reliable source-IP visibility. The gap being closed is that this was never written anywhere an operator would find it -- an `IpAddress` condition is accepted and behaves as though it works. **If you use `IpAddress` or `NotIpAddress` conditions, note that they were not enforceable before this change**, including behind a reverse proxy whose `X-Forwarded-For` recipe appends rather than overwrites. If you do not, the change affects only the accuracy of client addresses in logs. The new variable is opt-in and inert when unset; `_MINIO_API_XFF_HEADER` keeps its exact upstream semantics, and upstream's `TestXFFDisabled` is retained unmodified as the proof. An `IpAddress` condition remains unenforceable by default against a client with direct network access to the API port -- that is the condition the allowlist exists to fix, not a regression introduced here. When enabling the allowlist: it must name proxies, not the subnet they sit in, because entries are skipped while walking the chain, so a range that also covers clients lets those clients forge. Multi-node deployments must include their own node addresses, since MinIO forwards some requests between nodes and a client can force a hop through the `ListObjectsV2` continuation token; prefer the allowlist over `none` on a cluster for that reason. Loopback is always trusted as a peer so FTP and SFTP keep attributing their sessions. A malformed value stops startup, as does one that names no proxy at all (`","`) or one whose `env://` remote could not be read -- `env.Get` discards that error and yields an empty string, which would otherwise read as unset. Whitespace-only remains equivalent to unset. The policy is read after `MINIO_CONFIG_ENV_FILE` is loaded so environment-file deployments are covered; `_MINIO_API_XFF_HEADER` deliberately keeps upstream's earlier read timing, where a value written into an environment file is ignored. `MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES` now shares the same list parser, but is behaviourally untouched: the extraction is pure code motion, verified identical to the previous implementation across every combination of 37 allowlist values and 21 peer addresses. See [Client source address trust](source-address-trust.md). |
| `SN-2026-004` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`97b7d2804`](https://github.com/pgsty/minio/commit/97b7d28040d109061c0a46a4c01bfc7800a97cc1) | IAM policy evaluation of bucket-level actions | Yes (policy-dependent) | Withholds twelve sensitive bucket-level writes from an object-only resource pattern. The IAM matcher appended a trailing slash for bucket-level requests (empty object name), so a resource of `arn:aws:s3:::bucket/*` matched `"bucket/"` and authorized bucket-level actions it was never meant to reach -- upstream [minio/minio#20449](https://github.com/minio/minio/issues/20449). The bucket-policy evaluation path never had the slash and was already reference-correct. | **This is an authorization tightening; read this row before upgrading if you write your own bucket-scoped policies.** Withheld from `bucket/*` on `Allow` statements only: `PutBucketPolicy`, `DeleteBucketPolicy`, `PutBucketObjectLockConfiguration`, `PutBucketVersioning`, `PutReplicationConfiguration`, `PutBucketLifecycle`, `DeleteBucket`, `ForceDeleteBucket`, `PutBucketCors`, `DeleteBucketCors`, `PutBucketQOS`, `PutInventoryConfiguration`. Membership was decided by one question -- does reaching this action give the caller something its object-scoped grant does not already give it? -- because the bug only fires when the statement already grants the bucket action, which in practice means `s3:*`, so the affected principal already holds full object CRUD. Only actions that hand out access to others, defeat a protection aimed at write-holders, act under server credentials, outlive the grant, or destroy the bucket entity qualify. **Deliberately not withheld, and asserted by test so re-adding one is a deliberate act**: `ListBucket`, `GetBucketLocation` and the read/list family, `PutBucketTagging`, `PutBucketEncryption`, `PutBucketNotification`, and `CreateBucket` -- so `mc ls`, SDK session setup and ordinary tenant self-service keep working through `bucket/*`. Breaking those is what got upstream's own full fix reverted. **What to change**: add the bare bucket ARN (`arn:aws:s3:::bucket`) alongside `arn:aws:s3:::bucket/*` in any statement that legitimately grants one of the twelve. Built-in canned policies are unaffected (all use `Resource: "*"`). `Deny` statements are untouched, so no bucket lock is ever weakened, and `NotResource` exclusions keep their full reach. The hardening is monotone by construction rather than by argument: the protected path requires **both** the bare and the historical `"bucket/"` form to match, an intersection with the historical decision -- without that, a fixed-width wildcard such as `mybucke?` would match `"mybucket"` while never having matched `"mybucket/"`, and the hardening would have granted a write the buggy matcher refused. `MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH=on` restores the historical behaviour in full; it is read once at startup. Still deferred to a migration-gated release: the read/list family, a startup audit naming affected policies, and a self-explaining denial log. |
| `SN-2026-004` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`97b7d2804`](https://github.com/pgsty/silo/commit/97b7d28040d109061c0a46a4c01bfc7800a97cc1) | IAM policy evaluation of bucket-level actions | Yes (policy-dependent) | Withholds twelve sensitive bucket-level writes from an object-only resource pattern. The IAM matcher appended a trailing slash for bucket-level requests (empty object name), so a resource of `arn:aws:s3:::bucket/*` matched `"bucket/"` and authorized bucket-level actions it was never meant to reach -- upstream [minio/minio#20449](https://github.com/minio/minio/issues/20449). The bucket-policy evaluation path never had the slash and was already reference-correct. | **This is an authorization tightening; read this row before upgrading if you write your own bucket-scoped policies.** Withheld from `bucket/*` on `Allow` statements only: `PutBucketPolicy`, `DeleteBucketPolicy`, `PutBucketObjectLockConfiguration`, `PutBucketVersioning`, `PutReplicationConfiguration`, `PutBucketLifecycle`, `DeleteBucket`, `ForceDeleteBucket`, `PutBucketCors`, `DeleteBucketCors`, `PutBucketQOS`, `PutInventoryConfiguration`. Membership was decided by one question -- does reaching this action give the caller something its object-scoped grant does not already give it? -- because the bug only fires when the statement already grants the bucket action, which in practice means `s3:*`, so the affected principal already holds full object CRUD. Only actions that hand out access to others, defeat a protection aimed at write-holders, act under server credentials, outlive the grant, or destroy the bucket entity qualify. **Deliberately not withheld, and asserted by test so re-adding one is a deliberate act**: `ListBucket`, `GetBucketLocation` and the read/list family, `PutBucketTagging`, `PutBucketEncryption`, `PutBucketNotification`, and `CreateBucket` -- so `mc ls`, SDK session setup and ordinary tenant self-service keep working through `bucket/*`. Breaking those is what got upstream's own full fix reverted. **What to change**: add the bare bucket ARN (`arn:aws:s3:::bucket`) alongside `arn:aws:s3:::bucket/*` in any statement that legitimately grants one of the twelve. Built-in canned policies are unaffected (all use `Resource: "*"`). `Deny` statements are untouched, so no bucket lock is ever weakened, and `NotResource` exclusions keep their full reach. The hardening is monotone by construction rather than by argument: the protected path requires **both** the bare and the historical `"bucket/"` form to match, an intersection with the historical decision -- without that, a fixed-width wildcard such as `mybucke?` would match `"mybucket"` while never having matched `"mybucket/"`, and the hardening would have granted a write the buggy matcher refused. `MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH=on` restores the historical behaviour in full; it is read once at startup. Still deferred to a migration-gated release: the read/list family, a startup audit naming affected policies, and a self-explaining denial log. |
| `SN-2026-005` | [`silo-pkg v3.12.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.12.0) and [`eee05a17c`](https://github.com/pgsty/silo/commit/eee05a17c34a07cebb27220d12697be74c8bd617) | IAM named-policy and service-account policy writes | No direct remote exploit; policy-dependent | Rejects S3, S3 Tables, and KMS ARN namespace prefixes that name no resource, including their historical `*arn:...` serialization, in both `Resource` and `NotResource`. A resource-matching `Deny` using such a prefix could silently fail to deny, while an `Allow` with the prefix in `NotResource` could match far more broadly than intended. The guard applies when creating named policies and when creating or updating service-account session policies. | **This is an authorization tightening for new and updated policies.** Existing policies keep loading, matching, importing, and replicating with unchanged runtime behavior, but a policy containing one of these prefixes cannot be submitted unchanged; replace it with the intended concrete resource, or use an explicit wildcard such as `arn:aws:s3:::*` only when all resources are intended. Enabling the strict path also rejects an admin statement that combines `Resource` with `NotResource`, and rejects non-S3 resources on bucket-scoped admin actions. Here “bare ARN prefix” means a namespace with no resource after it (`arn:aws:s3:::`); it is distinct from the valid “bare bucket ARN” in `SN-2026-004` (`arn:aws:s3:::bucket`). IAM import, site-replication receive paths, stored-policy loading, and STS inline policies remain on the permissive compatibility path in this release. |
| `SN-2026-006` | [`b73581b05`](https://github.com/pgsty/silo/commit/b73581b05) and [`c4fd97d0b`](https://github.com/pgsty/silo/commit/c4fd97d0b) ([#82](https://github.com/pgsty/silo/issues/82)) | SSE-C reads of zero-byte objects (`GetObject`, `HeadObject`, `CopyObject` source, `GetObjectAttributes`) | Yes; requires read access to the object | Zero-byte SSE-C objects never unsealed the customer-provided key, so a wrong key was accepted with `200` instead of `403`, and a copy or new version could be created under a key of the caller's choosing without knowing the current one. | Wrong keys now fail with `403 AccessDenied` as on AWS; correct keys behave as before and no client change is needed. Inherited from upstream; every earlier release is affected. |
| `SN-2026-007` | [`474cd5801`](https://github.com/pgsty/silo/commit/474cd5801), [`74c97d005`](https://github.com/pgsty/silo/commit/74c97d005), [`21870fa2e`](https://github.com/pgsty/silo/commit/21870fa2e) ([#84](https://github.com/pgsty/silo/issues/84)) | `GetObjectAttributes` on SSE-C objects | Yes; requires read access to the object | Attributes of SSE-C objects were returned without authenticating the customer key, and a bare `X-Minio-Source-Replication-Request` header skipped the check entirely. | A wrong key returns `403`, a replication marker without the key returns `400`; replication peers holding `s3:ReplicateObject` are unaffected. Inherited from upstream. |
| `SN-2026-008` | [PR #101](https://github.com/pgsty/silo/pull/101) ([`938603458`](https://github.com/pgsty/silo/commit/938603458) through [`04b097fd9`](https://github.com/pgsty/silo/commit/04b097fd9)) | Internal replication request headers such as `X-Minio-Source-Etag`, `X-Minio-Source-Mtime`, `X-Minio-Source-Replication-Request`, the replication SSE key headers, and `X-Amz-Bucket-Replication-Status` on object reads, writes, multipart uploads, deletes, Snowball extraction, and bucket events | Yes; any authenticated principal that can read or write the object | Completes CVE-2026-34204. The server still trusted these internal headers on presence in most handlers: any client could preserve arbitrary ETags and modification times, read SSE-C ciphertext without the key, inject replication checksums and Object Lock timestamps, suppress bucket notifications, and route deletes as replication deletes. | Replication semantics now require the exact marker value together with `s3:ReplicateObject` or `s3:ReplicateDelete`; other requests have these headers removed after signature verification and are processed as ordinary requests. Site replication service accounts and bucket-replication targets that already hold the replication permissions are unaffected. Inherited from upstream. |
| `SN-2026-009` | [`58735ee38`](https://github.com/pgsty/silo/commit/58735ee38) and [`229fe2b3c`](https://github.com/pgsty/silo/commit/229fe2b3c) ([PR #73](https://github.com/pgsty/silo/pull/73)) | Admin `SetUserStatus` and `SetGroupStatus` | Yes; authenticated admin API | Status changes were authorized against `admin:EnableUser` / `admin:EnableGroup` regardless of the requested status, so a principal allowed only to enable could also disable, and vice versa. | Enable and disable now require the action matching the target status. Policies that grant only one of the pair lose the other operation; `admin:*` and the built-in `consoleAdmin` policy are unaffected. Inherited from upstream. |
| `SN-2026-010` | [PR #104](https://github.com/pgsty/silo/pull/104) ([`75a6734e4`](https://github.com/pgsty/silo/commit/75a6734e4) through [`d2d47a41f`](https://github.com/pgsty/silo/commit/d2d47a41f), [#58](https://github.com/pgsty/silo/issues/58)) | `DeleteObject` and `DeleteObjects` with an explicit `versionId` | Yes; authenticated S3 API | Explicit version deletes were authorized as `s3:DeleteObject` with only a deny check on `s3:DeleteObjectVersion`, diverging from AWS. | Explicit version deletes now require `s3:DeleteObjectVersion`, as on AWS. **Two policy effects:** principals granted only `s3:DeleteObject` can no longer delete specific versions, and a policy that relied on `Deny s3:DeleteObject` to block permanent deletes must also deny `s3:DeleteObjectVersion`, because `Allow s3:*` now permits explicit version deletes. Replication targets keep the `s3:ReplicateDelete` contract. Inherited from upstream. |
## Dependency security updates
@@ -27,11 +41,15 @@ Entries carry a CVE identifier where one exists. Where none does, they carry a f
| `CVE-2026-34986` | `68e0ba997` | Upgrades `go-jose` to `v4.1.4`. |
| `CVE-2026-39883` | `1869bd30b`, `e4fa06394` | Updates OpenTelemetry dependencies. |
| Upstream Go security fixes | [Go 1.26.5](https://go.dev/doc/devel/release#go1.26.5) | Bumps the required toolchain to Go 1.26.5, which includes security fixes to `crypto/tls` and `os`. |
| Toolchain and dependency refresh | [Go 1.27.1](https://go.dev/doc/devel/release#go1.27.1) via [`43f4bb7ed`](https://github.com/pgsty/silo/commit/43f4bb7ed), [`edc8be6ed`](https://github.com/pgsty/silo/commit/edc8be6ed), [`4d6e1ea8e`](https://github.com/pgsty/silo/commit/4d6e1ea8e) | Moves the toolchain to Go 1.27 (1.27.1 as of the release) and refreshes the dependency stack (etcd client v3.7.1, `jwx` v3.0.13, `klauspost/compress` v1.19.2). The pre-release cleanup then returns to upstream `minio-go` (v7.3.1 pre-release) and retires the `silo-go` fork; `govulncheck` reports no reachable vulnerability on the release candidate. |
| [GO-2026-6354](https://pkg.go.dev/vuln/GO-2026-6354) / [GO-2026-6355](https://pkg.go.dev/vuln/GO-2026-6355) | `golang.org/x/crypto` `v0.56.0` ([`edf36bcbf`](https://github.com/pgsty/silo/commit/edf36bcbf)) | Updates `x/crypto/ssh` to the first fixed version for denial of service on deadlocked undecided and established channels. Reachable through the SFTP server (`startSFTPServer` → `sftp.Server.Listen` → `ssh.NewServerConn`); every earlier release that enables SFTP is affected. |
| [GO-2026-6061](https://pkg.go.dev/vuln/GO-2026-6061) / [GHSA-hrxh-6v49-42gf](https://github.com/advisories/GHSA-hrxh-6v49-42gf) | gRPC `v1.82.1` | Updates gRPC to the first fixed version for vulnerabilities in the xDS RBAC authorization engine and HTTP/2 transport server. |
| [CVE-2026-84304](https://github.com/advisories/GHSA-vp52-pcj8-j9qc) | gRPC `v1.83.1` | Updates gRPC-Go to the first fixed version for unauthenticated heap exhaustion through highly fragmented HTTP/2 DATA frames. Silo pulls gRPC transitively rather than registering a gRPC server itself, but selects the fixed version for the complete module graph. |
| [GO-2026-5970](https://pkg.go.dev/vuln/GO-2026-5970) / `CVE-2026-56852` | `x/text` `v0.39.0` | Updates `x/text` to the first fixed version for an infinite loop on invalid input. |
## Operationally significant security-related fixes
| Change | Fixed by | Summary |
| :-- | :-- | :-- |
| Replicated Object Lock updates ignored their timestamps | pre-release cleanup for the release after 20260806 | A replicated `CopyObject` rebuilt the metadata from the request before comparing replication timestamps, so the stored retention and legal-hold timestamps were never seen: any replica update was applied regardless of order, and the legal-hold timestamp was written under the retention key. A stale replica could therefore turn a newer legal hold off or shorten a newer retention. The stored state is now captured first, a replica update is applied only when its timestamp is newer, a stale one leaves the stored state in place, and each timestamp is kept under its own key. Inherited from upstream; every earlier release is affected. |
| LDAP TLS regression | `ce1c537eb` | Restores TLS configuration propagation for `ldaps://` `DialURL()` connections so `MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY` and custom root CAs work again. |
docs/site-replication/CORS-LWW-DESIGN.md
+503
View File
@@ -0,0 +1,503 @@
# Per-Bucket CORS Site-Replication Convergence Design
## Status
- Issue: [pgsty/silo#75](https://github.com/pgsty/silo/issues/75), closed; follow-ups
[#77](https://github.com/pgsty/silo/issues/77) and [#102](https://github.com/pgsty/silo/issues/102)
- Merged: [PR #80](https://github.com/pgsty/silo/pull/80) implemented this register
(2026-08-29); [PR #101](https://github.com/pgsty/silo/pull/101) restricted the
pre-authentication lookup to resident metadata; [PR #103](https://github.com/pgsty/silo/pull/103)
replaced the CORS-specific lock with the shared `metadata.lock`
- Release state: on `main`, not yet in a tagged release as of 2026-09-02
This document defines the replication state, ordering, persistence, status,
healing, concurrency, compatibility, and test contract for per-bucket CORS.
It is an implementation design record, not public upgrade or rollback guidance.
Public operator documentation belongs in the separate `silo.pgsty.com`
repository.
## Scope
This design covers the current-version CORS path:
```text
PutBucketCors / DeleteBucketCors
-> persist local CORS state
-> BucketMetaHook
-> madmin SRBucketMeta transport
-> SRPeerReplicateBucketItem dispatch
-> PeerBucketCorsConfigHandler
-> SiteReplicationMetaInfo
-> siteReplicationStatus
-> latestCORSConfig
-> healCORSMetadata
```
It also covers retry, duplicate delivery, reordering, equal timestamps,
initial site sync, missed DELETE recovery, cache reload, process restart, and
concurrent CORS mutations on different nodes of one cluster.
The following are deliberately out of scope:
- redesigning the replication semantics of policy, tags, SSE, quota,
versioning, or Object Lock;
- changing higher-level lifecycle merge semantics or serializing bucket
deletion against in-flight metadata updates; those follow-ups remain under
[pgsty/silo#102](https://github.com/pgsty/silo/issues/102);
- mixed-version support that permits CORS writes before every site runs a
CORS-aware binary;
- public downgrade, rollback, and global-fallback documentation;
- Console UI for bucket CORS.
### Adjacent issue-75 changes in the same candidate
The final issue-75 candidate also contains CORS work outside the LWW register
itself:
- a strict, namespace-tolerant XML wire parser that rejects trailing roots,
unknown/nested elements, duplicate singleton fields, invalid integer shape,
and non-whitespace character data;
- Unicode code-point ID counting, exact uppercase S3 methods, non-empty header
elements, and int32-compatible MaxAge validation;
- a single-`*` matcher and response-selection changes needed to distinguish a
literal `*` origin from a patterned or explicit `null` match;
- fail-closed metadata-error handling in the HTTP middleware;
- complete allowed-method, explicit MaxAge=0, expose-header, credentials, and
`Vary` preflight behavior;
- checksum mismatch classification as `BadDigest`; and
- parser, signed-handler, browser-response, and protocol adversarial tests.
Those changes share the same CORS release gate and are present in the reviewed
diff, but they are not part of the replication conflict key or join algorithm.
This document describes them only where they constrain replication validation
or the final verification boundary.
## Confirmed Failures in the Pre-Fix Candidate
The pre-fix issue-75 candidate had four independently reproduced convergence
defects:
1. Heal compared only payloads. If two sites stored identical payload bytes
with different source timestamps, heal skipped the older site. The sites
retained different ordering barriers and could disagree on a later delayed
event.
2. `isBucketMetadataEqual` used `strings.EqualFold` for base64. For example,
`QQ==` and `qQ==` decode to different bytes but compared equal.
3. Status derived `CorsCfgMismatch` from live payload count and payload set.
It did not include source timestamp or tombstone state, so it could report
divergent sites as converged and suppress healing.
4. Equal-timestamp conflicting events had no stable tie-breaker. Peer apply
accepted whichever event arrived last, while heal selected whichever map
entry happened to be visited first.
Two additional correctness requirements followed from the state model:
- the read, compare, and save transition must be atomic across nodes in one
cluster; and
- a successful local PUT or DELETE must advance beyond an already stored
future source timestamp instead of moving the local barrier backwards.
## Constraints
The minimum fix must satisfy these constraints:
- preserve `madmin.SRBucketMeta` and `madmin.SRBucketInfo` wire schemas;
- preserve the exact source `UpdatedAt` on peer apply and heal;
- distinguish a never-configured bucket from a persisted deletion;
- converge without relying on event arrival order, map iteration order, or a
particular site being the healer;
- serialize CORS-versus-CORS transitions cluster-wide without introducing a
broad bucket-metadata redesign;
- reject malformed replication payloads before persistence;
- remain idempotent under retry and initial-sync replay;
- keep the replication state-machine change limited to CORS except for the
directly shared base64 equality bug; do not infer that the same dirty
candidate contains no adjacent CORS protocol or middleware changes.
## State Model
For one bucket lineage, the persisted CORS state is:
```text
State = (Payload, SourceUpdatedAt)
```
`BucketMetadata.Created` is not part of the conflict key. It is the lineage
floor used to reject an event from an older incarnation of the bucket.
### State kinds
| Kind | Payload | `CorsConfigUpdatedAt` | Meaning |
| --- | --- | --- | --- |
| Baseline | nil | zero | CORS has never been configured for this bucket lineage |
| Live | non-empty XML bytes | non-zero | A live per-bucket CORS configuration |
| Tombstone | nil | non-zero | CORS was explicitly deleted at the source timestamp |
The baseline uses a zero timestamp deliberately. Defaulting a missing CORS
timestamp to `CreatedAt` would make classification depend on two values that
can be obtained from different cache/disk snapshots. It would also make a
never-configured state indistinguishable from a deletion at bucket creation.
Per-bucket CORS and `CorsConfigUpdatedAt` were introduced together, so there
is no released legacy live-CORS state that requires synthesizing a timestamp.
### Wire canonicalization
A non-nil wire payload must satisfy all of the following:
1. strict standard base64 decoding succeeds;
2. re-encoding the decoded bytes produces exactly the received string;
3. the decoded payload is non-empty;
4. CORS XML parsing succeeds; and
5. `cors.Config.Validate()` succeeds.
The canonical re-encode check rejects ignored newlines and alternate textual
representations. Equality is therefore equality of decoded bytes, with exact
base64 string equality remaining safe for the shared metadata helper.
An invalid wire value is not a candidate winner and is never propagated.
Peer apply rejects it before any metadata write.
A bucket may nevertheless contain a CORS document written by a pre-release,
more lenient build. Loading such metadata keeps policy, lifecycle, versioning,
and the other bucket fields available, but stashes the CORS parse/validation
error and exposes no active CORS config. CORS GET and middleware lookup return
that error, so browser handling fails closed. A valid PUT or DELETE can repair
the record; any attempt to save a newly invalid CORS document remains rejected.
## Deterministic Ordering
States use the following total order:
```text
1. SourceUpdatedAt
2. Kind: baseline < live < tombstone
3. For live/live ties: lexicographic decoded payload bytes
```
The greater state wins.
Consequences:
- a newer source event wins regardless of arrival order;
- the same payload with a newer timestamp is a greater state and advances the
ordering barrier;
- a DELETE wins an equal-timestamp PUT/DELETE conflict;
- two equal-timestamp live payloads choose the same bytewise winner at every
site;
- an exact duplicate is equal and therefore a no-op;
- retry, reordering, and duplicate delivery cannot move local state backward.
The live-payload tie-breaker is not intended to identify the human's temporal
intent. It supplies the deterministic result required when the timestamp has
already failed to distinguish two writes.
## Why No Source-Site Tie-Breaker
The rejected source-site alternative ordered states by timestamp plus origin
deployment ID. It would require a new origin field in madmin-go transport and
a persisted origin field in `BucketMetadata`. That adds a dependency release,
wire compatibility work, and an on-disk schema change without improving the
convergence guarantee over the content-based total order.
If a future product requirement needs provenance-aware conflict explanation,
the source-site design can be introduced as a versioned protocol. It is not
required to make the current register converge.
## Bucket Lineage and `CreatedAt`
`CreatedAt` protects a recreated bucket from delayed metadata events belonging
to the prior bucket incarnation:
```text
if incoming.SourceUpdatedAt < local.CreatedAt:
ignore and log once per bucket
```
The floor is retained because removing it could install an old CORS grant on a
new bucket with the same name. It is intentionally not used to classify the
baseline.
For current-version site replication, local events are generated strictly
after `max(CreatedAt, current CORS barrier)`, and bucket creation timestamps are
propagated before initial metadata sync. A floor rejection therefore indicates
a stale lineage event, clock/history corruption, or a mixed/unsupported setup.
The rejection is observable through a bucket-scoped log-once message and the
remaining status mismatch.
## Local Transition
PUT and DELETE use the same CORS-specific transition helper.
Under the bucket CORS namespace lock:
1. load the current `.metadata.bin` through the migration-aware parsed loader;
2. validate the new live payload, if any;
3. choose:
```text
UpdatedAt = max(UTCNow, CreatedAt + epsilon, CurrentBarrier + epsilon)
```
4. store either the live bytes or a nil tombstone with that timestamp;
5. save and refresh the parsed cache; and
6. release the lock before invoking `BucketMetaHook`.
This preserves HTTP semantics while ensuring a local administrative action is
strictly greater than the state it observed, including a future-dated peer
barrier caused by clock skew. The peer path deliberately uses a raw metadata
read instead: it must preserve the exact zero baseline and reject missing
metadata rather than implicitly creating a peer bucket record.
## Peer and Legacy-Bulk Transition
Typed CORS dispatch decodes and validates the payload, then performs this join
under the same lock:
```text
if incoming timestamp is zero:
reject
if incoming timestamp is before CreatedAt:
ignore and log
if incoming state <= local state:
no-op
otherwise:
persist incoming payload and exact source timestamp
```
The admin handler's legacy/default bulk metadata path can also carry a non-nil
CORS field. It therefore takes the shared metadata lock, applies strict
decoding and validation, and uses the same state comparison before saving. A
nil CORS field in that untyped legacy shape means "not included" and cannot
represent a tombstone; current producers use the typed CORS event for deletion.
## Concurrency and Locking
The transition lock is:
```text
.minio.sys / buckets/<bucket>/metadata.lock
```
It is a virtual distributed namespace lock. The name deliberately differs from
the real `buckets/<bucket>/.metadata.bin` object because the metadata save path
locks that object internally and namespace locks are not re-entrant.
The lock serializes CORS transitions with ordinary `Update`/`Delete`, legacy
bulk metadata, imports, bucket creation/adoption, and metadata migrations.
Every whole-record writer reads the latest disk state while holding the same
lock, so a writer for another configuration type cannot restore stale CORS
columns. Per-type validation, timestamp, and deletion semantics remain
independent.
No cross-site admin call or `BucketMetaHook` dispatch is made while holding the
metadata lock. The local disk save and resident-cache update complete under the
lock; intra-cluster metadata reload fan-out happens only after release. This
avoids a peer reload that needs migration from waiting on a lock held by the
notifying node. Reordered cross-site delivery is handled by the total-order
join.
The lock name changes from `cors-config.lock` to `metadata.lock`. During a
rolling upgrade, old and new nodes therefore do not serialize metadata writers
with each other; the shared-lock guarantee begins only after every node in the
cluster runs the new binary. Operators should avoid bucket-metadata changes
during that window. The on-disk record is unchanged, so rollback remains
format-compatible.
## Dispatch and Retry
PUT sends a typed `SRBucketMetaTypeCorsConfig` event with canonical base64 XML
and the local source timestamp. DELETE sends the same type with `Cors == nil`
and the tombstone timestamp.
`BucketMetaHook` may deliver concurrently to sites, fail on a subset, or be
retried by an external operation. The receiver transition is idempotent, so the
transport does not need to impose a global event order.
Current-version admin dispatch routes the typed event directly to
`PeerBucketCorsConfigHandler`. The legacy/default path is hardened only to
prevent a non-nil CORS field from bypassing the join; it is not a tombstone
compatibility protocol.
## Status Projection
`SiteReplicationMetaInfo` always exports `CorsConfigUpdatedAt`, including zero
baseline and nil tombstone states. It exports `CorsConfig` only for a live
payload.
Status considers sites converged if and only if every site has the same full
CORS state:
```text
(kind, decoded payload bytes, SourceUpdatedAt)
```
Live payload counts remain useful for per-site summary totals, but they do not
determine `CorsCfgMismatch`.
Examples:
| Site A | Site B | Mismatch |
| --- | --- | --- |
| baseline | baseline | no |
| same live bytes at same timestamp | same live bytes at same timestamp | no |
| same live bytes at different timestamps | same live bytes at different timestamps | yes |
| same tombstone timestamp | same tombstone timestamp | no |
| tombstones at different timestamps | tombstones at different timestamps | yes |
| live | tombstone | yes |
| invalid wire state | any state | yes |
## Winner Selection and Heal
Heal computes the maximum non-baseline valid state using the total order.
Selection is independent of Go map iteration. Deployment ID is used only as a
stable log-source choice when two sites already expose exactly equal states.
For each different site:
- the local site delegates to the normal peer CORS transition, preserving the
source timestamp and lock discipline;
- a remote site receives a typed `SRBucketMetaTypeCorsConfig` event with the
winner's canonical payload or nil tombstone and exact timestamp.
Payload equality alone is insufficient. A site with identical bytes at an
older timestamp is healed so it acquires the same future ordering barrier.
If every reported state is baseline, there is no event to propagate. If every
reported state is invalid, status remains mismatched and heal does not select
corrupt input as a source.
## Initial Sync
Initial sync emits:
- a live event when `CorsConfigUpdatedAt` is non-zero and payload is live;
- a tombstone event when `CorsConfigUpdatedAt` is non-zero and payload is nil;
- no event for the zero baseline.
Replaying initial sync is idempotent. A missed DELETE is recoverable because
the tombstone is part of the snapshot rather than being inferred from the
absence of a live payload.
All sites must run the CORS-aware implementation before enabling or mutating
per-bucket CORS. An older receiver can route an unknown typed event through a
legacy path that cannot represent deletion and does not provide this ordering
contract.
## Persistence and Restart
`CorsConfigXML` and `CorsConfigUpdatedAt` are persisted together in
`BucketMetadata` msgpack. Zero time round-trips as zero; CORS is deliberately
not defaulted to `CreatedAt` during load.
`BucketMetadata.Save` parses the live CORS XML before writing and before the
metadata system replaces the local cache. Therefore a rejected payload cannot
poison disk or cache, and a successful peer/heal transition immediately serves
the newly persisted parsed configuration.
After cache removal or process restart:
- a live state restores the same parsed rules and source timestamp;
- a tombstone restores nil payload plus its non-zero timestamp;
- a baseline remains nil plus zero timestamp.
- a legacy-invalid raw document leaves the non-CORS bucket metadata readable,
disables per-bucket CORS fail-closed, and remains repairable through a valid
CORS PUT or DELETE.
## Error Handling
| Error | Behavior |
| --- | --- |
| zero source timestamp on a peer live/delete event | reject the event |
| invalid/non-canonical base64 | reject before locking or saving |
| empty non-nil payload | reject |
| malformed XML | reject before saving |
| semantically invalid CORS rules | reject before saving |
| legacy-invalid CORS already on disk | load other metadata, return a CORS-specific error, and permit CORS replacement or deletion |
| event before bucket `CreatedAt` | ignore and log once per bucket |
| missing bucket metadata | return an error; do not create metadata implicitly |
| exact duplicate or lower state | successful no-op |
| remote heal failure | log the peer error; future heal cycles retry |
## Alternatives Considered
### Timestamp only
Rejected. Ignoring or accepting every equal-timestamp conflict leaves an
already divergent pair without a deterministic repair rule.
### Timestamp plus source deployment ID
Rejected for the current protocol. It is convergent, but requires madmin-go,
wire, and persisted-schema changes without improving convergence over the
selected total order.
### Payload-only status and heal
Rejected. It cannot distinguish ordering barriers and suppresses the exact
heal needed to make later event acceptance consistent.
### Default baseline timestamp to bucket creation
Rejected. It conflates baseline classification with a mutable value that may
come from a different snapshot and can turn a never-configured site into a
false tombstone source.
### Reuse `.metadata.bin` as the transition lock
Rejected. The save path takes the same namespace lock internally; reusing it
would self-deadlock.
### Give every metadata type a new state machine
Rejected. The shared lock prevents whole-record lost updates without changing
the independent replication, validation, or deletion semantics of policy,
tags, SSE, quota, versioning, and Object Lock. Those semantic audits remain
separate from the persistence fix in issue #102.
## Invariants
The implementation is acceptable only while all of these invariants hold:
1. Zero timestamp plus nil payload is the only baseline representation.
2. Nil payload plus non-zero timestamp is a durable tombstone.
3. A live payload has canonical base64 on the wire, valid CORS XML, and a
non-zero source timestamp.
4. Every intentional current-version CORS state transition and every other
whole-record metadata writer is serialized by `metadata.lock` from the
authoritative disk read through save and local cache publication.
5. Peer apply and heal never replace local state with a lower or equal state.
6. Local PUT and DELETE create a state strictly greater than the state observed
under the lock.
7. A peer event before local bucket creation cannot modify the new bucket
lineage.
8. Status reports convergence only for identical full states.
9. Heal selects the same maximum regardless of arrival order, site, or map
iteration order.
10. Same-payload/newer-timestamp heal advances the older barrier.
11. Initial sync and retry preserve tombstones and source timestamps.
12. Disk reload and cache reload preserve state kind, payload, and timestamp.
13. A legacy-invalid CORS document cannot activate global fallback, hide other
bucket metadata, or prevent a valid CORS PUT/DELETE repair.
## Test Contract
The required test matrix is:
| Area | Required evidence |
| --- | --- |
| Wire | canonical base64 accepted; case-different decoded bytes differ; malformed and non-canonical base64 rejected |
| Validation | invalid XML and semantically invalid origin/method/rule rejected without mutation |
| Strict wire | standard S3 namespace accepted; trailing root, unknown/nested elements, duplicate singleton fields, lowercase methods, byte-counted Unicode IDs, and invalid MaxAge rejected |
| Ordering | older event ignored; newer event applied; duplicate no-op; equal live/live order-independent; equal PUT/DELETE chooses tombstone |
| Barrier | same payload with newer timestamp is persisted and healed |
| Tombstone | delayed PUT cannot resurrect; missed DELETE wins heal; repeated DELETE is idempotent |
| Status | baseline, live, tombstone, payload mismatch, and timestamp-only mismatch classified correctly |
| Winner | three-site equal-timestamp selection remains deterministic across repeated map iteration |
| Concurrency | concurrent peer and legacy-bulk events converge to the total-order maximum |
| Local concurrency | concurrent local PUT/DELETE timestamps are unique and final state matches the last serialized transition |
| Initial sync | baseline omitted; live and tombstone emitted with exact source timestamp |
| Lineage | pre-creation event ignored; post-creation event applied |
| Restart | cache removal/disk reload preserves tombstone or live state and status timestamp |
| Legacy repair | a lenient historical document loads fail-closed without hiding other metadata and can be deleted or replaced |
| Full seam | signed admin dispatch -> peer apply -> real status collection -> local heal -> cache reload -> remote heal dispatch |
docs/site-replication/run-ssec-object-replication-with-compression.sh
+64 -11
View File
@@ -62,13 +62,14 @@ echo "Hello world" >/tmp/data/plainfile
echo "Hello from encrypted world" >/tmp/data/encrypted
touch /tmp/data/defpartsize
shred -s 500M /tmp/data/defpartsize
touch /tmp/data/mpartobj.txt
shred -s 500M /tmp/data/mpartobj.txt
# Compressible, and large enough for a multipart upload, so the object would be
# stored compressed if SSE-C were not excluded from compression.
yes "silo compression and sse-c replication payload" | head -c 100000000 >/tmp/data/mpartobj.txt
echo "done"
# Enable compression for site silo1
./mc admin config set silo1 compression enable=on extensions=".txt" --insecure
./mc admin config set silo1 compression allow_encryption=off --insecure
./mc admin config set silo1 compression enable=on extensions=".txt" --insecure || exit_1
./mc admin config set silo1 compression allow_encryption=on --insecure || exit_1
# Create bucket in source cluster
echo "Create bucket in source Silo instance"
@@ -80,13 +81,13 @@ echo "Loading objects to source Silo instance"
./mc cp /tmp/data/encrypted silo1/test-bucket/encrypted --enc-c "silo1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure
./mc cp /tmp/data/defpartsize silo1/test-bucket/defpartsize --enc-c "silo1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure
# Below should fail as compression and SSEC used at the same time
# DISABLED: We must check the response header to see if compression was actually applied
#RESULT=$({ ./mc put /tmp/data/mpartobj.txt silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure; } 2>&1)
#if [[ ${RESULT} != *"Server side encryption specified with SSE-C with compression not allowed"* ]]; then
# echo "BUG: Loading an SSE-C object to site with compression should fail. Succeeded though."
# exit_1
#fi
# A compressible .txt object written with SSE-C while allow_encryption=on. SSE-C
# is excluded from compression whatever allow_encryption says, because
# replication ships SSE-C objects as raw ciphertext and the wire cannot carry the
# compression metadata. Were the object stored compressed, the replica would hold
# the compressed bytes with no compression marker and decrypt to a raw S2 stream,
# which the size and content checks below detect.
./mc cp /tmp/data/mpartobj.txt silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure
# Add replication site
./mc admin replicate add silo1 silo2 --insecure
@@ -111,6 +112,11 @@ if [ "${count3}" -ne 1 ]; then
echo "BUG: object silo1/test-bucket/defpartsize not found"
exit_1
fi
count4=$(./mc ls silo1/test-bucket/mpartobj.txt --insecure | wc -l)
if [ "${count4}" -ne 1 ]; then
echo "BUG: object silo1/test-bucket/mpartobj.txt not found"
exit_1
fi
sleep 120
# List the objects from replicated site
@@ -131,6 +137,11 @@ if [ "${repcount3}" -ne 1 ]; then
echo "BUG: object test-bucket/defpartsize not replicated"
exit_1
fi
repcount4=$(./mc ls silo2/test-bucket/mpartobj.txt --insecure | wc -l)
if [ "${repcount4}" -ne 1 ]; then
echo "BUG: object test-bucket/mpartobj.txt not replicated"
exit_1
fi
# Stat the SSEC objects from source site
echo "Stat silo1/test-bucket/encrypted"
@@ -145,6 +156,25 @@ stat_out2=$(./mc stat --no-list silo1/test-bucket/defpartsize --enc-c "silo1/tes
src_obj2_etag=$(echo "${stat_out2}" | jq '.etag')
src_obj2_size=$(echo "${stat_out2}" | jq '.size')
src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat silo1/test-bucket/mpartobj.txt"
./mc stat --no-list silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json
# The compression marker reaches the client only as the X-Minio-Compressed
# response header, which the SDK filters out of `mc stat --json`, so read the
# raw HTTP trace instead. The sentinel check keeps the assertion from passing
# vacuously if the trace format ever changes.
stat_trace=$(./mc --debug stat --no-list silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure 2>&1) || exit_1
if ! grep -qi "X-Amz-Request-Id" <<<"${stat_trace}"; then
echo "BUG: 'mc --debug stat' printed no response headers, so the compression check below proves nothing"
exit_1
fi
if grep -qi "X-Minio-Compressed" <<<"${stat_trace}"; then
echo "BUG: SSE-C object 'silo1/test-bucket/mpartobj.txt' was stored compressed despite the SSE-C compression exclusion"
exit_1
fi
stat_out3=$(./mc stat --no-list silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json)
src_obj3_etag=$(echo "${stat_out3}" | jq '.etag')
src_obj3_size=$(echo "${stat_out3}" | jq '.size')
src_obj3_md5=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Stat the SSEC objects from replicated site
echo "Stat silo2/test-bucket/encrypted"
@@ -159,6 +189,12 @@ stat_out2_rep=$(./mc stat --no-list silo2/test-bucket/defpartsize --enc-c "silo2
rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag')
rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size')
rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat silo2/test-bucket/mpartobj.txt"
./mc stat --no-list silo2/test-bucket/mpartobj.txt --enc-c "silo2/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json
stat_out3_rep=$(./mc stat --no-list silo2/test-bucket/mpartobj.txt --enc-c "silo2/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json)
rep_obj3_etag=$(echo "${stat_out3_rep}" | jq '.etag')
rep_obj3_size=$(echo "${stat_out3_rep}" | jq '.size')
rep_obj3_md5=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Check the etag and size of replicated SSEC objects
if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then
@@ -177,10 +213,23 @@ if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then
echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'silo2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'"
exit_1
fi
if [ "${rep_obj3_etag}" != "${src_obj3_etag}" ]; then
echo "BUG: Etag: '${rep_obj3_etag}' of replicated object: 'silo2/test-bucket/mpartobj.txt' doesn't match with source value: '${src_obj3_etag}'"
exit_1
fi
if [ "${rep_obj3_size}" != "${src_obj3_size}" ]; then
echo "BUG: Size: '${rep_obj3_size}' of replicated object: 'silo2/test-bucket/mpartobj.txt' doesn't match with source value: '${src_obj3_size}'"
exit_1
fi
# Check content of replicated SSEC objects
./mc cat silo2/test-bucket/encrypted --enc-c "silo2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure
./mc cat silo2/test-bucket/defpartsize --enc-c "silo2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure >/dev/null || exit_1
./mc cat silo2/test-bucket/mpartobj.txt --enc-c "silo2/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure >/tmp/data/mpartobj.replica || exit_1
if ! cmp -s /tmp/data/mpartobj.txt /tmp/data/mpartobj.replica; then
echo "BUG: replicated object 'silo2/test-bucket/mpartobj.txt' does not match the source; a compressed SSE-C object decrypts to a raw S2 stream on the replica"
exit_1
fi
# Check the MD5 checksums of encrypted objects from source and target
if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then
@@ -191,5 +240,9 @@ if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then
echo "BUG: MD5 checksum of object 'silo2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'"
exit_1
fi
if [ "${src_obj3_md5}" != "${rep_obj3_md5}" ]; then
echo "BUG: MD5 checksum of object 'silo2/test-bucket/mpartobj.txt' doesn't match with source. Expected: '${src_obj3_md5}', Found: '${rep_obj3_md5}'"
exit_1
fi
cleanup