feat: add a trusted-proxy boundary for the client source address

The address MinIO attributes a request to is read from X-Forwarded-For,
X-Real-IP or RFC 7239 Forwarded, and never from the connection unless all
three are absent. It becomes aws:SourceIp and the audit remotehost field,
so any client that can reach the API port chooses the value an IpAddress
condition is evaluated against and the address every logged action is
attributed to.

MINIO_API_TRUSTED_PROXIES now selects who may make that claim:

  unset     the historical behaviour, unchanged
  none      no forwarded header is believed; the TCP peer wins
  <CIDRs>   believed only from listed peers, chains read right-to-left

Reading right-to-left is what makes an appending proxy safe: each hop
appends the peer it actually saw, so an entry a client injected can only
sit to the left of one a proxy wrote. The stock nginx recipe
$proxy_add_x_forwarded_for appends, which leaves the client's entry
left-most - exactly where the untrusted path reads - so a deployment with
no direct route to the API port was forgeable too.

_MINIO_API_XFF_HEADER is deliberately untouched, in semantics and in read
timing. Widening it to mean "trust nothing" was implemented and reverted:
it is the only part of this change that could alter a deployed
configuration, and the new variable expresses the same guarantee at no
compatibility cost. Upstream's TestXFFDisabled is retained verbatim.

Notes on the allow-list mode, all covered by tests:

  - it must name proxies, not the subnet they sit in; listed entries are
    skipped while walking, so a range covering clients lets them forge
  - a cluster must list its own nodes, because MinIO forwards between
    them and a client can force a hop via the ListObjectsV2 token
  - loopback is trusted as a peer, not as a chain entry, so FTP and SFTP
    keep attributing their sessions
  - the node-to-node forwarder drops X-Real-IP and Forwarded from a peer
    not entitled to have set them
  - the walk scans the header in place and stops after 100 hops, so a
    long chain costs neither allocation nor unbounded work

No behaviour change for any deployment that does not set the new
variable: the untrusted path is a verbatim copy of the previous function
body, differentially verified against it over ~5.1M header combinations.
The LDAP STS allow-list now shares the list parser as pure code motion,
verified identical across every combination of 37 allow-list values and
21 peer addresses.

Co-authored-by: ChatGPT <noreply@openai.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Feng Ruohang
2026-08-04 13:09:06 +08:00
parent 744a9dcd71
commit fe6dc47804
10 changed files with 1369 additions and 74 deletions
+102 -1
View File
@@ -21,6 +21,7 @@ import (
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
"os"
"slices" "slices"
"strings" "strings"
"testing" "testing"
@@ -317,6 +318,104 @@ func TestBucketPolicySourceIPCannotBeForged(t *testing.T) {
} }
} }
// The trust policy has to reach the decision, not merely the resolver. This
// drives a forged X-Forwarded-For all the way through getConditionValues into a
// real IpAddress evaluation under each mode. Everything else about the trust
// modes is tested where the logic lives; this is the only test that would notice
// if the resolver were correct but the policy engine were reading something else.
func TestBucketPolicySourceIPForgeryAcrossTrustModes(t *testing.T) {
_, cidr, err := net.ParseCIDR("10.0.0.0/8")
if err != nil {
t.Fatal(err)
}
fn, err := condition.NewIPAddressFunc(condition.AWSSourceIP.ToKey(), cidr)
if err != nil {
t.Fatal(err)
}
bp := policy.BucketPolicy{
Version: policy.DefaultVersion,
Statements: []policy.BPStatement{{
Effect: policy.Allow,
Principal: policy.NewPrincipal("*"),
Actions: policy.NewActionSet(policy.GetObjectAction),
Resources: policy.NewResourceSet(policy.NewResource("bkt/*")),
Conditions: condition.NewFunctions(fn),
}},
}
// An address inside the permitted range, asserted by a client that is not.
const forgedClaim = "10.1.2.3"
const outsider = "203.0.113.5:12345"
const proxy = "192.0.2.7:9000"
tests := []struct {
name string
proxies string
peer string
allowed bool
}{{
// Documented, and the reason the allow-list exists.
name: "default mode believes the claim",
peer: outsider,
allowed: true,
}, {
name: "trusting nobody ignores the claim",
proxies: handlers.TrustNoProxies,
peer: outsider,
allowed: false,
}, {
name: "allow-list ignores an unlisted peer's claim",
proxies: "192.0.2.7",
peer: outsider,
allowed: false,
}, {
// The allow-list must not break the deployment it exists to serve.
name: "allow-list still honors its own proxy",
proxies: "192.0.2.7",
peer: proxy,
allowed: true,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Cleanup(func() {
os.Unsetenv(handlers.EnvTrustedProxies)
if err := handlers.ConfigureSourceIPTrust(); err != nil {
t.Fatalf("restoring the default policy: %v", err)
}
})
if tt.proxies == "" {
os.Unsetenv(handlers.EnvTrustedProxies)
} else {
t.Setenv(handlers.EnvTrustedProxies, tt.proxies)
}
if err := handlers.ConfigureSourceIPTrust(); err != nil {
t.Fatalf("configuring %q: %v", tt.proxies, err)
}
r, err := http.NewRequest(http.MethodGet, "http://minio.local/bkt/obj", nil)
if err != nil {
t.Fatal(err)
}
r.RemoteAddr = tt.peer
r.Header.Set("X-Forwarded-For", forgedClaim)
if err := r.ParseForm(); err != nil {
t.Fatal(err)
}
got := bp.IsAllowed(policy.BucketPolicyArgs{
Action: policy.GetObjectAction,
BucketName: "bkt",
ObjectName: "obj",
ConditionValues: getConditionValues(r, "us-east-1", auth.Credentials{AccessKey: "lowpriv"}),
})
if got != tt.allowed {
t.Errorf("IsAllowed = %v, want %v (peer %s claiming %s)", got, tt.allowed, tt.peer, forgedClaim)
}
})
}
}
// aws:SourceIp must be whatever the hardened resolver decided and nothing else. // aws:SourceIp must be whatever the hardened resolver decided and nothing else.
// The resolver is where the forwarded-header trust policy is enforced and where // The resolver is where the forwarded-header trust policy is enforced and where
// its three modes are tested (internal/handlers/proxy_test.go); this pins the // its three modes are tested (internal/handlers/proxy_test.go); this pins the
@@ -327,7 +426,9 @@ func TestBucketPolicySourceIPCannotBeForged(t *testing.T) {
// each of the three forwarded headers still sets aws:SourceIp, and so an // each of the three forwarded headers still sets aws:SourceIp, and so an
// IpAddress condition is only as good as the network path to the API port. // IpAddress condition is only as good as the network path to the API port.
// Enforcing such a condition against a client with direct access requires // Enforcing such a condition against a client with direct access requires
// MINIO_API_TRUSTED_PROXIES or _MINIO_API_XFF_HEADER=off. // MINIO_API_TRUSTED_PROXIES. Note that _MINIO_API_XFF_HEADER=off does not
// achieve it: the loop below covers all three headers precisely because
// suppressing one of them only moves the answer to the next.
func TestGetConditionValuesSourceIPMatchesResolver(t *testing.T) { func TestGetConditionValuesSourceIPMatchesResolver(t *testing.T) {
for _, header := range []map[string]string{ for _, header := range []map[string]string{
nil, nil,
+11
View File
@@ -54,6 +54,7 @@ import (
"github.com/minio/minio/internal/auth" "github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/color" "github.com/minio/minio/internal/color"
"github.com/minio/minio/internal/config" "github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/handlers"
"github.com/minio/minio/internal/kms" "github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger" "github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/certs" "github.com/minio/pkg/v3/certs"
@@ -711,6 +712,16 @@ func serverHandleEarlyEnvVars() {
func serverHandleEnvVars() { func serverHandleEnvVars() {
var err error var err error
// Re-read the source-address trust policy now that loadEnvVarsFromFiles has
// run: a policy taken at package initialisation would miss every deployment
// configured through MINIO_CONFIG_ENV_FILE. Refuse to start on a malformed
// allow-list rather than resolve aws:SourceIp and every audit client address
// by a rule the operator did not write.
if err := handlers.ConfigureSourceIPTrust(); err != nil {
logger.Fatal(err, "Invalid %s value in environment variable", handlers.EnvTrustedProxies)
}
if globalBrowserEnabled { if globalBrowserEnabled {
if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" { if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" {
u, err := xnet.ParseHTTPURL(redirectURL) u, err := xnet.ParseHTTPURL(redirectURL)
+2
View File
@@ -2,6 +2,8 @@
For fork-specific security advisories and upgrade notes in `pgsty/minio`, see [advisories.md](advisories.md). For fork-specific security advisories and upgrade notes in `pgsty/minio`, see [advisories.md](advisories.md).
For which peers may tell the server where a request came from — and therefore whether `aws:SourceIp` conditions and audit client addresses can be relied on — see [Client source address trust](source-address-trust.md).
## Server-Side Encryption ## Server-Side Encryption
MinIO supports two different types of server-side encryption ([SSE](#sse)): MinIO supports two different types of server-side encryption ([SSE](#sse)):
+223
View File
@@ -0,0 +1,223 @@
# Client source address trust
MinIO decides where a request came from, and that decision is load-bearing. The
address it settles on becomes:
- `aws:SourceIp`, so it decides `IpAddress` and `NotIpAddress` policy conditions
- the audit log's `remotehost` field, so it decides who every logged action is
attributed to
- the `Host` field of S3 event notifications, and the client shown by
`mc admin trace`
None of that is derived from the TCP connection by default. It is read out of the
`X-Forwarded-For`, `X-Real-IP` and RFC 7239 `Forwarded` request headers, which
any client can set to any value. This document states which peers MinIO believes,
how to change that, and what each choice costs.
## The three modes
One setting selects all three. It is read from the environment, not from
`mc admin config`.
| Mode | `MINIO_API_TRUSTED_PROXIES` | Source address |
| :-- | :-- | :-- |
| Untrusted (default) | unset | left-most `X-Forwarded-For` entry, else `X-Real-IP`, else `Forwarded`, else the TCP peer — from any client |
| Trust nobody | `none` | always the TCP peer |
| Allow-listed | a list of addresses and CIDR blocks | forwarded headers, but only from listed peers |
`off` is accepted as a synonym for `none`, and the value is matched
case-insensitively.
The setting is read once at startup, after `MINIO_CONFIG_ENV_FILE` is loaded, so
an environment file is a valid place to put it. A malformed value stops the
server rather than silently changing what every policy condition resolves to.
> **`_MINIO_API_XFF_HEADER` is not this setting.** It suppresses parsing of
> `X-Forwarded-For` and nothing else, leaving `X-Real-IP` and `Forwarded`
> honoured, so it cannot stop a client naming its own address — a client refused
> one header simply sends another. It keeps its original upstream meaning, and
> applies *within* whichever mode above is in force. If you set it hoping to stop
> source-address forgery, that is what `MINIO_API_TRUSTED_PROXIES` is for.
>
> It also keeps upstream's read timing, which is *earlier* than the one described
> above: it is taken at package initialisation, before environment files are
> read, so writing it into `MINIO_CONFIG_ENV_FILE` has no effect. That quirk is
> deliberately left in place rather than repaired, because repairing it would
> make an already-deployed setting start taking effect. Set it in the process
> environment if you want it honoured.
### Untrusted (default)
Every deployment behaves exactly as it did before this setting existed. Any
client that can open a connection to the S3 API port can name its own address,
so under this mode:
**An `IpAddress` policy condition is not enforceable, and audit client addresses
are not evidence.** Both are attacker-chosen for anyone with direct network
access to the API port.
This mode is sound only when every route to the API port passes through a proxy
that overwrites all three headers. Two things commonly break that assumption:
- **A second way in.** On Kubernetes an Ingress and a ClusterIP Service usually
coexist. The Ingress sanitises headers; the Service does not, and any pod in
the cluster can reach it. The boundary is assumed to be the Ingress but is
actually the pod network.
- **Appending proxies.** The stock nginx recipe
`proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;` *appends* to
what the client sent. A client sending `X-Forwarded-For: 1.2.3.4` produces
`1.2.3.4, <real client>` at MinIO, and this mode reads the left-most entry —
the client's. Overwriting with `$remote_addr` avoids this; so does the
allow-listed mode below, which reads the chain from the other end.
### Trust nobody
```
MINIO_API_TRUSTED_PROXIES=none
```
No forwarded header is believed. The source address is the TCP peer, always.
Use this when MinIO is reached directly and you would rather have a correct
address that is sometimes a proxy than a plausible one that is sometimes a lie.
Behind a proxy, every request will be attributed to the proxy.
**On a multi-node deployment this also applies to MinIO's own nodes.** Some
requests are forwarded between nodes (see [Multi-node](#multi-node-deployments)),
and the receiving node's peer is the forwarding node, so those requests are
attributed to it rather than to the client. Nothing corrects this under this
mode, because it believes nothing. Multi-node clusters that want direct requests
protected *and* internally-forwarded ones attributed correctly should use the
allow-listed mode with the node addresses included.
The scheme headers (`X-Forwarded-Proto`, `X-Forwarded-Scheme`) are deliberately
unaffected by any of this. They feed the `Location` URL in S3 responses, not a
policy decision, and suppressing them would hand `http://` URLs to every
deployment terminating TLS at a proxy.
### Allow-listed
```
MINIO_API_TRUSTED_PROXIES=10.0.0.1,10.0.0.2
```
A comma-, semicolon- or whitespace-separated list of addresses and CIDR blocks.
Forwarded headers are believed only when the request's TCP peer matches. Every
other peer is attributed to itself, whatever its headers claim.
Write entries in their plain form. An address written in IPv4-mapped notation
(`::ffff:192.168.1.10`) is accepted but matches nothing, because peers are
reduced to plain form before matching — so the entry grants no trust and the
proxy is treated as any other client. This is long-standing behaviour shared
with `MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES`, and it fails closed.
This is the only mode under which `aws:SourceIp` is enforceable against a client
that can reach the API port directly.
Catch-all ranges (`0.0.0.0/0`, `::/0`) are rejected. They would trust every peer
and quietly reinstate the untrusted mode under a name that suggests otherwise.
Note that only `/0` itself is rejected — a pair like `0.0.0.0/1,128.0.0.0/1`
covers the same ground and is accepted, so the check is a guardrail, not a proof.
> **List proxies, not the subnet they sit in.** This is the one way to configure
> the setting so that it makes things worse rather than better, so it is worth
> stating plainly.
>
> Entries on the list are skipped during the chain walk described below. If the
> list covers addresses that *clients* also occupy — `10.0.0.0/8` when the load
> balancer is `10.0.0.1` and application servers are elsewhere in `10/8` — then
> those clients' addresses are skipped too, and the walk continues past the real
> client into whatever it placed to the left. A client at `10.5.5.5` sending
> `X-Forwarded-For: 8.8.8.8` would then be recorded as `8.8.8.8`.
>
> A broad list does not merely trust more peers; it lets those peers forge.
> Prefer exact host entries or the narrowest prefix that contains only proxies.
**Chain handling.** `X-Forwarded-For` and `Forwarded` are read right-to-left,
stepping over entries that name a listed proxy, and the first remaining address
wins. Each proxy appends the peer it actually saw, so an entry the client
injected sits to the left of the one its proxy wrote, and the walk stops before
reaching it. Appending proxies are therefore safe here. List every proxy hop's
address so intermediate hops are skipped. Repeated header lines are handled as
one chain, so proxies that add a second `X-Forwarded-For` line rather than
extending the first — HAProxy's `option forwardfor` — work correctly.
**`X-Real-IP` is a single value with no chain, so it cannot be checked against
the list.** It is trusted verbatim, and only when the chain headers yield
nothing. The deployment contract is that a listed proxy overwrites whichever
headers it sets — for nginx, `proxy_set_header X-Real-IP $remote_addr;`. A proxy
that relays the client's copy instead is choosing to let the client answer the
question, and nothing MinIO does can undo that.
> **Strip at the edge every source-address header your proxy does not itself
> write.** This is the one rule that covers every case, and it is worth following
> even if the rest of this section is skipped.
>
> Listing a peer means believing all three headers from it, and they are
> consulted in order: `X-Forwarded-For`, then `X-Real-IP`, then `Forwarded`. A
> header your proxy does not write is entirely under the client's control, and if
> it is consulted before the one your proxy *does* write, the client wins.
>
> The common way to get bitten: a proxy that authors only `X-Real-IP` (some nginx
> configurations) or only `Forwarded` (RFC 7239-native proxies) relays the
> client's `X-Forwarded-For` untouched — and `X-Forwarded-For` is read first, so
> a client sending `X-Forwarded-For: 8.8.8.8` is recorded as `8.8.8.8` despite
> the proxy having correctly written the real address elsewhere.
(`MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES` orders the first two the other way
round. Neither order is safe for every proxy — the mirror hazard is a proxy that
authors only `X-Forwarded-For` and relays a client's `X-Real-IP`, which is what
AWS ALB does. This setting prefers the header it can chain-validate, because it
decides access control rather than rate-limit bucketing. Stripping what your
proxy does not write makes the ordering irrelevant, which is why it is the rule
worth remembering.)
**Loopback is always trusted as a peer**, even when not listed. The FTP and SFTP
front-ends reach the S3 layer over `127.0.0.1` and declare their session's client
with `X-Forwarded-For`; excluding loopback would attribute every FTP and SFTP
request to the server itself. A `127.0.0.1` entry appearing *inside* a chain is
not skipped — only peers are exempt, not chain entries.
## Multi-node deployments
**A cluster must list its own nodes.** MinIO forwards some requests between
nodes — bucket-DNS and site-replication routing, listing continuation,
heal-by-token, batch jobs and pool decommissioning. The receiving node's peer is
the forwarding node, so unless the cluster's own addresses are on the list, those
requests resolve to the forwarding node rather than to the client. Include the
node addresses alongside your proxy's.
This is not a rare path. A `ListObjectsV2` continuation token carries the node
index, so any client can cause its own request to be forwarded. If the nodes are
not listed, those requests are evaluated with `aws:SourceIp` set to an internal
node address and audited against one — which an `IpAddress` condition that allows
internal ranges would treat as a pass.
## Choosing
Behind a proxy you control, with no other route to the API port, the default is
fine and you need not set anything — but confirm the proxy overwrites rather than
appends, or you are in the second case below. Otherwise:
- Direct exposure, no proxy: `MINIO_API_TRUSTED_PROXIES=none`.
- Behind a proxy, but the port is also reachable directly (the usual Kubernetes
Ingress-plus-Service case, and the usual Pigsty case): set
`MINIO_API_TRUSTED_PROXIES` to the proxy addresses plus the MinIO node
addresses. This is the configuration that makes an `IpAddress` condition mean
something.
- Multi-node clusters: use the allow-list with node addresses included, not
`none`.
- Behind a proxy you are not sure about: allow-list it, and strip at the edge
every source-address header it does not itself write.
Whichever applies, two rules govern every allow-list deployment: name proxies,
not the subnet they occupy; and strip what your proxy does not write.
## Relationship to LDAP STS trusted proxies
`MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES` is a separate allow-list governing
which peers may supply the client address used for **LDAP STS login rate-limit
bucketing** only. It does not affect `aws:SourceIp` or audit addresses, and this
setting does not affect STS rate limiting. The two use the same list syntax and
the same chain-walking rules, and in most deployments should be set to the same
value. See [`docs/sts/ldap.md`](../sts/ldap.md).
+2
View File
@@ -119,6 +119,8 @@ Only requests whose peer address matches this allowlist may supply forwarded cli
The RFC 7239 `Forwarded` header is not used for this bucket; deployments that only send `Forwarded` fall back to the peer-address bucket. The RFC 7239 `Forwarded` header is not used for this bucket; deployments that only send `Forwarded` fall back to the peer-address bucket.
This allowlist governs LDAP STS rate-limit bucketing only. The client address used for `aws:SourceIp`, audit logs and event notifications is governed separately by `MINIO_API_TRUSTED_PROXIES` — see [Client source address trust](../security/source-address-trust.md). The two use the same list syntax and the same chain-walking rules, and in most deployments should be set to the same value. They differ deliberately in one respect: this setting prefers `X-Real-IP` over `X-Forwarded-For`, while `MINIO_API_TRUSTED_PROXIES` prefers the chain-validated `X-Forwarded-For`, because it decides access control rather than rate-limit bucketing. Neither order is safe for every proxy; the linked document explains the trade-off.
### Lookup-Bind ### Lookup-Bind
A low-privilege read-only LDAP service account is configured in the MinIO server by providing the account's Distinguished Name (DN) and password. This service account is used to perform directory lookups as needed. A low-privilege read-only LDAP service account is configured in the MinIO server by providing the account's Distinguished Name (DN) and password. This service account is used to perform directory lookups as needed.
+8 -59
View File
@@ -22,9 +22,7 @@ import (
"crypto/x509" "crypto/x509"
"errors" "errors"
"net" "net"
"net/netip"
"sort" "sort"
"strings"
"time" "time"
"github.com/minio/madmin-go/v3" "github.com/minio/madmin-go/v3"
@@ -45,7 +43,7 @@ type Config struct {
LDAP ldap.Config LDAP ldap.Config
stsExpiryDuration time.Duration // contains converted value stsExpiryDuration time.Duration // contains converted value
stsTrustedProxies []netip.Prefix stsTrustedProxies config.TrustedProxies
} }
// Enabled returns if LDAP is enabled. // Enabled returns if LDAP is enabled.
@@ -61,7 +59,7 @@ func (l *Config) Clone() Config {
cfg := Config{ cfg := Config{
LDAP: l.LDAP.Clone(), LDAP: l.LDAP.Clone(),
stsExpiryDuration: l.stsExpiryDuration, stsExpiryDuration: l.stsExpiryDuration,
stsTrustedProxies: append([]netip.Prefix(nil), l.stsTrustedProxies...), stsTrustedProxies: append(config.TrustedProxies(nil), l.stsTrustedProxies...),
} }
return cfg return cfg
} }
@@ -168,49 +166,9 @@ var (
} }
) )
func parseSTSTrustedProxies(value string) ([]netip.Prefix, error) {
fields := strings.FieldsFunc(value, func(r rune) bool {
switch r {
case ',', ';', ' ', '\n', '\r', '\t':
return true
default:
return false
}
})
if len(fields) == 0 {
return nil, nil
}
prefixes := make([]netip.Prefix, 0, len(fields))
for _, field := range fields {
if prefix, err := netip.ParsePrefix(field); err == nil {
masked := prefix.Masked()
// Reject catch-all ranges (0.0.0.0/0, ::/0): they would trust
// forwarded headers from every peer and defeat the allowlist.
if masked.Bits() == 0 {
return nil, config.Errorf("LDAP STS trusted proxy %q is too broad", field)
}
prefixes = append(prefixes, masked)
continue
}
addr, err := netip.ParseAddr(field)
if err != nil {
return nil, config.Errorf("invalid LDAP STS trusted proxy %q", field)
}
bits := 32
if addr.Is6() {
bits = 128
}
prefixes = append(prefixes, netip.PrefixFrom(addr, bits))
}
return prefixes, nil
}
// SetSTSTrustedProxies parses and stores the LDAP STS trusted proxy allowlist. // SetSTSTrustedProxies parses and stores the LDAP STS trusted proxy allowlist.
func (l *Config) SetSTSTrustedProxies(value string) error { func (l *Config) SetSTSTrustedProxies(value string) error {
prefixes, err := parseSTSTrustedProxies(value) prefixes, err := config.ParseTrustedProxies(value, "LDAP STS trusted proxy")
if err != nil { if err != nil {
return err return err
} }
@@ -219,23 +177,14 @@ func (l *Config) SetSTSTrustedProxies(value string) error {
} }
// IsSTSTrustedProxy reports whether the peer IP is allowed to supply forwarded headers // IsSTSTrustedProxy reports whether the peer IP is allowed to supply forwarded headers
// for LDAP STS source bucketing. // for LDAP STS source bucketing. This allowlist covers STS login rate-limit
// bucketing only; the allowlist governing aws:SourceIp and the audit client
// address lives in internal/handlers.
func (l *Config) IsSTSTrustedProxy(peerIP string) bool { func (l *Config) IsSTSTrustedProxy(peerIP string) bool {
if l == nil || len(l.stsTrustedProxies) == 0 { if l == nil {
return false return false
} }
return l.stsTrustedProxies.Contains(peerIP)
addr, err := netip.ParseAddr(peerIP)
if err != nil {
return false
}
for _, prefix := range l.stsTrustedProxies {
if prefix.Contains(addr) {
return true
}
}
return false
} }
// Enabled returns if LDAP config is enabled. // Enabled returns if LDAP config is enabled.
+104
View File
@@ -0,0 +1,104 @@
// Copyright (c) 2015-2025 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package config
import (
"net/netip"
"strings"
)
// TrustedProxies is an allow-list of peer addresses whose forwarded headers may
// be believed. The zero value trusts nobody, which is the safe default: a
// request whose peer is not on the list is attributed to the peer itself rather
// than to whatever address the request claims.
type TrustedProxies []netip.Prefix
// ParseTrustedProxies parses a list of IP addresses and CIDR blocks separated by
// commas, semicolons or whitespace. A bare address is widened to a single-host
// prefix. what names the setting being parsed and is used in error messages.
//
// Catch-all prefixes are rejected rather than accepted, because a list
// containing 0.0.0.0/0 or ::/0 trusts every peer and so silently reinstates the
// behavior the allow-list exists to prevent.
func ParseTrustedProxies(value, what string) (TrustedProxies, error) {
fields := strings.FieldsFunc(value, func(r rune) bool {
switch r {
case ',', ';', ' ', '\n', '\r', '\t':
return true
default:
return false
}
})
if len(fields) == 0 {
return nil, nil
}
prefixes := make(TrustedProxies, 0, len(fields))
for _, field := range fields {
if prefix, err := netip.ParsePrefix(field); err == nil {
masked := prefix.Masked()
if masked.Bits() == 0 {
return nil, Errorf("%s %q is too broad", what, field)
}
prefixes = append(prefixes, masked)
continue
}
addr, err := netip.ParseAddr(field)
if err != nil {
return nil, Errorf("invalid %s %q", what, field)
}
bits := 32
if addr.Is6() {
bits = 128
}
prefixes = append(prefixes, netip.PrefixFrom(addr, bits))
}
return prefixes, nil
}
// Contains reports whether ip is covered by the allow-list. ip must already be a
// bare address; anything that does not parse is not trusted.
//
// Matching is deliberately literal. An entry written in IPv4-mapped form
// (::ffff:192.168.1.10) is a 128-bit prefix and will not match a peer presenting
// as 192.168.1.10, because netip.Prefix.Contains is false across differing
// widths - so such an entry is accepted and then matches nothing. Rewriting
// those entries to the IPv4 prefix they denote was tried and reverted: callers
// already reduce the address through net.ParseIP(...).String() before arriving
// here, which collapses the mapped form anyway, so the rewrite reached no real
// request and served only to change what this shared function means for the
// LDAP STS allow-list that was using it first. Write entries in plain form.
func (t TrustedProxies) Contains(ip string) bool {
if len(t) == 0 {
return false
}
addr, err := netip.ParseAddr(ip)
if err != nil {
return false
}
for _, prefix := range t {
if prefix.Contains(addr) {
return true
}
}
return false
}
+11
View File
@@ -181,6 +181,17 @@ func ipv6fix(clientIP string) string {
} }
func (rw *headerRewriter) Rewrite(req *http.Request) { func (rw *headerRewriter) Rewrite(req *http.Request) {
// Forwarding this request attaches the node's own identity to it, so anything
// the peer was not entitled to claim has to go first. X-Forwarded-For needs no
// such handling: the reverse proxy appends the peer we actually saw, and the
// receiving node reaches that entry before any the client injected to its
// left. X-Real-IP and RFC 7239 Forwarded carry no chain, so a client's copy
// would otherwise arrive at the next node vouched for by this one.
if !TrustsForwardedHeaders(req) {
req.Header.Del(xRealIP)
req.Header.Del(forwarded)
}
if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil { if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
clientIP = ipv6fix(clientIP) clientIP = ipv6fix(clientIP)
if req.Header.Get(xRealIP) == "" { if req.Header.Get(xRealIP) == "" {
+338 -14
View File
@@ -54,8 +54,116 @@ var (
protoRegex = regexp.MustCompile(`(?i)^(;|,| )+(?:proto=)(https|http)`) protoRegex = regexp.MustCompile(`(?i)^(;|,| )+(?:proto=)(https|http)`)
) )
// Used to disable all processing of the X-Forwarded-For header in source IP discovery. // Environment variables governing how much of a request's claimed source address
var enableXFFHeader = env.Get("_MINIO_API_XFF_HEADER", config.EnableOn) == config.EnableOn // the server is willing to believe.
const (
// EnvXFFHeader disables processing of X-Forwarded-For, and only of
// X-Forwarded-For. Inherited from upstream with its meaning deliberately
// unchanged: it is a parsing switch, not a trust boundary. Setting it to
// "off" still leaves X-Real-IP and RFC 7239 Forwarded honored, so it is not
// a way to stop a client naming its own address - EnvTrustedProxies is.
EnvXFFHeader = "_MINIO_API_XFF_HEADER"
// EnvTrustedProxies selects the trust policy. Unset honors forwarded headers
// from any peer, which is the historical behavior; TrustNoProxies believes
// none of them; anything else is a list of peer addresses and CIDR blocks
// whose headers are honored, which turns the source address from a claim any
// client can make into one only a named proxy can make.
EnvTrustedProxies = "MINIO_API_TRUSTED_PROXIES"
// TrustNoProxies is the EnvTrustedProxies value that believes no forwarded
// source-address header from anyone, whichever of the three it arrives in.
TrustNoProxies = "none"
)
// sourceIPTrust decides which peers may tell the server where a request came
// from. The address they choose becomes aws:SourceIp and the audit client
// address, so this is an access-control decision, not a logging preference.
type sourceIPTrust int
const (
// trustAnyPeer honors forwarded headers from whoever sent them. Historical
// default, sound only where every route to the API port passes through a
// proxy that overwrites those headers.
trustAnyPeer sourceIPTrust = iota
// trustNoPeer ignores forwarded headers; the source address is the TCP peer.
trustNoPeer
// trustListedPeers honors forwarded headers only from allow-listed peers.
trustListedPeers
)
var (
sourceIPPolicy sourceIPTrust
trustedProxies config.TrustedProxies
)
// enableXFFHeader carries upstream's X-Forwarded-For parsing switch. It applies
// within whichever trust policy is in force, and is orthogonal to it.
//
// Read at package initialisation, exactly as upstream does, and deliberately not
// re-read by ConfigureSourceIPTrust. Environment files are loaded after this
// point, so upstream silently ignores the setting when it is written there;
// picking it up would make an already-deployed setting start taking effect,
// which is a behavior change this fork has no reason to make on its way past.
var enableXFFHeader = env.Get(EnvXFFHeader, config.EnableOn) == config.EnableOn
// init establishes a policy from the process environment so that no code path
// runs without one. A server re-applies it from ConfigureSourceIPTrust once the
// environment is complete; an error here is dropped because the failure mode it
// leaves behind - trustNoPeer - is the safe one, and it is reported there.
func init() {
_ = ConfigureSourceIPTrust()
}
// ConfigureSourceIPTrust reads the trust policy out of the environment and
// installs it. The server calls this after loading MINIO_CONFIG_ENV_FILE, which
// happens long after package initialisation: a policy read only at init would
// miss every deployment that configures MinIO through an environment file and
// would silently leave the historical trust-any-peer mode in place.
//
// Not safe to call once requests are being served.
func ConfigureSourceIPTrust() error {
// Read through LookupEnv rather than env.Get, which discards the error from a
// remote env:// lookup and hands back the empty string. That would read here
// as "unset" and quietly reinstate the trust-any-peer default: a fetch that
// failed is not a statement that no proxy is trusted.
value, _, _, err := env.LookupEnv(EnvTrustedProxies)
if err != nil {
sourceIPPolicy, trustedProxies = trustNoPeer, nil
return config.Errorf("%s could not be read: %v", EnvTrustedProxies, err)
}
policy, prefixes, err := lookupSourceIPTrust(value)
sourceIPPolicy, trustedProxies = policy, prefixes
return err
}
// lookupSourceIPTrust derives the trust policy from EnvTrustedProxies. A
// malformed allow-list yields trustNoPeer alongside the error, so that a caller
// which fails to check the error still fails closed.
func lookupSourceIPTrust(proxies string) (sourceIPTrust, config.TrustedProxies, error) {
switch strings.ToLower(strings.TrimSpace(proxies)) {
case "":
return trustAnyPeer, nil, nil
case TrustNoProxies, config.EnableOff:
return trustNoPeer, nil, nil
}
prefixes, err := config.ParseTrustedProxies(proxies, EnvTrustedProxies)
if err != nil {
return trustNoPeer, nil, err
}
if len(prefixes) == 0 {
// Separators and nothing else. The value is not blank, so it was written
// on purpose, yet it names no proxy. Falling back to the permissive
// default here would answer a deliberate configuration with the one
// behavior it cannot have been asking for.
return trustNoPeer, nil, config.Errorf("%s %q names no proxy", EnvTrustedProxies, proxies)
}
return trustListedPeers, prefixes, nil
}
// GetSourceScheme retrieves the scheme from the X-Forwarded-Proto and RFC7239 // GetSourceScheme retrieves the scheme from the X-Forwarded-Proto and RFC7239
// Forwarded headers (in that order). // Forwarded headers (in that order).
@@ -85,20 +193,64 @@ func GetSourceScheme(r *http.Request) string {
return scheme return scheme
} }
// SECURITY NOTE: these headers are trusted from any peer. There is no // GetSourceIPFromHeaders retrieves the client address a request claims to come
// trusted-proxy boundary, X-Forwarded-For is honoured by default, and X-Real-IP // from, or the empty string when no claim may be believed and the caller should
// and Forwarded are not gated at all, so any client that can reach the server // fall back to the TCP peer.
// directly can set the address the rest of the process believes it came from.
// That includes aws:SourceIp, which means an IpAddress policy condition is not
// enforceable on a directly reachable deployment - put MinIO behind a proxy
// that overwrites these headers, or set _MINIO_API_XFF_HEADER=off and keep the
// other two out at the edge. Adding a trusted-proxy allowlist here would change
// what every deployment behind a load balancer resolves to, so it is recorded
// rather than changed.
// //
// GetSourceIPFromHeaders retrieves the IP from the X-Forwarded-For, X-Real-IP // SECURITY CONTRACT. The value returned here becomes aws:SourceIp and the audit
// and RFC7239 Forwarded headers (in that order) // log's client address, so whoever controls it controls both IP-based policy
// decisions and the attribution of every logged action. Which of the three
// interchangeable source-address headers - X-Forwarded-For, X-Real-IP, RFC 7239
// Forwarded - a client sends is irrelevant; they are equally forgeable, so the
// trust decision is taken over all three at once by EnvTrustedProxies:
//
// - Unset (default). Any peer may set the headers, and the left-most
// X-Forwarded-For entry wins. aws:SourceIp is then only as trustworthy as
// the network: any client that can open a connection to the API port can
// name its own address. An IpAddress condition is not enforceable under this
// mode unless every route to the port passes through a proxy that overwrites
// all three headers. Note that the stock nginx recipe,
// proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for, appends
// rather than overwrites, and so leaves a client-supplied entry in the
// left-most position where this mode will read it.
//
// - TrustNoProxies. No header is believed; the source address is always the
// TCP peer.
//
// - A list of addresses and CIDR blocks. Headers are believed only when the
// TCP peer is on the allow-list, and the chains are read right-to-left. This
// is the only mode under which aws:SourceIp is enforceable against a client
// with direct network access.
//
// EnvXFFHeader is not one of these modes. It suppresses parsing of
// X-Forwarded-For within whichever mode is in force, leaving X-Real-IP and
// Forwarded honored, so it cannot stop a client naming its own address - a
// client refused one header simply sends another. It is kept at its upstream
// meaning rather than widened into a trust switch, because widening it would
// change what an already-deployed setting resolves to; TrustNoProxies is the
// setting that means what it says.
//
// The scheme headers are deliberately not covered: GetSourceScheme feeds the
// Location URL rather than a policy decision, and suppressing it would hand
// http:// URLs to every deployment terminating TLS at a proxy.
func GetSourceIPFromHeaders(r *http.Request) string { func GetSourceIPFromHeaders(r *http.Request) string {
switch sourceIPPolicy {
case trustNoPeer:
return ""
case trustListedPeers:
if !peerMayForward(r) {
return ""
}
return forwardedSourceIP(r)
default:
return unverifiedSourceIP(r)
}
}
// unverifiedSourceIP reads the headers the way MinIO always has, taking the
// left-most X-Forwarded-For entry and falling back through X-Real-IP to RFC 7239
// Forwarded. Every value here is a claim by whoever sent it.
func unverifiedSourceIP(r *http.Request) string {
var addr string var addr string
if enableXFFHeader { if enableXFFHeader {
@@ -136,6 +288,178 @@ func GetSourceIPFromHeaders(r *http.Request) string {
return addr return addr
} }
// forwardedSourceIP resolves the client address for a request whose peer is an
// allow-listed proxy.
//
// The forwarding chains are read right-to-left, stepping over entries that name
// a configured proxy, and the first remaining address wins. That direction is
// what makes the header usable: each proxy appends the peer it actually saw, so
// an entry a client injected sits to the left of the one its proxy wrote, and
// the walk stops before reaching it.
//
// That holds only while the allow-list names proxies. A list broad enough to
// cover addresses clients also occupy makes those clients skippable too, and the
// walk then continues past a real client into whatever it placed to the left. A
// broad list therefore does not merely trust more peers - it lets those peers
// forge. Configure proxy addresses, not the subnet the proxies sit in.
//
// X-Real-IP carries no chain and so cannot be checked against the allow-list; it
// is taken at face value, and only when the chain headers yield nothing. The
// deployment contract is that a configured proxy overwrites whichever headers it
// sets. A proxy that instead relays a client's copy is choosing to let the
// client answer this question, and no amount of parsing here can undo that.
//
// Note this orders the headers differently from getSTSLDAPTrustedProxySourceIP
// (cmd/sts-handlers.go), which prefers X-Real-IP. Neither order is safe for
// every proxy - preferring X-Real-IP is wrong where the proxy authors only
// X-Forwarded-For and relays the client's X-Real-IP (AWS ALB), and preferring
// X-Forwarded-For is wrong in the mirror case (an nginx that sets only
// X-Real-IP). The chain-validated header is preferred here because this decides
// access control rather than rate-limit bucketing, so the value that can be
// checked against the allow-list should win; it also keeps the header precedence
// identical to the default mode. Deployments whose proxy authors only X-Real-IP
// must strip X-Forwarded-For at the edge.
func forwardedSourceIP(r *http.Request) string {
if enableXFFHeader {
if addr := untrustedHop(r.Header.Values(xForwardedFor), canonicalSourceIP); addr != "" {
return addr
}
}
if addr := canonicalSourceIP(lastValue(r.Header.Values(xRealIP))); addr != "" {
return addr
}
return untrustedHop(r.Header.Values(forwarded), forwardedForAddr)
}
// maxForwardedHops bounds how far back along a chain the walk will look.
//
// Real chains are a handful of hops and the answer sits at the right-hand end,
// so this is far above anything a deployment produces. It exists because the
// chain arrives from the network: without it, a client behind a trusted proxy
// could spend a megabyte of header on a walk this server has to finish. Running
// out of budget yields no address, so the request falls back to the peer - the
// same safe direction as a chain of entirely trusted hops.
const maxForwardedHops = 100
// lastValue returns the final line of a repeated header. X-Real-IP carries no
// chain to walk, so where a client's line and a proxy's line both survive, the
// later one is the one added closer to this server.
func lastValue(values []string) string {
if len(values) == 0 {
return ""
}
return values[len(values)-1]
}
// untrustedHop walks a forwarding chain from the right and returns the first
// address that is not itself a trusted hop, using addrOf to read one element.
//
// values holds the header's lines in the order received; repeated field lines
// are equivalent to one comma-joined line, and proxies disagree on which they
// emit - Go's reverse proxy and nginx rewrite a single line, while HAProxy's
// forwardfor adds a second. Reading only the first would leave a client's own
// line ahead of the proxy's, which is the position this walk exists to step over.
//
// The scan runs backwards over the raw text rather than over a split slice, so
// that a long chain costs no allocation.
func untrustedHop(values []string, addrOf func(string) string) string {
budget := maxForwardedHops
for i := len(values) - 1; i >= 0 && budget > 0; i-- {
for s := values[i]; len(s) > 0 && budget > 0; budget-- {
element := s
if comma := strings.LastIndexByte(s, ','); comma >= 0 {
element, s = s[comma+1:], s[:comma]
} else {
s = ""
}
if addr := addrOf(element); addr != "" && !isTrustedHop(addr) {
return addr
}
}
}
return ""
}
// forwardedForAddr reads the for= address out of one RFC 7239 Forwarded element.
func forwardedForAddr(element string) string {
match := forRegex.FindStringSubmatch(element)
if len(match) <= 1 {
return ""
}
return canonicalSourceIP(strings.Trim(match[1], `"`))
}
// canonicalSourceIP reduces one chain element to a bare IP address, or to the
// empty string when it does not hold one. Ports, brackets and surrounding space
// are stripped. Values an allow-list cannot reason about - a hostname, or an
// RFC 7239 obfuscated identifier such as for=_gazonk - are discarded rather than
// passed on, since a trust decision cannot be made about them.
func canonicalSourceIP(addr string) string {
addr = strings.TrimSpace(addr)
if addr == "" {
return ""
}
if host, _, err := net.SplitHostPort(addr); err == nil {
addr = host
}
addr = strings.TrimPrefix(addr, "[")
addr = strings.TrimSuffix(addr, "]")
// A link-local peer arrives with a zone ("fe80::1%eth0"), which net.ParseIP
// rejects outright - so without this the address resolves to nothing and the
// peer could never be a configured proxy.
if zone := strings.IndexByte(addr, '%'); zone != -1 {
addr = addr[:zone]
}
if ip := net.ParseIP(addr); ip != nil {
return ip.String()
}
return ""
}
// isTrustedHop reports whether a chain entry names one of the configured
// proxies, and so is an address to step over rather than attribute a request to.
//
// This deliberately does not extend the loopback exemption peerMayForward
// grants. Loopback is trusted as a *peer* because the local front-ends connect
// from there; a 127.0.0.1 entry inside a forwarding chain is just an address,
// and stepping over it would discard a real answer in favor of whatever sits
// further left.
func isTrustedHop(addr string) bool {
return trustedProxies.Contains(addr)
}
// peerMayForward reports whether the request's TCP peer is allowed to speak for
// someone else.
//
// Loopback is always allowed, configured or not. The FTP and SFTP front-ends
// reach the S3 layer over 127.0.0.1 and declare their session's client with
// X-Forwarded-For (see cmd/sftp-server-driver.go), so excluding loopback would
// attribute every FTP and SFTP request to the server itself.
func peerMayForward(r *http.Request) bool {
peer := canonicalSourceIP(r.RemoteAddr)
if peer == "" {
return false
}
if ip := net.ParseIP(peer); ip != nil && ip.IsLoopback() {
return true
}
return trustedProxies.Contains(peer)
}
// TrustsForwardedHeaders reports whether the source-address headers already
// present on a request may be believed. The node-to-node forwarder uses this to
// decide whether to relay what it received or overwrite it.
func TrustsForwardedHeaders(r *http.Request) bool {
switch sourceIPPolicy {
case trustNoPeer:
return false
case trustListedPeers:
return peerMayForward(r)
default:
return true
}
}
// GetSourceIPRaw retrieves the IP from the request headers // GetSourceIPRaw retrieves the IP from the request headers
// and falls back to r.RemoteAddr when necessary. // and falls back to r.RemoteAddr when necessary.
// however returns without bracketing. // however returns without bracketing.
+568
View File
@@ -19,7 +19,11 @@ package handlers
import ( import (
"net/http" "net/http"
"net/http/httptest"
"strings"
"testing" "testing"
"github.com/minio/minio/internal/config"
) )
type headerTest struct { type headerTest struct {
@@ -85,6 +89,33 @@ func TestGetSourceIP(t *testing.T) {
} }
} }
// withSourceIPTrust installs the trust policy the given EnvTrustedProxies value
// would produce and restores the previous one when the test ends.
func withSourceIPTrust(t *testing.T, proxies string) {
t.Helper()
policy, prefixes, err := lookupSourceIPTrust(proxies)
if err != nil {
t.Fatalf("%s=%q: unexpected error: %v", EnvTrustedProxies, proxies, err)
}
prevPolicy, prevProxies := sourceIPPolicy, trustedProxies
sourceIPPolicy, trustedProxies = policy, prefixes
t.Cleanup(func() {
sourceIPPolicy, trustedProxies = prevPolicy, prevProxies
})
}
func requestFrom(peer string, header http.Header) *http.Request {
if header == nil {
header = http.Header{}
}
return &http.Request{RemoteAddr: peer, Header: header}
}
// Upstream's test, unchanged: _MINIO_API_XFF_HEADER keeps its original meaning,
// suppressing X-Forwarded-For alone and leaving X-Real-IP to answer instead.
// That behavior is exactly why it is not the trust switch - see the test below.
func TestXFFDisabled(t *testing.T) { func TestXFFDisabled(t *testing.T) {
req := &http.Request{ req := &http.Request{
Header: http.Header{ Header: http.Header{
@@ -108,3 +139,540 @@ func TestXFFDisabled(t *testing.T) {
t.Errorf("wrong header, xff is disabled: got %s, want: 1.1.1.1", res) t.Errorf("wrong header, xff is disabled: got %s, want: 1.1.1.1", res)
} }
} }
// TestTrustNoProxiesIgnoresEveryForwardedHeader pins the property the setting
// exists for: an operator who turns forwarded-header trust off cannot be talked
// out of it by switching to another header. This is the guarantee
// _MINIO_API_XFF_HEADER=off does not provide, since suppressing one of three
// interchangeable headers only moves the answer to the next one.
func TestTrustNoProxiesIgnoresEveryForwardedHeader(t *testing.T) {
const peer = "203.0.113.9:44321"
forged := []headerTest{
{xForwardedFor, "8.8.8.8", "203.0.113.9"},
{xRealIP, "8.8.8.8", "203.0.113.9"},
{forwarded, "for=8.8.8.8", "203.0.113.9"},
}
// Default mode honors every one of them, which is the behavior being opted
// out of.
withSourceIPTrust(t, "")
for _, v := range forged {
if res := GetSourceIP(requestFrom(peer, http.Header{v.key: []string{v.val}})); res != "8.8.8.8" {
t.Errorf("%s: default mode should honor the header: got %s, want 8.8.8.8", v.key, res)
}
}
withSourceIPTrust(t, TrustNoProxies)
for _, v := range forged {
res := GetSourceIP(requestFrom(peer, http.Header{v.key: []string{v.val}}))
if res != v.expected {
t.Errorf("%s should be ignored under %s: got %s, want %s", v.key, TrustNoProxies, res, v.expected)
}
}
// All three at once, in case one merely shadows another.
res := GetSourceIP(requestFrom(peer, http.Header{
xForwardedFor: []string{"8.8.8.8"},
xRealIP: []string{"1.1.1.1"},
forwarded: []string{"for=9.9.9.9"},
}))
if res != "203.0.113.9" {
t.Errorf("wrong source with all headers set and no proxies trusted: got %s, want 203.0.113.9", res)
}
}
// TestTrustedProxiesResolution covers the allow-list mode, where the peer decides
// whether the request may speak for anyone else.
func TestTrustedProxiesResolution(t *testing.T) {
withSourceIPTrust(t, "10.0.0.0/8, 192.0.2.7")
tests := []struct {
name string
peer string
header http.Header
want string
}{{
name: "trusted proxy overwrote the header",
peer: "10.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"1.2.3.4"}},
want: "1.2.3.4",
}, {
// The stock nginx $proxy_add_x_forwarded_for recipe appends, so a client
// that sends its own X-Forwarded-For keeps the left-most slot. Reading
// right-to-left is what steps over it.
name: "client injected the left-most entry",
peer: "10.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"9.9.9.9, 1.2.3.4"}},
want: "1.2.3.4",
}, {
name: "two proxy hops after the client",
peer: "10.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"9.9.9.9, 1.2.3.4, 10.0.0.5"}},
want: "1.2.3.4",
}, {
name: "untrusted peer may not speak for anyone",
peer: "203.0.113.9:44321",
header: http.Header{xForwardedFor: []string{"1.2.3.4"}},
want: "203.0.113.9",
}, {
name: "untrusted peer cannot fall back to X-Real-IP either",
peer: "203.0.113.9:44321",
header: http.Header{xRealIP: []string{"1.2.3.4"}},
want: "203.0.113.9",
}, {
name: "untrusted peer cannot fall back to Forwarded either",
peer: "203.0.113.9:44321",
header: http.Header{forwarded: []string{"for=1.2.3.4"}},
want: "203.0.113.9",
}, {
name: "X-Real-IP from a trusted proxy",
peer: "192.0.2.7:9000",
header: http.Header{xRealIP: []string{"1.2.3.4"}},
want: "1.2.3.4",
}, {
// FTP and SFTP reach the S3 layer over loopback and declare the session's
// client with X-Forwarded-For.
name: "loopback front-end is trusted implicitly",
peer: "127.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"1.2.3.4"}},
want: "1.2.3.4",
}, {
name: "Forwarded chain is also read right-to-left",
peer: "10.0.0.1:9000",
header: http.Header{forwarded: []string{"for=9.9.9.9, for=1.2.3.4"}},
want: "1.2.3.4",
}, {
name: "unparseable chain entries are skipped",
peer: "10.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"_gazonk, workstation.local, 1.2.3.4"}},
want: "1.2.3.4",
}, {
name: "a chain of nothing but trusted hops falls back to the peer",
peer: "10.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"10.0.0.2, 10.0.0.3"}},
want: "10.0.0.1",
}, {
name: "X-Forwarded-For outranks X-Real-IP, as in the untrusted path",
peer: "10.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"1.2.3.4"}, xRealIP: []string{"9.9.9.9"}},
want: "1.2.3.4",
}, {
name: "IPv6 client keeps its bracketed form",
peer: "10.0.0.1:9000",
header: http.Header{xForwardedFor: []string{"2001:db8::1"}},
want: "[2001:db8::1]",
}, {
name: "no headers at all",
peer: "10.0.0.1:9000",
want: "10.0.0.1",
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if res := GetSourceIP(requestFrom(tt.peer, tt.header)); res != tt.want {
t.Errorf("got %s, want %s", res, tt.want)
}
})
}
}
// TestTrustedProxiesRepeatedHeaderLines covers proxies that add a second header
// line instead of extending the first. HAProxy's `option forwardfor` does this,
// and Header.Get would return only the client's line - putting the value the
// right-to-left walk exists to step over back in front of it.
func TestTrustedProxiesRepeatedHeaderLines(t *testing.T) {
withSourceIPTrust(t, "10.0.0.0/8")
tests := []struct {
name string
header http.Header
want string
}{{
name: "client line then proxy line",
header: http.Header{xForwardedFor: []string{"9.9.9.9", "1.2.3.4"}},
want: "1.2.3.4",
}, {
name: "client sends several lines",
header: http.Header{xForwardedFor: []string{"9.9.9.9", "8.8.8.8", "1.2.3.4"}},
want: "1.2.3.4",
}, {
name: "mixed: a comma chain and an added line",
header: http.Header{xForwardedFor: []string{"9.9.9.9, 8.8.8.8", "1.2.3.4"}},
want: "1.2.3.4",
}, {
name: "repeated Forwarded lines",
header: http.Header{forwarded: []string{"for=9.9.9.9", "for=1.2.3.4"}},
want: "1.2.3.4",
}, {
// The proxy's own line is the later one; a client's cannot displace it.
name: "repeated X-Real-IP takes the last line",
header: http.Header{xRealIP: []string{"9.9.9.9", "1.2.3.4"}},
want: "1.2.3.4",
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if res := GetSourceIP(requestFrom("10.0.0.1:9000", tt.header)); res != tt.want {
t.Errorf("got %s, want %s", res, tt.want)
}
})
}
}
// The chain arrives from the network, so the work it costs has to be bounded and
// must not scale into the heap. A client sitting behind a trusted proxy can make
// this header as long as the server's header limit allows.
func TestTrustedProxiesBoundsChainWork(t *testing.T) {
withSourceIPTrust(t, "10.0.0.0/8")
// The budget runs out before reaching the left-hand entry, so the request
// falls back to the peer - the same safe direction as an all-trusted chain.
hops := []string{"1.2.3.4"}
for range maxForwardedHops + 5 {
hops = append(hops, "10.0.0.9")
}
res := GetSourceIP(requestFrom("10.0.0.1:9000", http.Header{
xForwardedFor: []string{strings.Join(hops, ",")},
}))
if res != "10.0.0.1" {
t.Errorf("an over-long chain should fall back to the peer: got %s, want 10.0.0.1", res)
}
// A chain just inside the budget must still resolve normally.
shortEnough := []string{"1.2.3.4"}
for range 10 {
shortEnough = append(shortEnough, "10.0.0.9")
}
res = GetSourceIP(requestFrom("10.0.0.1:9000", http.Header{
xForwardedFor: []string{strings.Join(shortEnough, ",")},
}))
if res != "1.2.3.4" {
t.Errorf("a normal chain regressed: got %s, want 1.2.3.4", res)
}
// Scanning must happen in place. Splitting a megabyte of separators would
// turn one request's header into tens of megabytes of slice headers.
huge := []string{strings.Repeat(",", 1<<20)}
if allocs := testing.AllocsPerRun(2, func() {
untrustedHop(huge, canonicalSourceIP)
}); allocs > 0 {
t.Errorf("walking a chain allocated %v times; it must scan the header in place", allocs)
}
}
// TestTrustedProxiesDoNotChangeUntrustedMode guards the release-window promise:
// with neither setting present, resolution is exactly what it was before.
func TestTrustedProxiesDoNotChangeUntrustedMode(t *testing.T) {
withSourceIPTrust(t, "")
tests := []struct {
header http.Header
want string
}{
{http.Header{xForwardedFor: []string{"9.9.9.9, 1.2.3.4"}}, "9.9.9.9"},
{http.Header{xForwardedFor: []string{"_gazonk"}}, "_gazonk"},
{http.Header{xRealIP: []string{"workstation.local"}}, "workstation.local"},
{http.Header{forwarded: []string{`for="[2001:db8:cafe::17]:4711`}}, "[2001:db8:cafe::17]"},
}
for _, tt := range tests {
if res := GetSourceIP(requestFrom("203.0.113.9:44321", tt.header)); res != tt.want {
t.Errorf("untrusted mode changed for %v: got %s, want %s", tt.header, res, tt.want)
}
}
}
// A chain entry naming loopback is an ordinary address, not a hop to step over.
// Stepping over it would discard the real answer for whatever a client put to
// its left - the same failure a too-broad allow-list produces.
func TestTrustedProxiesDoesNotSkipLoopbackChainEntries(t *testing.T) {
withSourceIPTrust(t, "10.0.0.0/8")
res := GetSourceIP(requestFrom("10.0.0.1:9000", http.Header{
xForwardedFor: []string{"8.8.8.8, 127.0.0.1"},
}))
if res != "127.0.0.1" {
t.Errorf("loopback chain entry was skipped: got %s, want 127.0.0.1", res)
}
}
// A link-local peer presents with a zone. net.ParseIP rejects those outright, so
// without stripping it the peer resolves to nothing and could never be trusted.
func TestTrustedProxiesAcceptsZonedIPv6Peer(t *testing.T) {
withSourceIPTrust(t, "fe80::1")
res := GetSourceIP(requestFrom("[fe80::1%eth0]:9000", http.Header{
xForwardedFor: []string{"1.2.3.4"},
}))
if res != "1.2.3.4" {
t.Errorf("zoned IPv6 peer was not trusted: got %s, want 1.2.3.4", res)
}
}
// An allow-list entry written in IPv4-mapped form grants no trust: it is a
// 128-bit prefix, and the peer address has already been reduced to its plain
// form by the time it is matched. This is a wart, and it is deliberately left
// alone - rewriting such entries was tried and reverted, because the rewrite
// reached no real request and changed what the shared parser means for the LDAP
// STS allow-list that was using it first. The direction is fail-closed: the
// peer is simply not trusted, and the request is attributed to it.
func TestTrustedProxiesIgnoreMappedIPv4AllowListEntries(t *testing.T) {
withSourceIPTrust(t, "::ffff:192.168.1.10")
res := GetSourceIP(requestFrom("192.168.1.10:9000", http.Header{
xForwardedFor: []string{"1.2.3.4"},
}))
if res != "192.168.1.10" {
t.Errorf("a mapped entry granted trust: got %s, want the peer itself (192.168.1.10)", res)
}
}
// The policy has to be re-read after the server loads MINIO_CONFIG_ENV_FILE.
// Reading it only at package initialisation left every environment-file
// deployment on the historical trust-any-peer mode with nothing reported.
func TestConfigureSourceIPTrustReadsEnvironmentLate(t *testing.T) {
prevPolicy, prevProxies := sourceIPPolicy, trustedProxies
t.Cleanup(func() {
sourceIPPolicy, trustedProxies = prevPolicy, prevProxies
})
t.Setenv(EnvTrustedProxies, "10.0.0.0/8")
if err := ConfigureSourceIPTrust(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if sourceIPPolicy != trustListedPeers {
t.Fatalf("policy = %v, want trustListedPeers", sourceIPPolicy)
}
if res := GetSourceIP(requestFrom("203.0.113.9:44321", http.Header{
xForwardedFor: []string{"8.8.8.8"},
})); res != "203.0.113.9" {
t.Errorf("late-configured allow-list not in effect: got %s, want 203.0.113.9", res)
}
// A malformed value must be reported, not silently ignored.
t.Setenv(EnvTrustedProxies, "nonsense")
if err := ConfigureSourceIPTrust(); err == nil {
t.Error("malformed allow-list was accepted")
}
if sourceIPPolicy != trustNoPeer {
t.Errorf("policy = %v after a malformed value, want trustNoPeer", sourceIPPolicy)
}
}
// A value written as a remote env:// reference whose fetch fails must not be
// mistaken for an unset variable. env.Get drops that error and returns "", which
// would land on the permissive default at exactly the moment the operator's
// intent could not be read.
func TestConfigureSourceIPTrustRejectsUnreadableValue(t *testing.T) {
prevPolicy, prevProxies := sourceIPPolicy, trustedProxies
t.Cleanup(func() {
sourceIPPolicy, trustedProxies = prevPolicy, prevProxies
})
// Port 1 on loopback refuses immediately, and no cached _-prefixed fallback
// exists for this key.
t.Setenv(EnvTrustedProxies, "env://user:pass@127.0.0.1:1/webhook/v1/getenv")
if err := ConfigureSourceIPTrust(); err == nil {
t.Fatal("an unreadable value was accepted")
}
if sourceIPPolicy != trustAnyPeer {
return // failed closed, which is the point
}
t.Error("an unreadable value resolved to trustAnyPeer; any peer may forge the source address")
}
// _MINIO_API_XFF_HEADER must keep upstream's read timing. Upstream takes it at
// package initialisation, before environment files are loaded, so a deployment
// that writes it into MINIO_CONFIG_ENV_FILE has it silently ignored. Picking it
// up in ConfigureSourceIPTrust would make that already-deployed setting start
// taking effect, which is the one kind of change this rework exists to avoid.
func TestConfigureSourceIPTrustLeavesXFFSwitchAlone(t *testing.T) {
prevXFF := enableXFFHeader
t.Cleanup(func() { enableXFFHeader = prevXFF })
enableXFFHeader = true
t.Setenv(EnvXFFHeader, config.EnableOff)
if err := ConfigureSourceIPTrust(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !enableXFFHeader {
t.Error("the X-Forwarded-For switch was re-read from the environment")
}
}
// forwardThroughNode sends a client request through the node-to-node forwarder
// and reports what the receiving node resolves as the source address. The
// receiving node's peer is the forwarding node, not the client, which is where
// the trust policy has its least obvious consequence.
func forwardThroughNode(t *testing.T, proxies, clientPeer, forwardingNode string) string {
t.Helper()
withSourceIPTrust(t, proxies)
var resolved string
receiving := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
// The test client's real peer is loopback, which is trusted
// unconditionally; substitute the address the forwarding node would
// present so the allow-list decision is the one under test.
r.RemoteAddr = forwardingNode
resolved = GetSourceIPRaw(r)
}))
defer receiving.Close()
in := httptest.NewRequest(http.MethodGet, receiving.URL+"/probe", nil)
in.RemoteAddr = clientPeer
in.RequestURI = "/probe"
in.URL.Scheme = "http"
in.URL.Host = strings.TrimPrefix(receiving.URL, "http://")
NewForwarder(&Forwarder{PassHost: true}).ServeHTTP(httptest.NewRecorder(), in)
return resolved
}
func TestForwardedBetweenNodesAttribution(t *testing.T) {
const (
client = "203.0.113.77:44321"
forwardingNode = "10.10.0.2:36000"
)
tests := []struct {
name string
proxies string
want string
}{{
name: "default mode carries the client through",
want: "203.0.113.77",
}, {
// Believing no header means believing nothing about the forwarding node
// either, so an internally forwarded request is attributed to it. There
// is no configuration that corrects this, which is why the documentation
// steers multi-node clusters to the allow-list instead.
name: "trusting nobody attributes to the forwarding node",
proxies: TrustNoProxies,
want: "10.10.0.2",
}, {
name: "allow-list omitting the cluster's own nodes",
proxies: "192.168.1.0/24",
want: "10.10.0.2",
}, {
name: "allow-list including the cluster's own nodes",
proxies: "192.168.1.0/24,10.10.0.2",
want: "203.0.113.77",
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := forwardThroughNode(t, tt.proxies, client, forwardingNode); got != tt.want {
t.Errorf("receiving node resolved %q, want %q", got, tt.want)
}
})
}
}
func TestLookupSourceIPTrust(t *testing.T) {
tests := []struct {
name string
proxies string
want sourceIPTrust
wantErr bool
}{
{name: "unset is the historical default", want: trustAnyPeer},
{name: "none", proxies: TrustNoProxies, want: trustNoPeer},
{name: "none is case-insensitive", proxies: "None", want: trustNoPeer},
{name: "off is accepted as a synonym", proxies: "off", want: trustNoPeer},
{name: "surrounding space is ignored", proxies: " none ", want: trustNoPeer},
{name: "allow-list", proxies: "10.0.0.0/8", want: trustListedPeers},
{name: "whitespace only is indistinguishable from unset", proxies: " ", want: trustAnyPeer},
// Written on purpose, but naming nobody. Falling back to the permissive
// default would be the one answer the operator cannot have wanted.
{name: "separators only rejected", proxies: ",", want: trustNoPeer, wantErr: true},
{name: "assorted separators rejected", proxies: " ,; ", want: trustNoPeer, wantErr: true},
// A catch-all would trust every peer and quietly undo the allow-list.
{name: "catch-all v4 rejected", proxies: "0.0.0.0/0", want: trustNoPeer, wantErr: true},
{name: "catch-all v6 rejected", proxies: "::/0", want: trustNoPeer, wantErr: true},
{name: "catch-all hidden in a list", proxies: "10.0.0.0/8,0.0.0.0/0", want: trustNoPeer, wantErr: true},
// A mapped spelling of the whole IPv4 space stays a /96 here, and so is
// accepted rather than rejected. It matches nothing, because a peer never
// reaches Contains in mapped form - the breadth guard covers what an
// operator would actually write, not every way to write it.
{name: "mapped whole-IPv4 space is a /96, not a catch-all", proxies: "::ffff:0:0/96", want: trustListedPeers},
{name: "garbage rejected", proxies: "10.0.0.0/8,nonsense", want: trustNoPeer, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, _, err := lookupSourceIPTrust(tt.proxies)
if (err != nil) != tt.wantErr {
t.Fatalf("error = %v, wantErr %v", err, tt.wantErr)
}
// A rejected list must fail closed, not fall back to trusting everyone.
if got != tt.want {
t.Fatalf("policy = %v, want %v", got, tt.want)
}
})
}
}
func TestCanonicalSourceIP(t *testing.T) {
tests := []struct{ in, want string }{
{"1.2.3.4", "1.2.3.4"},
{" 1.2.3.4 ", "1.2.3.4"},
{"1.2.3.4:443", "1.2.3.4"},
{"2001:db8::1", "2001:db8::1"},
{"[2001:db8::1]", "2001:db8::1"},
{"[2001:db8::1]:443", "2001:db8::1"},
{"::ffff:1.2.3.4", "1.2.3.4"},
{"_gazonk", ""},
{"workstation.local", ""},
{"", ""},
}
for _, tt := range tests {
if got := canonicalSourceIP(tt.in); got != tt.want {
t.Errorf("canonicalSourceIP(%q) = %q, want %q", tt.in, got, tt.want)
}
}
}
// TestRewriteDropsUnvouchedClaims covers the node-to-node forwarder. Relaying a
// header the peer was not entitled to set would hand the client this node's
// authority at the next hop, which does trust it.
func TestRewriteDropsUnvouchedClaims(t *testing.T) {
rw := &headerRewriter{}
withSourceIPTrust(t, "10.0.0.0/8")
untrusted := requestFrom("203.0.113.9:44321", http.Header{
xRealIP: []string{"1.2.3.4"},
forwarded: []string{"for=1.2.3.4"},
})
rw.Rewrite(untrusted)
if got := untrusted.Header.Get(xRealIP); got != "203.0.113.9" {
t.Errorf("X-Real-IP from an untrusted peer should be replaced: got %s, want 203.0.113.9", got)
}
if got := untrusted.Header.Get(forwarded); got != "" {
t.Errorf("Forwarded from an untrusted peer should be dropped: got %s", got)
}
// A trusted proxy's own chain must survive, or a two-hop deployment loses the
// client it correctly identified.
trusted := requestFrom("10.0.0.1:9000", http.Header{
xRealIP: []string{"1.2.3.4"},
forwarded: []string{"for=1.2.3.4"},
})
rw.Rewrite(trusted)
if got := trusted.Header.Get(xRealIP); got != "1.2.3.4" {
t.Errorf("X-Real-IP from a trusted peer should be relayed: got %s, want 1.2.3.4", got)
}
if got := trusted.Header.Get(forwarded); got != "for=1.2.3.4" {
t.Errorf("Forwarded from a trusted peer should be relayed: got %s", got)
}
// Default mode must behave exactly as it did: fill in only what is missing.
withSourceIPTrust(t, "")
legacy := requestFrom("203.0.113.9:44321", http.Header{xRealIP: []string{"1.2.3.4"}})
rw.Rewrite(legacy)
if got := legacy.Header.Get(xRealIP); got != "1.2.3.4" {
t.Errorf("default mode should relay X-Real-IP untouched: got %s, want 1.2.3.4", got)
}
}