astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-09-13 05:54:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/access-based-ilm

Browse Source
This commit is contained in:
Feng Ruohang
2026-09-08 13:04:25 +08:00
committed by GitHub
parent 7a060cab1e 3abe0d95a5
commit 89b75e7913
308 changed files with 25104 additions and 12023 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/bucket/cors/cors.go
+364
View File
@@ -0,0 +1,364 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package cors implements the S3 per-bucket CORS configuration type,
// its validation, and origin/method/header matching helpers.
package cors
import (
"encoding/xml"
"errors"
"fmt"
"io"
"strconv"
"strings"
"unicode/utf8"
)
// maxCORSRules is the maximum number of rules allowed per bucket (AWS S3 limit).
const maxCORSRules = 100
// maxCORSRuleIDLen is the maximum length of a CORSRule <ID> (AWS S3 limit).
const maxCORSRuleIDLen = 255
// maxCORSMaxAgeSeconds is the largest value representable by the int32
// MaxAgeSeconds shape used by the S3 API model.
const maxCORSMaxAgeSeconds = 1<<31 - 1
// supportedMethods are the HTTP methods permitted in an AllowedMethod element.
var supportedMethods = map[string]bool{
"GET": true,
"PUT": true,
"HEAD": true,
"POST": true,
"DELETE": true,
}
// Config is the S3 <CORSConfiguration> document.
type Config struct {
XMLName xml.Name `xml:"CORSConfiguration"`
CORSRules []Rule `xml:"CORSRule"`
}
// Rule is a single <CORSRule>.
type Rule struct {
ID string `xml:"ID,omitempty"`
AllowedHeaders []string `xml:"AllowedHeader"`
AllowedMethods []string `xml:"AllowedMethod"`
AllowedOrigins []string `xml:"AllowedOrigin"`
ExposeHeaders []string `xml:"ExposeHeader"`
MaxAgeSeconds int `xml:"MaxAgeSeconds"`
maxAgeSecondsSet bool
}
type corsXMLUnknown struct {
XMLName xml.Name
}
type corsXMLValue struct {
Text string `xml:",chardata"`
Unknown []corsXMLUnknown `xml:",any"`
}
type configXML struct {
XMLName xml.Name `xml:"CORSConfiguration"`
CORSRules []ruleXML `xml:"CORSRule"`
Text string `xml:",chardata"`
Unknown []corsXMLUnknown `xml:",any"`
}
type ruleXML struct {
ID []corsXMLValue `xml:"ID"`
AllowedHeaders []corsXMLValue `xml:"AllowedHeader"`
AllowedMethods []corsXMLValue `xml:"AllowedMethod"`
AllowedOrigins []corsXMLValue `xml:"AllowedOrigin"`
ExposeHeaders []corsXMLValue `xml:"ExposeHeader"`
MaxAgeSeconds []corsXMLValue `xml:"MaxAgeSeconds"`
Text string `xml:",chardata"`
Unknown []corsXMLUnknown `xml:",any"`
}
// ParseBucketCorsConfig parses a CORS configuration from the given reader.
func ParseBucketCorsConfig(r io.Reader) (*Config, error) {
var parsed configXML
decoder := xml.NewDecoder(r)
if err := decoder.Decode(&parsed); err != nil {
return nil, err
}
if strings.TrimSpace(parsed.Text) != "" {
return nil, xml.UnmarshalError("unexpected character data in CORSConfiguration")
}
if len(parsed.Unknown) > 0 {
return nil, xml.UnmarshalError(fmt.Sprintf("unexpected element <%s> in CORSConfiguration", parsed.Unknown[0].XMLName.Local))
}
c := Config{
XMLName: parsed.XMLName,
CORSRules: make([]Rule, len(parsed.CORSRules)),
}
for i := range parsed.CORSRules {
rule, err := parseCORSRuleXML(parsed.CORSRules[i])
if err != nil {
return nil, err
}
c.CORSRules[i] = rule
}
// Decode consumes one document element. Only XML whitespace, comments, and
// processing instructions are permitted after it.
for {
token, err := decoder.Token()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
switch token := token.(type) {
case xml.CharData:
if strings.TrimSpace(string(token)) == "" {
continue
}
case xml.Comment, xml.ProcInst:
continue
}
return nil, errors.New("unexpected XML content after CORSConfiguration")
}
return &c, nil
}
func parseCORSRuleXML(parsed ruleXML) (Rule, error) {
if strings.TrimSpace(parsed.Text) != "" {
return Rule{}, xml.UnmarshalError("unexpected character data in CORSRule")
}
if len(parsed.Unknown) > 0 {
return Rule{}, xml.UnmarshalError(fmt.Sprintf("unexpected element <%s> in CORSRule", parsed.Unknown[0].XMLName.Local))
}
if len(parsed.ID) > 1 {
return Rule{}, xml.UnmarshalError("duplicate ID element in CORSRule")
}
if len(parsed.MaxAgeSeconds) > 1 {
return Rule{}, xml.UnmarshalError("duplicate MaxAgeSeconds element in CORSRule")
}
rule := Rule{}
var err error
if len(parsed.ID) == 1 {
if rule.ID, err = corsXMLText("ID", parsed.ID[0]); err != nil {
return Rule{}, err
}
}
if rule.AllowedHeaders, err = corsXMLTexts("AllowedHeader", parsed.AllowedHeaders); err != nil {
return Rule{}, err
}
if rule.AllowedMethods, err = corsXMLTexts("AllowedMethod", parsed.AllowedMethods); err != nil {
return Rule{}, err
}
if rule.AllowedOrigins, err = corsXMLTexts("AllowedOrigin", parsed.AllowedOrigins); err != nil {
return Rule{}, err
}
if rule.ExposeHeaders, err = corsXMLTexts("ExposeHeader", parsed.ExposeHeaders); err != nil {
return Rule{}, err
}
if len(parsed.MaxAgeSeconds) == 1 {
value, valueErr := corsXMLText("MaxAgeSeconds", parsed.MaxAgeSeconds[0])
if valueErr != nil {
return Rule{}, valueErr
}
age, parseErr := strconv.ParseInt(strings.TrimSpace(value), 10, 32)
if parseErr != nil {
return Rule{}, xml.UnmarshalError("invalid MaxAgeSeconds value")
}
rule.MaxAgeSeconds = int(age)
rule.maxAgeSecondsSet = true
}
return rule, nil
}
func corsXMLTexts(name string, values []corsXMLValue) ([]string, error) {
result := make([]string, len(values))
for i := range values {
value, err := corsXMLText(name, values[i])
if err != nil {
return nil, err
}
result[i] = value
}
return result, nil
}
func corsXMLText(name string, value corsXMLValue) (string, error) {
if len(value.Unknown) > 0 {
return "", xml.UnmarshalError(fmt.Sprintf("element <%s> must not contain child element <%s>", name, value.Unknown[0].XMLName.Local))
}
return value.Text, nil
}
// Validate checks the config against the S3 constraints.
func (c *Config) Validate() error {
if len(c.CORSRules) == 0 {
return errors.New("CORSConfiguration must contain at least one rule")
}
if len(c.CORSRules) > maxCORSRules {
return errors.New("CORSConfiguration exceeds the maximum number of rules")
}
for _, r := range c.CORSRules {
if !utf8.ValidString(r.ID) {
return errors.New("CORSRule ID must contain valid UTF-8")
}
if utf8.RuneCountInString(r.ID) > maxCORSRuleIDLen {
return errors.New("CORSRule ID exceeds the maximum length of 255 characters")
}
if len(r.AllowedOrigins) == 0 {
return errors.New("CORSRule must contain at least one AllowedOrigin")
}
if len(r.AllowedMethods) == 0 {
return errors.New("CORSRule must contain at least one AllowedMethod")
}
for _, o := range r.AllowedOrigins {
if o == "" {
return errors.New("AllowedOrigin must not be empty")
}
if strings.Contains(o, "?") {
return errors.New("AllowedOrigin may not contain wildcard '?': " + o)
}
if strings.Count(o, "*") > 1 {
return errors.New("AllowedOrigin may contain at most one wildcard '*': " + o)
}
}
for _, m := range r.AllowedMethods {
if !supportedMethods[m] {
return errors.New("unsupported method in CORSRule: " + m)
}
}
for _, h := range r.AllowedHeaders {
if h == "" {
return errors.New("AllowedHeader must not be empty")
}
if strings.Contains(h, "?") {
return errors.New("AllowedHeader may not contain wildcard '?': " + h)
}
if strings.Count(h, "*") > 1 {
return errors.New("AllowedHeader may contain at most one wildcard '*': " + h)
}
}
for _, h := range r.ExposeHeaders {
if h == "" {
return errors.New("ExposeHeader must not be empty")
}
}
if r.MaxAgeSeconds < 0 {
return errors.New("MaxAgeSeconds must not be negative")
}
if int64(r.MaxAgeSeconds) > maxCORSMaxAgeSeconds {
return errors.New("MaxAgeSeconds exceeds the maximum S3 integer value")
}
}
return nil
}
func matchSingleWildcard(pattern, value string) bool {
prefix, suffix, found := strings.Cut(pattern, "*")
if !found {
return pattern == value
}
return len(value) >= len(prefix)+len(suffix) &&
strings.HasPrefix(value, prefix) && strings.HasSuffix(value, suffix)
}
func (r Rule) matchAllowedOrigin(origin string) (string, bool) {
for _, allowedOrigin := range r.AllowedOrigins {
if matchSingleWildcard(allowedOrigin, origin) {
return allowedOrigin, true
}
}
return "", false
}
// HasAllowedMethod reports whether the rule allows the given HTTP method.
func (r Rule) HasAllowedMethod(method string) bool {
for _, m := range r.AllowedMethods {
if m == method {
return true
}
}
return false
}
// FilterAllowedHeaders returns the subset of reqHeaders permitted by the rule
// and whether every requested header was allowed.
func (r Rule) FilterAllowedHeaders(reqHeaders []string) ([]string, bool) {
var allowed []string
for _, h := range reqHeaders {
h = strings.TrimSpace(h)
if h == "" {
continue
}
if !r.headerAllowed(h) {
return nil, false
}
allowed = append(allowed, h)
}
return allowed, true
}
func (r Rule) headerAllowed(header string) bool {
for _, h := range r.AllowedHeaders {
if matchSingleWildcard(strings.ToLower(h), strings.ToLower(header)) {
return true
}
}
return false
}
// MatchRule returns the first rule whose origin and method both match, along
// with the configured origin pattern that matched.
func (c *Config) MatchRule(origin, method string) (rule *Rule, allowedOrigin string, ok bool) {
for i := range c.CORSRules {
r := &c.CORSRules[i]
matchedOrigin, originOK := r.matchAllowedOrigin(origin)
if originOK && r.HasAllowedMethod(method) {
return r, matchedOrigin, true
}
}
return nil, "", false
}
// MatchPreflight returns the first rule whose origin and method match and
// whose AllowedHeaders permit every header in reqHeaders. Unlike MatchRule,
// this keeps evaluating subsequent rules until one fully satisfies the
// preflight request, since an earlier origin/method match with a more
// restrictive header list must not shadow a later, more permissive rule.
func (c *Config) MatchPreflight(origin, method string, reqHeaders []string) (rule *Rule, allowedOrigin string, allowedHeaders []string, maxAgeSeconds *int, ok bool) {
for i := range c.CORSRules {
r := &c.CORSRules[i]
matchedOrigin, originOK := r.matchAllowedOrigin(origin)
if !originOK || !r.HasAllowedMethod(method) {
continue
}
allowed, headersOK := r.FilterAllowedHeaders(reqHeaders)
if !headersOK {
continue
}
if r.maxAgeSecondsSet || r.MaxAgeSeconds != 0 {
maxAgeSeconds = &r.MaxAgeSeconds
}
return r, matchedOrigin, allowed, maxAgeSeconds, true
}
return nil, "", nil, nil, false
}
internal/bucket/cors/cors_adversarial_test.go
+222
View File
@@ -0,0 +1,222 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
package cors
import (
"strconv"
"strings"
"testing"
)
func TestParseStandardS3Namespace(t *testing.T) {
doc := `<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><CORSRule><AllowedOrigin>https://app.example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`
cfg, err := ParseBucketCorsConfig(strings.NewReader(doc))
if err != nil {
t.Fatal(err)
}
if err = cfg.Validate(); err != nil {
t.Fatal(err)
}
if _, _, ok := cfg.MatchRule("https://app.example.com", "GET"); !ok {
t.Fatal("standard S3 namespace document did not produce a matching rule")
}
}
const minimalCORSConfig = `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`
func TestParseRejectsTrailingXMLRoot(t *testing.T) {
for name, suffix := range map[string]string{
"second root": `<Extra/>`,
"text": `junk`,
"dangling close": `</Extra>`,
} {
t.Run(name, func(t *testing.T) {
if _, err := ParseBucketCorsConfig(strings.NewReader(minimalCORSConfig + suffix)); err == nil {
t.Fatalf("expected trailing %s to be rejected", name)
}
})
}
}
func TestParseAllowsXMLMiscAfterRoot(t *testing.T) {
for name, suffix := range map[string]string{
"whitespace": " \n\t",
"comment": `<!-- trailing comment -->`,
"processing instruction": `<?cors-test done?>`,
} {
t.Run(name, func(t *testing.T) {
if _, err := ParseBucketCorsConfig(strings.NewReader(minimalCORSConfig + suffix)); err != nil {
t.Fatalf("valid trailing XML misc was rejected: %v", err)
}
})
}
}
func TestValidateCORSRuleIDCountsCharacters(t *testing.T) {
doc := `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("界", 255) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`
cfg, err := ParseBucketCorsConfig(strings.NewReader(doc))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if err = cfg.Validate(); err != nil {
t.Fatalf("255-character rule ID must be accepted: %v", err)
}
cfg.CORSRules[0].ID += "界"
if err = cfg.Validate(); err == nil {
t.Fatal("256-character rule ID must be rejected")
}
}
func TestValidateRejectsNonCanonicalAllowedMethod(t *testing.T) {
doc := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>get</AllowedMethod></CORSRule></CORSConfiguration>`
cfg, err := ParseBucketCorsConfig(strings.NewReader(doc))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if err = cfg.Validate(); err == nil {
t.Fatal("expected lowercase AllowedMethod to be rejected")
}
}
func TestAllowedMethodMatchingIsCaseSensitive(t *testing.T) {
rule := Rule{AllowedMethods: []string{"GET"}}
if !rule.HasAllowedMethod("GET") {
t.Fatal("expected canonical GET to match")
}
if rule.HasAllowedMethod("get") {
t.Fatal("lowercase request method must not match canonical GET")
}
}
func TestParseRejectsElementsOutsideCORSShape(t *testing.T) {
tests := map[string]string{
"unknown root child": `<CORSConfiguration><Unknown/><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
"unknown rule child": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><Unknown/></CORSRule></CORSConfiguration>`,
"nested origin child": `<CORSConfiguration><CORSRule><AllowedOrigin><Unknown/></AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
"duplicate id": `<CORSConfiguration><CORSRule><ID>a</ID><ID>b</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
"duplicate max age": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>1</MaxAgeSeconds><MaxAgeSeconds>2</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
"empty max age": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds/></CORSRule></CORSConfiguration>`,
"overflow max age": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>2147483648</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
}
for name, doc := range tests {
t.Run(name, func(t *testing.T) {
if _, err := ParseBucketCorsConfig(strings.NewReader(doc)); err == nil {
t.Fatal("expected parse error")
}
})
}
}
func TestMaxAgeSecondsPresence(t *testing.T) {
tests := []struct {
name string
element string
value int
present bool
}{
{name: "absent"},
{name: "zero", element: `<MaxAgeSeconds>0</MaxAgeSeconds>`, present: true},
{name: "positive", element: `<MaxAgeSeconds>3000</MaxAgeSeconds>`, value: 3000, present: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
doc := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod>` + tt.element + `</CORSRule></CORSConfiguration>`
cfg, err := ParseBucketCorsConfig(strings.NewReader(doc))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
rule := cfg.CORSRules[0]
_, _, _, maxAgeSeconds, ok := cfg.MatchPreflight("https://example.com", "GET", nil)
if !ok {
t.Fatal("expected rule to match")
}
present := maxAgeSeconds != nil
if rule.MaxAgeSeconds != tt.value || present != tt.present {
t.Fatalf("MaxAgeSeconds = %d, present = %v", rule.MaxAgeSeconds, present)
}
})
}
}
func TestValidateRuleCountBoundary(t *testing.T) {
rule := Rule{AllowedOrigins: []string{"*"}, AllowedMethods: []string{"GET"}}
cfg := Config{CORSRules: make([]Rule, 100)}
for i := range cfg.CORSRules {
cfg.CORSRules[i] = rule
}
if err := cfg.Validate(); err != nil {
t.Fatalf("100 rules must be accepted: %v", err)
}
cfg.CORSRules = append(cfg.CORSRules, rule)
if err := cfg.Validate(); err == nil {
t.Fatal("101 rules must be rejected")
}
}
func TestValidateMaxAgeSecondsBoundary(t *testing.T) {
cfg := Config{CORSRules: []Rule{{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET"},
MaxAgeSeconds: maxCORSMaxAgeSeconds,
}}}
if err := cfg.Validate(); err != nil {
t.Fatalf("MaxAgeSeconds int32 maximum must be accepted: %v", err)
}
if strconv.IntSize > 32 {
overflow := int64(maxCORSMaxAgeSeconds) + 1
cfg.CORSRules[0].MaxAgeSeconds = int(overflow)
if err := cfg.Validate(); err == nil {
t.Fatal("MaxAgeSeconds above int32 maximum must be rejected")
}
}
}
func TestSingleWildcardMatching(t *testing.T) {
tests := []struct {
pattern string
value string
want bool
}{
{"*", "https://example.com", true},
{"https://*.example.com", "https://api.example.com", true},
{"https://*.example.com", "https://.example.com", true},
{"https://*.example.com", "http://api.example.com", false},
{"https://?.example.com", "https://a.example.com", false},
}
for _, tt := range tests {
if got := matchSingleWildcard(tt.pattern, tt.value); got != tt.want {
t.Errorf("matchSingleWildcard(%q, %q) = %v, want %v", tt.pattern, tt.value, got, tt.want)
}
}
}
func TestMatchRuleReturnsMatchedOriginPattern(t *testing.T) {
cfg := Config{CORSRules: []Rule{{
AllowedOrigins: []string{"https://app.example.com", "https://*", "*"},
AllowedMethods: []string{"GET"},
}}}
tests := []struct {
origin string
want string
}{
{"https://app.example.com", "https://app.example.com"},
{"https://other.example.com", "https://*"},
{"http://other.example.com", "*"},
}
for _, tt := range tests {
_, got, ok := cfg.MatchRule(tt.origin, "GET")
if !ok || got != tt.want {
t.Errorf("origin %q matched %q, ok=%v; want %q", tt.origin, got, ok, tt.want)
}
}
}
internal/bucket/cors/cors_test.go
+170
View File
@@ -0,0 +1,170 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cors
import (
"strings"
"testing"
)
const sampleCORS = `<CORSConfiguration>
<CORSRule>
<ID>rule1</ID>
<AllowedOrigin>http://www.example.com</AllowedOrigin>
<AllowedOrigin>https://*.example.org</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedMethod>PUT</AllowedMethod>
<AllowedHeader>x-amz-*</AllowedHeader>
<ExposeHeader>ETag</ExposeHeader>
<MaxAgeSeconds>3000</MaxAgeSeconds>
</CORSRule>
</CORSConfiguration>`
func TestParseAndValidate(t *testing.T) {
c, err := ParseBucketCorsConfig(strings.NewReader(sampleCORS))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if err := c.Validate(); err != nil {
t.Fatalf("validate failed: %v", err)
}
if len(c.CORSRules) != 1 {
t.Fatalf("expected 1 rule, got %d", len(c.CORSRules))
}
if c.CORSRules[0].MaxAgeSeconds != 3000 {
t.Fatalf("MaxAgeSeconds mismatch: %d", c.CORSRules[0].MaxAgeSeconds)
}
}
func TestValidateRejections(t *testing.T) {
cases := map[string]string{
"bad method": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>TRACE</AllowedMethod></CORSRule></CORSConfiguration>`,
"no origin": `<CORSConfiguration><CORSRule><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
"empty origin": `<CORSConfiguration><CORSRule><AllowedOrigin></AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
"no method": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin></CORSRule></CORSConfiguration>`,
"negative age": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>-1</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
"multi wildcard origin": `<CORSConfiguration><CORSRule><AllowedOrigin>https://*.*.example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
"multi wildcard header": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedHeader>x-*-*</AllowedHeader></CORSRule></CORSConfiguration>`,
"question mark origin": `<CORSConfiguration><CORSRule><AllowedOrigin>https://?.example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
"question mark header": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedHeader>x-amz-?</AllowedHeader></CORSRule></CORSConfiguration>`,
"empty allowed header": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedHeader/></CORSRule></CORSConfiguration>`,
"empty expose header": `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><ExposeHeader/></CORSRule></CORSConfiguration>`,
"overlong id": `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("a", 256) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
}
for name, doc := range cases {
c, err := ParseBucketCorsConfig(strings.NewReader(doc))
if err != nil {
continue // parse-level rejection is acceptable
}
if err := c.Validate(); err == nil {
t.Errorf("%s: expected validation error, got nil", name)
}
}
}
func TestMatching(t *testing.T) {
c, _ := ParseBucketCorsConfig(strings.NewReader(sampleCORS))
rule, _, ok := c.MatchRule("https://api.example.org", "GET")
if !ok {
t.Fatal("expected origin+method to match")
}
if _, _, ok := c.MatchRule("http://evil.com", "GET"); ok {
t.Fatal("did not expect match for disallowed origin")
}
if _, _, ok := c.MatchRule("http://www.example.com", "DELETE"); ok {
t.Fatal("did not expect match for disallowed method")
}
allowed, ok := rule.FilterAllowedHeaders([]string{"x-amz-date", "x-amz-content-sha256"})
if !ok || len(allowed) != 2 {
t.Fatalf("expected both headers allowed via wildcard, got %v ok=%v", allowed, ok)
}
if _, ok := rule.FilterAllowedHeaders([]string{"authorization"}); ok {
t.Fatal("did not expect authorization to be allowed")
}
}
func TestMatchPreflightFallsThroughToLaterRule(t *testing.T) {
// Rule A matches origin+method but only allows a restrictive header set.
// Rule B, listed after A, matches the same origin+method and allows any
// header. A preflight requesting a header only B permits must not be
// rejected just because A was tried first.
const doc = `<CORSConfiguration>
<CORSRule>
<ID>A-restrictive</ID>
<AllowedOrigin>https://app.example.com</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedHeader>x-amz-date</AllowedHeader>
</CORSRule>
<CORSRule>
<ID>B-permissive</ID>
<AllowedOrigin>https://app.example.com</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<AllowedHeader>*</AllowedHeader>
</CORSRule>
</CORSConfiguration>`
c, err := ParseBucketCorsConfig(strings.NewReader(doc))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
rule, _, allowed, _, ok := c.MatchPreflight("https://app.example.com", "GET", []string{"x-custom-header"})
if !ok {
t.Fatal("expected MatchPreflight to succeed via the later, permissive rule")
}
if rule.ID != "B-permissive" {
t.Fatalf("expected rule B-permissive to be selected, got %q", rule.ID)
}
if len(allowed) != 1 || allowed[0] != "x-custom-header" {
t.Fatalf("unexpected allowed headers: %v", allowed)
}
}
func TestMatchAllowedOriginReturnsFirstMatchingPattern(t *testing.T) {
rule := Rule{AllowedOrigins: []string{"https://app.example.com", "https://*", "*"}}
tests := []struct {
origin string
want string
}{
{"https://app.example.com", "https://app.example.com"},
{"https://other.example.com", "https://*"},
{"http://other.example.com", "*"},
}
for _, tt := range tests {
got, ok := rule.matchAllowedOrigin(tt.origin)
if !ok {
t.Fatalf("expected %q to match", tt.origin)
}
if got != tt.want {
t.Fatalf("origin %q matched %q, want %q", tt.origin, got, tt.want)
}
}
}
func TestFilterAllowedHeadersPreservesRequestedNames(t *testing.T) {
rule := Rule{AllowedHeaders: []string{"x-amz-*"}}
allowed, ok := rule.FilterAllowedHeaders([]string{"X-Amz-Date", " X-AMZ-Meta-Test "})
if !ok {
t.Fatal("expected both request headers to match")
}
if got := strings.Join(allowed, ","); got != "X-Amz-Date,X-AMZ-Meta-Test" {
t.Fatalf("allowed headers = %q", got)
}
}
internal/bucket/object/lock/lock.go
+3 -7
View File
@@ -26,16 +26,14 @@ import (
"io"
"maps"
"net/http"
"net/textproto"
"strings"
"time"
"github.com/beevik/ntp"
"github.com/minio/minio/internal/amztime"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
const (
@@ -435,7 +433,7 @@ func IsObjectLockRequested(h http.Header) bool {
}
// ParseObjectLockRetentionHeaders parses http headers to extract retention mode and retention date
func ParseObjectLockRetentionHeaders(h http.Header) (rmode RetMode, r RetentionDate, err error) {
func ParseObjectLockRetentionHeaders(h http.Header, allowPastRetainDate bool) (rmode RetMode, r RetentionDate, err error) {
retMode := h.Get(AmzObjectLockMode)
dateStr := h.Get(AmzObjectLockRetainUntilDate)
if len(retMode) == 0 || len(dateStr) == 0 {
@@ -455,15 +453,13 @@ func ParseObjectLockRetentionHeaders(h http.Header) (rmode RetMode, r RetentionD
if err != nil {
return rmode, r, ErrInvalidRetentionDate
}
_, replReq := h[textproto.CanonicalMIMEHeaderKey(xhttp.MinIOSourceReplicationRequest)]
t, err := UTCNowNTP()
if err != nil {
lockLogIf(context.Background(), err)
return rmode, r, ErrPastObjectLockRetainDate
}
if retDate.Before(t) && !replReq {
if retDate.Before(t) && !allowPastRetainDate {
return rmode, r, ErrPastObjectLockRetainDate
}
internal/bucket/object/lock/lock_test.go
+9 -1
View File
@@ -386,7 +386,7 @@ func TestParseObjectLockRetentionHeaders(t *testing.T) {
}
for i, tt := range tests {
_, _, err := ParseObjectLockRetentionHeaders(tt.header)
_, _, err := ParseObjectLockRetentionHeaders(tt.header, false)
//nolint:gocritic
if tt.expectedErr == nil {
if err != nil {
@@ -398,6 +398,14 @@ func TestParseObjectLockRetentionHeaders(t *testing.T) {
t.Fatalf("Case %d error: expected = %v, got = %v", i, tt.expectedErr, err)
}
}
past := http.Header{
xhttp.AmzObjectLockMode: []string{"governance"},
xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02T15:04:05Z"},
}
if _, _, err := ParseObjectLockRetentionHeaders(past, true); err != nil {
t.Fatalf("trusted replica past retention date: %v", err)
}
}
func TestGetObjectRetentionMeta(t *testing.T) {
internal/bucket/replication/destination.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"fmt"
"strings"
"github.com/minio/pkg/v3/wildcard"
"github.com/pgsty/silo-pkg/v3/wildcard"
)
// DestinationARNPrefix - destination ARN prefix as per AWS S3 specification.
internal/bucket/versioning/versioning.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"io"
"strings"
"github.com/minio/pkg/v3/wildcard"
"github.com/pgsty/silo-pkg/v3/wildcard"
)
// State - enabled/disabled/suspended states
internal/config/api/api.go
+1 -1
View File
@@ -28,7 +28,7 @@ import (
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// API sub-system constants
internal/config/batch/batch.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Batch job environment variables
internal/config/browser/browser.go
+1 -1
View File
@@ -23,7 +23,7 @@ import (
"sync"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Browser sub-system constants
internal/config/callhome/callhome.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Callhome related keys
internal/config/certs.go
+1 -1
View File
@@ -25,7 +25,7 @@ import (
"errors"
"os"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// EnvCertPassword is the environment variable which contains the password used
internal/config/compress/compress.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"strings"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Config represents the compression settings.
internal/config/config.go
+1 -1
View File
@@ -31,7 +31,7 @@ import (
"github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// ErrorConfig holds the config error types
internal/config/drive/drive.go
+1 -1
View File
@@ -22,7 +22,7 @@ import (
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Drive specific timeout environment variables
internal/config/etcd/etcd.go
+2 -2
View File
@@ -25,8 +25,8 @@ import (
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/crypto"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/namespace"
"go.uber.org/zap"
internal/config/heal/heal.go
+1 -1
View File
@@ -26,7 +26,7 @@ import (
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Compression environment variables
internal/config/identity/ldap/config.go
+1 -1
View File
@@ -28,7 +28,7 @@ import (
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/crypto"
"github.com/minio/pkg/v3/ldap"
"github.com/pgsty/silo-pkg/v3/ldap"
)
const (
internal/config/identity/ldap/ldap.go
+1 -1
View File
@@ -27,7 +27,7 @@ import (
ldap "github.com/go-ldap/ldap/v3"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
xldap "github.com/minio/pkg/v3/ldap"
xldap "github.com/pgsty/silo-pkg/v3/ldap"
)
var errAuthentication = errors.New("ldap authentication failed")
internal/config/identity/openid/jwt.go
+2 -2
View File
@@ -31,8 +31,8 @@ import (
jwtgo "github.com/golang-jwt/jwt/v4"
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/auth"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/policy"
)
type publicKeys struct {
internal/config/identity/openid/jwt_test.go
+1 -1
View File
@@ -39,7 +39,7 @@ import (
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/config"
jwtm "github.com/minio/minio/internal/jwt"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
func TestUpdateClaimsExpiry(t *testing.T) {
internal/config/identity/openid/openid.go
+3 -3
View File
@@ -39,9 +39,9 @@ import (
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/config/identity/openid/provider"
"github.com/minio/minio/internal/hash/sha256"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/policy"
)
// OpenID keys and envs.
internal/config/identity/openid/providercfg.go
+1 -1
View File
@@ -28,7 +28,7 @@ import (
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/config/identity/openid/provider"
xhttp "github.com/minio/minio/internal/http"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
type providerCfg struct {
internal/config/identity/plugin/config.go
+2 -2
View File
@@ -34,8 +34,8 @@ import (
"github.com/minio/minio/internal/arn"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
func authNLogIf(ctx context.Context, err error) {
internal/config/identity/tls/config.go
+1 -1
View File
@@ -23,7 +23,7 @@ import (
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
const (
internal/config/ilm/ilm.go
+1 -1
View File
@@ -25,7 +25,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Errors returned when the access tiering configuration is unusable.
internal/config/lambda/parse.go
+2 -2
View File
@@ -27,8 +27,8 @@ import (
"github.com/minio/minio/internal/config/lambda/event"
"github.com/minio/minio/internal/config/lambda/target"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
const (
internal/config/lambda/target/webhook.go
+2 -2
View File
@@ -32,8 +32,8 @@ import (
"github.com/minio/minio/internal/config/lambda/event"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/certs"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/certs"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// Webhook constants
internal/config/notify/legacy.go
+47 -42
View File
@@ -26,6 +26,25 @@ import (
"github.com/minio/minio/internal/event/target"
)
// LegacyDatabaseTargetError reports a pre-KV database notification target
// that cannot be migrated safely. It deliberately carries no configuration
// values so credentials cannot escape through startup logs.
type LegacyDatabaseTargetError struct {
subsystem string
target string
connectionKey string
invalid bool
}
func (e *LegacyDatabaseTargetError) Error() string {
if e.invalid {
return fmt.Sprintf("%s:%s has invalid %s or target settings; fix the target before migrating to SILO",
e.subsystem, e.target, e.connectionKey)
}
return fmt.Sprintf("%s:%s requires %s; discrete database connection fields are not migrated to SILO",
e.subsystem, e.target, e.connectionKey)
}
// SetNotifyKafka - helper for config migration from older config.
func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error {
if !cfg.Enable {
@@ -325,8 +344,21 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg
return nil
}
if cfg.ConnectionString == "" {
return &LegacyDatabaseTargetError{
subsystem: config.NotifyPostgresSubSys,
target: psqName,
connectionKey: target.PostgresConnectionString,
}
}
if err := cfg.Validate(); err != nil {
return err
return &LegacyDatabaseTargetError{
subsystem: config.NotifyPostgresSubSys,
target: psqName,
connectionKey: target.PostgresConnectionString,
invalid: true,
}
}
s[config.NotifyPostgresSubSys][psqName] = config.KVS{
@@ -346,26 +378,6 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg
Key: target.PostgresTable,
Value: cfg.Table,
},
config.KV{
Key: target.PostgresHost,
Value: cfg.Host.String(),
},
config.KV{
Key: target.PostgresPort,
Value: cfg.Port,
},
config.KV{
Key: target.PostgresUsername,
Value: cfg.Username,
},
config.KV{
Key: target.PostgresPassword,
Value: cfg.Password,
},
config.KV{
Key: target.PostgresDatabase,
Value: cfg.Database,
},
config.KV{
Key: target.PostgresQueueDir,
Value: cfg.QueueDir,
@@ -538,8 +550,21 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error
return nil
}
if cfg.DSN == "" {
return &LegacyDatabaseTargetError{
subsystem: config.NotifyMySQLSubSys,
target: sqlName,
connectionKey: target.MySQLDSNString,
}
}
if err := cfg.Validate(); err != nil {
return err
return &LegacyDatabaseTargetError{
subsystem: config.NotifyMySQLSubSys,
target: sqlName,
connectionKey: target.MySQLDSNString,
invalid: true,
}
}
s[config.NotifyMySQLSubSys][sqlName] = config.KVS{
@@ -559,26 +584,6 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error
Key: target.MySQLTable,
Value: cfg.Table,
},
config.KV{
Key: target.MySQLHost,
Value: cfg.Host.String(),
},
config.KV{
Key: target.MySQLPort,
Value: cfg.Port,
},
config.KV{
Key: target.MySQLUsername,
Value: cfg.User,
},
config.KV{
Key: target.MySQLPassword,
Value: cfg.Password,
},
config.KV{
Key: target.MySQLDatabase,
Value: cfg.Database,
},
config.KV{
Key: target.MySQLQueueDir,
Value: cfg.QueueDir,
internal/config/notify/legacy_test.go
+191 -1
View File
@@ -18,14 +18,35 @@
package notify
import (
"errors"
"strings"
"testing"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/event/target"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/rabbitmq/amqp091-go"
)
func assertLegacyDatabaseTargetError(t *testing.T, err error, subsystem, name, key string, secrets ...string) {
t.Helper()
var targetErr *LegacyDatabaseTargetError
if !errors.As(err, &targetErr) {
t.Fatalf("error = %v, want *LegacyDatabaseTargetError", err)
}
msg := err.Error()
for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} {
if !strings.Contains(msg, want) {
t.Errorf("error %q does not contain %q", msg, want)
}
}
for _, secret := range secrets {
if secret != "" && strings.Contains(msg, secret) {
t.Errorf("error leaks configuration value %q: %s", secret, msg)
}
}
}
// T5 (NATS): a config produced by the legacy migration must survive validation
// and round-trip back through the parser unchanged. Before the fix the
// migration wrote an env var name as a config key, so every migrated NATS
@@ -113,3 +134,172 @@ func TestSetNotifyAMQPRoundTrip(t *testing.T) {
t.Errorf("Internal = true, want false (immediate must not be written to the internal key)")
}
}
func TestSetNotifyDatabaseTargetsRequireConnectionStrings(t *testing.T) {
postgresHost, err := xnet.ParseHost("legacy-postgres.example")
if err != nil {
t.Fatal(err)
}
mysqlHost, err := xnet.ParseURL("legacy-mysql.example")
if err != nil {
t.Fatal(err)
}
tests := []struct {
name string
subsystem string
key string
set func(config.Config) error
secrets []string
}{
{
name: "postgres",
subsystem: config.NotifyPostgresSubSys,
key: target.PostgresConnectionString,
set: func(s config.Config) error {
return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{
Enable: true,
Format: formatNamespace,
Table: "events",
Host: *postgresHost,
Port: "5432",
Username: "legacy-user",
Password: "legacy-postgres-password",
Database: "legacy-database",
})
},
secrets: []string{postgresHost.String(), "5432", "legacy-user", "legacy-postgres-password", "legacy-database"},
},
{
name: "mysql",
subsystem: config.NotifyMySQLSubSys,
key: target.MySQLDSNString,
set: func(s config.Config) error {
return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{
Enable: true,
Format: formatNamespace,
Table: "events",
Host: *mysqlHost,
Port: "3306",
User: "legacy-user",
Password: "legacy-mysql-password",
Database: "legacy-database",
})
},
secrets: []string{mysqlHost.String(), "3306", "legacy-user", "legacy-mysql-password", "legacy-database"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := config.Config{test.subsystem: map[string]config.KVS{}}
err := test.set(s)
assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secrets...)
if _, ok := s[test.subsystem][testTargetName]; ok {
t.Fatal("unsupported target was emitted despite migration error")
}
})
}
}
func TestSetNotifyDisabledDatabaseTargetsAreIgnored(t *testing.T) {
s := config.Config{
config.NotifyPostgresSubSys: map[string]config.KVS{},
config.NotifyMySQLSubSys: map[string]config.KVS{},
}
if err := SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{Password: "discarded-postgres-secret"}); err != nil {
t.Fatalf("SetNotifyPostgres: %v", err)
}
if err := SetNotifyMySQL(s, testTargetName, target.MySQLArgs{Password: "discarded-mysql-secret"}); err != nil {
t.Fatalf("SetNotifyMySQL: %v", err)
}
if _, ok := s[config.NotifyPostgresSubSys][testTargetName]; ok {
t.Fatal("disabled Postgres target was emitted")
}
if _, ok := s[config.NotifyMySQLSubSys][testTargetName]; ok {
t.Fatal("disabled MySQL target was emitted")
}
}
func TestSetNotifyInvalidDatabaseTargetsDoNotLeak(t *testing.T) {
tests := []struct {
name string
subsystem string
key string
secret string
set func(config.Config) error
}{
{
name: "postgres",
subsystem: config.NotifyPostgresSubSys,
key: target.PostgresConnectionString,
secret: "postgres-dsn-secret",
set: func(s config.Config) error {
return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{
Enable: true,
Format: formatNamespace,
ConnectionString: "host=db password=postgres-dsn-secret",
})
},
},
{
name: "mysql",
subsystem: config.NotifyMySQLSubSys,
key: target.MySQLDSNString,
secret: "mysql-dsn-secret",
set: func(s config.Config) error {
return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{
Enable: true,
Format: formatNamespace,
DSN: "user:mysql-dsn-secret@tcp(db:3306/events",
Table: "events",
})
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := config.Config{test.subsystem: map[string]config.KVS{}}
err := test.set(s)
assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secret)
})
}
}
func TestDatabaseConnectionStringsSurviveKVTokenization(t *testing.T) {
tests := []struct {
name string
subsystem string
key string
input string
want string
}{
{
name: "postgres",
subsystem: config.NotifyPostgresSubSys,
key: target.PostgresConnectionString,
input: `notify_postgres:dsn connection_string="host=db port=5432 dbname=events user=app password=inside" table="events"`,
want: "host=db port=5432 dbname=events user=app password=inside",
},
{
name: "mysql",
subsystem: config.NotifyMySQLSubSys,
key: target.MySQLDSNString,
input: `notify_mysql:dsn dsn_string="user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside" table="events"`,
want: "user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := config.Config{test.subsystem: map[string]config.KVS{}}
if _, err := s.SetKVS(test.input, DefaultNotificationKVS); err != nil {
t.Fatalf("SetKVS: %v", err)
}
if got := s[test.subsystem]["dsn"].Get(test.key); got != test.want {
t.Errorf("%s = %q, want %q", test.key, got, test.want)
}
})
}
}
internal/config/notify/parse.go
+2 -2
View File
@@ -32,8 +32,8 @@ import (
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/event/target"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/rabbitmq/amqp091-go"
)
internal/config/notify/parse_test.go
+3 -15
View File
@@ -414,21 +414,9 @@ var configPkgConsts = map[string]string{
"Comment": config.Comment,
}
// knownUnregisteredWrites records pre-existing instances of the exact defect
// this audit exists to catch: a legacy migration writing config keys that no
// default KVS registers, so the migrated config is rejected on the next load.
//
// These are inherited from upstream and are the same class as issue #39, but
// they are NOT part of the issue #39 fix and were left untouched deliberately.
// The Postgres/MySQL keys below are the pre-connection-string DSN fields; the
// migration still writes them and `password` carries a plaintext database
// password.
//
// This list must only ever shrink. Do not add entries to silence a new gap.
var knownUnregisteredWrites = map[string][]string{
"SetNotifyPostgres": {"host", "port", "username", "password", "database"},
"SetNotifyMySQL": {"host", "port", "username", "password", "database"},
}
// knownUnregisteredWrites is a shrink-only ratchet for inherited migration
// gaps. Do not add entries to silence a new mismatch.
var knownUnregisteredWrites = map[string][]string{}
func TestNotifyConfigKeysAreRegistered(t *testing.T) {
targetConsts, err := parseTargetPkgStringConsts("../../event/target")
internal/config/policy/opa/config.go
+3 -3
View File
@@ -24,9 +24,9 @@ import (
"net/http"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/policy"
)
// Env IAM OPA URL
internal/config/policy/plugin/config.go
+2 -2
View File
@@ -26,8 +26,8 @@ import (
"github.com/minio/minio/internal/config"
xhttp "github.com/minio/minio/internal/http"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/policy"
)
// Authorization Plugin config and env variables
internal/config/scanner/scanner.go
+1 -1
View File
@@ -23,7 +23,7 @@ import (
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Compression environment variables
internal/config/storageclass/storage-class.go
+1 -1
View File
@@ -28,7 +28,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
// Standard constants for all storage class
internal/config/subnet/config.go
+2 -2
View File
@@ -25,8 +25,8 @@ import (
"sync"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// DefaultKVS - default KV config for subnet settings
internal/crypto/auto-encryption.go
+1 -1
View File
@@ -19,7 +19,7 @@ package crypto
import (
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
const (
internal/disk/directio_unix.go
+1 -1
View File
@@ -43,7 +43,7 @@ func DisableDirectIO(f *os.File) error {
if err != nil {
return err
}
flag &= ^(syscall.O_DIRECT)
flag &= ^syscall.O_DIRECT
_, err = unix.FcntlInt(fd, unix.F_SETFL, flag)
return err
}
internal/dsync/drwmutex.go
+2 -2
View File
@@ -29,8 +29,8 @@ import (
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/minio/minio/internal/mcontext"
"github.com/minio/pkg/v3/console"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/console"
"github.com/pgsty/silo-pkg/v3/env"
)
// Indicator if logging is enabled.
internal/event/rules.go
+1 -1
View File
@@ -20,7 +20,7 @@ package event
import (
"strings"
"github.com/minio/pkg/v3/wildcard"
"github.com/pgsty/silo-pkg/v3/wildcard"
)
// NewPattern - create new pattern for prefix/suffix.
internal/event/target/amqp.go
+1 -1
View File
@@ -32,7 +32,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/rabbitmq/amqp091-go"
)
internal/event/target/elasticsearch.go
+1 -1
View File
@@ -38,7 +38,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/pkg/errors"
)
internal/event/target/kafka.go
+1 -1
View File
@@ -35,7 +35,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/IBM/sarama"
saramatls "github.com/IBM/sarama/tools/tls"
internal/event/target/mqtt.go
+1 -1
View File
@@ -33,7 +33,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
const (
internal/event/target/mysql.go
+1 -6
View File
@@ -35,7 +35,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
const (
@@ -72,11 +72,6 @@ const (
EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT"
EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING"
EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE"
EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST"
EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT"
EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME"
EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD"
EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE"
EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT"
EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR"
EnvMySQLMaxOpenConnections = "MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS"
internal/event/target/nats.go
+1 -1
View File
@@ -33,9 +33,9 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
"github.com/nats-io/nats.go"
"github.com/nats-io/stan.go"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// NATS related constants
internal/event/target/nats_contrib_test.go
+5 -5
View File
@@ -21,8 +21,8 @@ import (
"github.com/nats-io/nats-server/v2/server"
xnet "github.com/minio/pkg/v3/net"
natsserver "github.com/nats-io/nats-server/v2/test"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
func TestNatsConnPlain(t *testing.T) {
@@ -35,7 +35,7 @@ func TestNatsConnPlain(t *testing.T) {
Enable: true,
Address: xnet.Host{
Name: "localhost",
Port: (xnet.Port(opts.Port)),
Port: xnet.Port(opts.Port),
IsPortSet: true,
},
Subject: "test",
@@ -59,7 +59,7 @@ func TestNatsConnUserPass(t *testing.T) {
Enable: true,
Address: xnet.Host{
Name: "localhost",
Port: (xnet.Port(opts.Port)),
Port: xnet.Port(opts.Port),
IsPortSet: true,
},
Subject: "test",
@@ -85,7 +85,7 @@ func TestNatsConnToken(t *testing.T) {
Enable: true,
Address: xnet.Host{
Name: "localhost",
Port: (xnet.Port(opts.Port)),
Port: xnet.Port(opts.Port),
IsPortSet: true,
},
Subject: "test",
@@ -116,7 +116,7 @@ func TestNatsConnNKeySeed(t *testing.T) {
Enable: true,
Address: xnet.Host{
Name: "localhost",
Port: (xnet.Port(opts.Port)),
Port: xnet.Port(opts.Port),
IsPortSet: true,
},
Subject: "test",
internal/event/target/nats_tls_contrib_test.go
+4 -4
View File
@@ -21,8 +21,8 @@ import (
"path/filepath"
"testing"
xnet "github.com/minio/pkg/v3/net"
natsserver "github.com/nats-io/nats-server/v2/test"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
func TestNatsConnTLSCustomCA(t *testing.T) {
@@ -33,7 +33,7 @@ func TestNatsConnTLSCustomCA(t *testing.T) {
Enable: true,
Address: xnet.Host{
Name: "localhost",
Port: (xnet.Port(opts.Port)),
Port: xnet.Port(opts.Port),
IsPortSet: true,
},
Subject: "test",
@@ -56,7 +56,7 @@ func TestNatsConnTLSCustomCAHandshakeFirst(t *testing.T) {
Enable: true,
Address: xnet.Host{
Name: "localhost",
Port: (xnet.Port(opts.Port)),
Port: xnet.Port(opts.Port),
IsPortSet: true,
},
Subject: "test",
@@ -80,7 +80,7 @@ func TestNatsConnTLSClientAuthorization(t *testing.T) {
Enable: true,
Address: xnet.Host{
Name: "localhost",
Port: (xnet.Port(opts.Port)),
Port: xnet.Port(opts.Port),
IsPortSet: true,
},
Subject: "test",
internal/event/target/nsq.go
+1 -1
View File
@@ -33,7 +33,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// NSQ constants
internal/event/target/nsq_test.go
+1 -1
View File
@@ -20,7 +20,7 @@ package target
import (
"testing"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
func TestNSQArgs_Validate(t *testing.T) {
internal/event/target/postgresql.go
+1 -6
View File
@@ -38,7 +38,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
const (
@@ -69,11 +69,6 @@ const (
EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT"
EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING"
EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE"
EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST"
EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT"
EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME"
EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD"
EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE"
EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR"
EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT"
EnvPostgresMaxOpenConnections = "MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS"
internal/event/target/redis.go
+1 -1
View File
@@ -33,7 +33,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// Redis constants
internal/event/target/webhook.go
+2 -2
View File
@@ -38,8 +38,8 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
"github.com/minio/pkg/v3/certs"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/certs"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// Webhook constants
internal/event/targetlist.go
+1 -1
View File
@@ -27,7 +27,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/store"
"github.com/minio/pkg/v3/workers"
"github.com/pgsty/silo-pkg/v3/workers"
)
const (
internal/grid/connection.go
+3 -3
View File
@@ -41,7 +41,7 @@ import (
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/pubsub"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/puzpuzpuz/xsync/v3"
"github.com/tinylib/msgp/msgp"
"github.com/zeebo/xxh3"
@@ -1806,8 +1806,8 @@ func (ww *wsWriter) writeFrame(w io.Writer, f ws.Frame) error {
const (
bit0 = 0x80
len7 = int64(125)
len16 = int64(^(uint16(0)))
len64 = int64(^(uint64(0)) >> 1)
len16 = int64(^uint16(0))
len64 = int64(^uint64(0) >> 1)
)
bts := ww.tmp[:]
internal/handlers/proxy.go
+1 -1
View File
@@ -28,7 +28,7 @@ import (
"strings"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
)
var (
internal/hash/checksum.go
+72 -13
View File
@@ -157,7 +157,6 @@ func ChecksumStringToType(alg string) ChecksumType {
case "SHA256":
return ChecksumSHA256
case "CRC64NVME":
// AWS seems to ignore full value, and just assume it.
return ChecksumCRC64NVME
case "":
return ChecksumNone
@@ -192,7 +191,9 @@ func NewChecksumType(alg, objType string) ChecksumType {
}
return ChecksumSHA256
case "CRC64NVME":
// AWS seems to ignore full value, and just assume it.
if objType == xhttp.AmzChecksumTypeComposite {
return ChecksumInvalid
}
return ChecksumCRC64NVME
case "":
if full != 0 {
@@ -247,7 +248,7 @@ func (c ChecksumType) StringFull() string {
// FullObjectRequested will return if the checksum type indicates full object checksum was requested.
func (c ChecksumType) FullObjectRequested() bool {
return c&(ChecksumFullObject) == ChecksumFullObject || c.Is(ChecksumCRC64NVME)
return c&ChecksumFullObject == ChecksumFullObject || c.Is(ChecksumCRC64NVME)
}
// IsMultipartComposite returns true if the checksum is multipart and full object was not requested.
@@ -657,22 +658,73 @@ func AddChecksumHeader(w http.ResponseWriter, c map[string]string) {
}
}
func isSupportedChecksumHeader(name string) bool {
switch {
case strings.EqualFold(name, xhttp.AmzChecksumAlgo),
strings.EqualFold(name, xhttp.AmzChecksumType),
strings.EqualFold(name, xhttp.AmzChecksumMode):
return true
}
for _, checksumType := range BaseChecksumTypes {
if strings.EqualFold(name, checksumType.Key()) {
return true
}
}
return false
}
func hasUnsupportedChecksumHeader(h http.Header) bool {
for name := range h {
if strings.HasPrefix(strings.ToLower(name), "x-amz-checksum-") && !isSupportedChecksumHeader(name) {
return true
}
}
return false
}
// GetContentChecksum returns content checksum.
// Returns ErrInvalidChecksum if so.
// Returns nil, nil if no checksum.
func GetContentChecksum(h http.Header) (*Checksum, error) {
if hasUnsupportedChecksumHeader(h) {
return nil, ErrInvalidChecksum
}
if trailing := h.Values(xhttp.AmzTrailer); len(trailing) > 0 {
var res *Checksum
for _, header := range trailing {
var duplicates bool
for _, t := range BaseChecksumTypes {
if strings.EqualFold(t.Key(), header) {
duplicates = res != nil
res = NewChecksumWithType(t|ChecksumTrailing, "")
for _, headers := range trailing {
for header := range strings.SplitSeq(headers, ",") {
header = strings.TrimSpace(header)
var duplicates bool
for _, t := range BaseChecksumTypes {
if strings.EqualFold(t.Key(), header) {
duplicates = res != nil
// A checksum can be advertised via x-amz-trailer while its
// value is still delivered in the request headers. The AWS
// Java SDK v2 does this on chunked (aws-chunked) uploads:
// it sends STREAMING-AWS4-HMAC-SHA256-PAYLOAD (no trailer),
// puts the precomputed value in x-amz-checksum-*, yet still
// lists it in x-amz-trailer, so no trailer ever arrives.
// When the value is present as a header, honor it directly
// instead of waiting for a trailer that will never be read.
if v := h.Get(t.Key()); v != "" {
res = NewChecksumWithType(t, v)
if res == nil {
// The value is supplied in the header but does
// not parse. A malformed client-supplied checksum
// is an error, not a reason to skip validation.
return nil, ErrInvalidChecksum
}
} else {
res = NewChecksumWithType(t|ChecksumTrailing, "")
}
}
}
if strings.HasPrefix(strings.ToLower(header), "x-amz-checksum-") && !isSupportedChecksumHeader(header) {
return nil, ErrInvalidChecksum
}
if duplicates {
return nil, ErrInvalidChecksum
}
}
if duplicates {
return nil, ErrInvalidChecksum
}
}
if res != nil {
@@ -682,7 +734,11 @@ func GetContentChecksum(h http.Header) (*Checksum, error) {
return nil, ErrInvalidChecksum
}
res.Type |= ChecksumFullObject
case xhttp.AmzChecksumTypeComposite, "":
case xhttp.AmzChecksumTypeComposite:
if res.Type.Base().Is(ChecksumCRC64NVME) {
return nil, ErrInvalidChecksum
}
case "":
default:
return nil, ErrInvalidChecksum
}
@@ -748,5 +804,8 @@ func getContentChecksum(h http.Header) (t ChecksumType, s string) {
for _, t := range BaseChecksumTypes {
checkType(t)
}
if t.Base().Is(ChecksumCRC64NVME) && h.Get(xhttp.AmzChecksumType) == xhttp.AmzChecksumTypeComposite {
return ChecksumInvalid, ""
}
return t, s
}
internal/hash/checksum_test.go
+123
View File
@@ -18,14 +18,58 @@
package hash
import (
"errors"
"net/http"
"net/http/httptest"
"testing"
xhttp "github.com/minio/minio/internal/http"
)
func TestGetContentChecksumRejectsUnsupportedHeaders(t *testing.T) {
unsupported := []string{
"x-amz-checksum-md5",
"x-amz-checksum-sha512",
"x-amz-checksum-xxhash64",
"x-amz-checksum-xxhash3",
"x-amz-checksum-xxhash128",
"x-amz-checksum-future",
}
for _, header := range unsupported {
t.Run("header/"+header, func(t *testing.T) {
h := http.Header{header: {"AA=="}}
if _, err := GetContentChecksum(h); !errors.Is(err, ErrInvalidChecksum) {
t.Fatalf("GetContentChecksum(%s) error = %v, want ErrInvalidChecksum", header, err)
}
})
t.Run("trailer/"+header, func(t *testing.T) {
h := http.Header{xhttp.AmzTrailer: {header}}
if _, err := GetContentChecksum(h); !errors.Is(err, ErrInvalidChecksum) {
t.Fatalf("GetContentChecksum(trailer %s) error = %v, want ErrInvalidChecksum", header, err)
}
})
}
for header, value := range map[string]string{
xhttp.AmzChecksumAlgo: "CRC32",
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite,
xhttp.AmzChecksumMode: "ENABLED",
"x-amz-sdk-checksum-algorithm": "SHA512",
} {
t.Run("control/"+header, func(t *testing.T) {
h := http.Header{header: {value}}
if _, err := GetContentChecksum(h); errors.Is(err, ErrInvalidChecksum) {
t.Fatalf("control header %s was rejected", header)
}
})
}
}
// TestChecksumAddToHeader tests that adding and retrieving a checksum on a header works
func TestChecksumAddToHeader(t *testing.T) {
if got := NewChecksumType("CRC64NVME", xhttp.AmzChecksumTypeComposite); !got.Is(ChecksumInvalid) {
t.Fatalf("CRC64NVME/COMPOSITE = %s, want invalid", got.StringFull())
}
tests := []struct {
name string
checksum ChecksumType
@@ -106,6 +150,16 @@ func TestChecksumAddToHeader(t *testing.T) {
}
}
func TestCRC64NVMECompositeTrailerIsInvalid(t *testing.T) {
h := http.Header{}
h.Set(xhttp.AmzTrailer, ChecksumCRC64NVME.Key())
h.Set(xhttp.AmzChecksumType, xhttp.AmzChecksumTypeComposite)
_, err := GetContentChecksum(h)
if !errors.Is(err, ErrInvalidChecksum) {
t.Fatalf("CRC64NVME/COMPOSITE trailer error = %v, want ErrInvalidChecksum", err)
}
}
// TestChecksumSerializeDeserialize checks AppendTo can be reversed by ChecksumFromBytes
func TestChecksumSerializeDeserialize(t *testing.T) {
myData := []byte("this-is-a-checksum-data-test")
@@ -203,3 +257,72 @@ func TestChecksumSerializeDeserializeMultiPart(t *testing.T) {
}
}
}
// TestGetContentChecksumTrailerWithHeaderValue covers the case where a checksum
// is advertised via x-amz-trailer while its value is delivered as a request
// header (no trailer is actually sent). The AWS Java SDK v2 does this on chunked
// (aws-chunked) uploads that use STREAMING-AWS4-HMAC-SHA256-PAYLOAD (non-trailer)
// but still list the checksum in x-amz-trailer. See issue #107. The header value
// must be honored as a non-trailing checksum instead of being treated as an empty
// trailing checksum.
func TestGetContentChecksumTrailerWithHeaderValue(t *testing.T) {
const crc = "Hkksgg==" // CRC32 of "Hello CRC32!"
// Trailer advertised AND value present in header -> non-trailing, value honored.
h := http.Header{}
h.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
h.Set(xhttp.AmzChecksumCRC32, crc)
cs, err := GetContentChecksum(h)
if err != nil {
t.Fatalf("GetContentChecksum error = %v, want nil", err)
}
if cs == nil {
t.Fatal("GetContentChecksum returned nil checksum")
}
if cs.Type.Trailing() {
t.Errorf("checksum reported as trailing; want non-trailing since value is in the header")
}
if !cs.Type.Is(ChecksumCRC32) {
t.Errorf("checksum type = %s, want CRC32", cs.Type.StringFull())
}
if cs.Encoded != crc {
t.Errorf("checksum value = %q, want %q", cs.Encoded, crc)
}
// Trailer advertised WITHOUT a header value -> stays trailing (unchanged).
h2 := http.Header{}
h2.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
cs2, err := GetContentChecksum(h2)
if err != nil {
t.Fatalf("GetContentChecksum (no header value) error = %v, want nil", err)
}
if cs2 == nil || !cs2.Type.Trailing() {
t.Errorf("checksum = %v, want a trailing CRC32 checksum", cs2)
}
}
// TestGetContentChecksumTrailerMalformedHeaderValue guards against turning a
// malformed client-supplied checksum into a no-op. When a checksum is advertised
// via x-amz-trailer and its header value is present but does not parse, the
// request must be rejected (ErrInvalidChecksum) rather than silently dropped.
// The mismatched x-amz-checksum-algorithm selector makes the regression visible:
// without the guard, execution falls through to getContentChecksum which would
// return (nil, nil) and install no validator at all.
func TestGetContentChecksumTrailerMalformedHeaderValue(t *testing.T) {
h := http.Header{}
h.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
h.Set(xhttp.AmzChecksumCRC32, "AQID") // decodes to 3 bytes -> invalid CRC32
h.Set(xhttp.AmzChecksumAlgo, "SHA256")
cs, err := GetContentChecksum(h)
if !errors.Is(err, ErrInvalidChecksum) {
t.Fatalf("GetContentChecksum error = %v (checksum %v), want ErrInvalidChecksum", err, cs)
}
// Same, without the misleading algorithm selector: still an error.
h2 := http.Header{}
h2.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
h2.Set(xhttp.AmzChecksumCRC32, "AQID")
if _, err := GetContentChecksum(h2); !errors.Is(err, ErrInvalidChecksum) {
t.Fatalf("GetContentChecksum (no algo selector) error = %v, want ErrInvalidChecksum", err)
}
}
internal/http/server_test.go
+1 -1
View File
@@ -24,7 +24,7 @@ import (
"reflect"
"testing"
"github.com/minio/pkg/v3/certs"
"github.com/pgsty/silo-pkg/v3/certs"
)
func TestNewServer(t *testing.T) {
internal/http/transports.go
+1 -1
View File
@@ -25,7 +25,7 @@ import (
"syscall"
"time"
"github.com/minio/pkg/v3/certs"
"github.com/pgsty/silo-pkg/v3/certs"
)
// tlsClientSessionCacheSize is the cache size for client sessions.
internal/ioutil/wait_pipe.go
+3 -7
View File
@@ -57,11 +57,7 @@ func WaitPipe() (*PipeReader, *PipeWriter) {
r, w := io.Pipe()
var wg sync.WaitGroup
wg.Add(1)
return &PipeReader{
PipeReader: r,
wait: wg.Wait,
}, &PipeWriter{
PipeWriter: w,
done: wg.Done,
}
pr := &PipeReader{PipeReader: r, wait: wg.Wait}
pw := &PipeWriter{PipeWriter: w, done: wg.Done}
return pr, pw
}
internal/kms/config.go
+3 -3
View File
@@ -36,9 +36,9 @@ import (
"aead.dev/mtls"
"github.com/minio/kms-go/kes"
"github.com/minio/kms-go/kms"
"github.com/minio/pkg/v3/certs"
"github.com/minio/pkg/v3/ellipses"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/certs"
"github.com/pgsty/silo-pkg/v3/ellipses"
"github.com/pgsty/silo-pkg/v3/env"
)
// Environment variables for MinIO KMS.
internal/kms/stub.go
+1 -1
View File
@@ -25,7 +25,7 @@ import (
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/pkg/v3/wildcard"
"github.com/pgsty/silo-pkg/v3/wildcard"
)
var (
internal/logger/config.go
+2 -2
View File
@@ -26,8 +26,8 @@ import (
"strings"
"time"
"github.com/minio/pkg/v3/env"
xnet "github.com/minio/pkg/v3/net"
"github.com/pgsty/silo-pkg/v3/env"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/logger/target/http"
internal/logger/target/http/http.go
+1 -1
View File
@@ -38,7 +38,7 @@ import (
types "github.com/minio/minio/internal/logger/target/loggertypes"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
"github.com/valyala/bytebufferpool"
)
internal/logger/target/kafka/kafka.go
+1 -1
View File
@@ -38,7 +38,7 @@ import (
types "github.com/minio/minio/internal/logger/target/loggertypes"
"github.com/minio/minio/internal/once"
"github.com/minio/minio/internal/store"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
// the suffix for the configured queue dir where the logs will be persisted.
internal/rest/client.go
+1 -1
View File
@@ -37,7 +37,7 @@ import (
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/mcontext"
xnet "github.com/minio/pkg/v3/net"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
const logSubsys = "internodes"
internal/s3select/message.go
+31 -18
View File
@@ -295,7 +295,15 @@ func (writer *messageWriter) start() {
select {
case data := <-writer.errCh:
quitFlag = true
// Flush collected records before sending error message
// A record accepted by SendRecord may still be queued when the
// error arrives, because select picks between the two channels
// at random. Stage it first so every record produced before the
// error precedes the error message instead of being dropped.
for len(writer.payloadCh) > 0 {
if !writer.stageRecord(<-writer.payloadCh) {
break
}
}
if !writer.flushRecords() {
break
}
@@ -316,23 +324,8 @@ func (writer *messageWriter) start() {
break
}
writer.write(endMessage)
} else {
for payload.Len() > 0 {
copiedLen := copy(writer.payloadBuffer[writer.payloadBufferIndex:], payload.Bytes())
writer.payloadBufferIndex += copiedLen
payload.Next(copiedLen)
// If buffer is filled, flush it now!
freeSpace := bufLength - writer.payloadBufferIndex
if freeSpace == 0 {
if !writer.flushRecords() {
quitFlag = true
break
}
}
}
bufPool.Put(payload)
} else if !writer.stageRecord(payload) {
quitFlag = true
}
case <-recordStagingTicker.C:
@@ -368,6 +361,26 @@ func (writer *messageWriter) start() {
}
}
// stageRecord copies a record into the payload buffer, flushing whenever
// the buffer fills, and returns the buffer to the pool. It reports false when
// a flush failed.
func (writer *messageWriter) stageRecord(payload *bytes.Buffer) bool {
defer bufPool.Put(payload)
for payload.Len() > 0 {
copiedLen := copy(writer.payloadBuffer[writer.payloadBufferIndex:], payload.Bytes())
writer.payloadBufferIndex += copiedLen
payload.Next(copiedLen)
// If buffer is filled, flush it now!
if bufLength-writer.payloadBufferIndex == 0 {
if !writer.flushRecords() {
return false
}
}
}
return true
}
// Sends a single whole record.
func (writer *messageWriter) SendRecord(payload *bytes.Buffer) error {
select {
internal/s3select/message_test.go
+193
View File
@@ -0,0 +1,193 @@
// Copyright (c) 2026 PGSTY
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
import (
"bytes"
"io"
"net/http"
"strings"
"testing"
"github.com/minio/minio-go/v7"
)
// decodeEvents parses a recorded event stream with the minio-go client and
// returns the concatenated record payloads and the terminal error, if any.
func decodeEvents(t *testing.T, response []byte) ([]byte, error) {
t.Helper()
body := &testResponseBody{Reader: bytes.NewReader(response), closed: make(chan struct{})}
res, err := minio.NewSelectResults(&http.Response{StatusCode: http.StatusOK, Body: body, ContentLength: int64(len(response))}, "testbucket")
if err != nil {
t.Fatal(err)
}
records, readErr := io.ReadAll(res)
<-body.closed
return records, readErr
}
func newTestRecord(content string) *bytes.Buffer {
payload := bufPool.Get()
payload.Reset()
payload.WriteString(content)
return payload
}
// eventOrder returns the byte offsets of the given markers in the response,
// or -1 for a marker that is absent.
func eventOrder(response []byte, markers ...string) []int {
offsets := make([]int, len(markers))
for i, marker := range markers {
offsets[i] = bytes.Index(response, []byte(marker))
}
return offsets
}
func ascending(offsets []int) bool {
for i, offset := range offsets {
if offset < 0 || (i > 0 && offset <= offsets[i-1]) {
return false
}
}
return true
}
// TestMessageWriterFlushesQueuedRecordBeforeError sends a record and an error
// back to back, so the writer goroutine sees both channels ready and picks
// one at random. The record must appear in the response before the error
// message every time.
func TestMessageWriterFlushesQueuedRecordBeforeError(t *testing.T) {
for i := range 200 {
w := &testResponseWriter{}
writer := newMessageWriter(w, nil)
payload := bufPool.Get()
payload.Reset()
payload.WriteString(`{"id":1}` + "\n")
if err := writer.SendRecord(payload); err != nil {
t.Fatalf("run %d: SendRecord: %v", i, err)
}
if err := writer.FinishWithError("OverMaxRecordSize", "too large"); err != nil {
t.Fatalf("run %d: FinishWithError: %v", i, err)
}
record := bytes.Index(w.response, []byte(`{"id":1}`))
errMsg := bytes.Index(w.response, []byte("OverMaxRecordSize"))
if record < 0 || errMsg < 0 || record > errMsg {
t.Fatalf("run %d: record at %d, error at %d: queued record was dropped or reordered", i, record, errMsg)
}
}
}
// TestMessageWriterFlushesBufferedAndQueuedRecordsBeforeError: one record has
// already been staged into the buffer and a second one is still queued when
// the error arrives. Both must precede the error, in order.
func TestMessageWriterFlushesBufferedAndQueuedRecordsBeforeError(t *testing.T) {
for i := range 100 {
w := &testResponseWriter{}
writer := newMessageWriter(w, nil)
if err := writer.SendRecord(newTestRecord(`{"id":1}` + "\n")); err != nil {
t.Fatal(err)
}
// Give the writer a chance to stage the first record; whether it
// did or not, the outcome must be the same.
if i%2 == 0 {
for range 100 {
if len(writer.payloadCh) == 0 {
break
}
}
}
if err := writer.SendRecord(newTestRecord(`{"id":2}` + "\n")); err != nil {
t.Fatal(err)
}
if err := writer.FinishWithError("OverMaxRecordSize", "too large"); err != nil {
t.Fatal(err)
}
if got := eventOrder(w.response, `{"id":1}`, `{"id":2}`, "OverMaxRecordSize"); !ascending(got) {
t.Fatalf("run %d: offsets %v: records must precede the error in order", i, got)
}
}
}
// TestMessageWriterErrorWithoutRecords: an error with nothing queued writes
// only the error message, no empty Records event.
func TestMessageWriterErrorWithoutRecords(t *testing.T) {
w := &testResponseWriter{}
writer := newMessageWriter(w, nil)
if err := writer.FinishWithError("InternalError", "boom"); err != nil {
t.Fatal(err)
}
if !bytes.Contains(w.response, []byte("InternalError")) || bytes.Contains(w.response, []byte("Records")) {
t.Fatalf("unexpected response: %q", w.response)
}
}
// TestMessageWriterStagesRecordsLargerThanTheBuffer: a record larger than the
// staging buffer is split across several Records events and nothing is lost
// or reordered, whether the stream ends with success or with an error.
func TestMessageWriterStagesRecordsLargerThanTheBuffer(t *testing.T) {
big := strings.Repeat("x", bufLength+bufLength/2)
want := "head\n" + big + "\n" + "tail\n"
for _, withError := range []bool{false, true} {
w := &testResponseWriter{}
writer := newMessageWriter(w, nil)
for _, rec := range []string{"head\n", big + "\n", "tail\n"} {
if err := writer.SendRecord(newTestRecord(rec)); err != nil {
t.Fatal(err)
}
}
if withError {
if err := writer.FinishWithError("OverMaxRecordSize", "too large"); err != nil {
t.Fatal(err)
}
} else if err := writer.Finish(10, 10); err != nil {
t.Fatal(err)
}
records, err := decodeEvents(t, w.response)
if string(records) != want {
t.Fatalf("withError=%v: got %d record bytes, want %d; head=%q", withError, len(records), len(want), string(records[:min(len(records), 8)]))
}
if withError && (err == nil || !strings.Contains(err.Error(), "OverMaxRecordSize")) {
t.Fatalf("expected OverMaxRecordSize after the records, got %v", err)
}
if !withError && err != nil {
t.Fatalf("unexpected error on the success path: %v", err)
}
}
}
// TestMessageWriterSuccessOrder: the success path is unchanged: every record,
// then Stats, then End, and the client sees no error.
func TestMessageWriterSuccessOrder(t *testing.T) {
w := &testResponseWriter{}
writer := newMessageWriter(w, nil)
for _, rec := range []string{`{"id":1}`, `{"id":2}`} {
if err := writer.SendRecord(newTestRecord(rec + "\n")); err != nil {
t.Fatal(err)
}
}
if err := writer.Finish(20, 20); err != nil {
t.Fatal(err)
}
if got := eventOrder(w.response, `{"id":1}`, `{"id":2}`, "Stats", "End"); !ascending(got) {
t.Fatalf("offsets %v", got)
}
records, err := decodeEvents(t, w.response)
if err != nil || string(records) != "{\"id\":1}\n{\"id\":2}\n" {
t.Fatalf("records %q err %v", records, err)
}
}
internal/s3select/select.go
+1 -1
View File
@@ -39,7 +39,7 @@ import (
"github.com/minio/minio/internal/s3select/json"
"github.com/minio/minio/internal/s3select/parquet"
"github.com/minio/minio/internal/s3select/sql"
"github.com/minio/pkg/v3/env"
"github.com/pgsty/silo-pkg/v3/env"
"github.com/pierrec/lz4/v4"
)