diff --git a/.github/workflows/vulncheck.yml b/.github/workflows/vulncheck.yml index 87a4aa7e9..63fc1c634 100644 --- a/.github/workflows/vulncheck.yml +++ b/.github/workflows/vulncheck.yml @@ -29,7 +29,7 @@ jobs: - name: Install govulncheck run: | - go install golang.org/x/vuln/cmd/govulncheck@v1.6.0 + go install golang.org/x/vuln/cmd/govulncheck@v1.7.0 echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}" - name: Run govulncheck diff --git a/.golangci.yml b/.golangci.yml index 0533d7cd9..2f5cfc6aa 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -59,6 +59,9 @@ formatters: exclusions: generated: lax paths: + # gofumpt v0.11.0 and Go 1.27's gofmt disagree on the indentation of + # multiple composite literals returned from a single statement. + - internal/ioutil/wait_pipe\.go$ - third_party$ - builtin$ - examples$ diff --git a/CREDITS b/CREDITS index 256a8bf37..4dc7f36c9 100644 --- a/CREDITS +++ b/CREDITS @@ -3396,7 +3396,7 @@ SOFTWARE. github.com/cheggaaa/pb https://github.com/cheggaaa/pb ---------------------------------------------------------------- -Copyright (c) 2012-2015, Sergey Cherepanov +Copyright (c) 2012-2024, Sergey Cherepanov All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -16641,33 +16641,6 @@ SOFTWARE. ================================================================ -github.com/lestrrat-go/option -https://github.com/lestrrat-go/option ----------------------------------------------------------------- -MIT License - -Copyright (c) 2021 lestrrat-go - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -================================================================ - github.com/lestrrat-go/option/v2 https://github.com/lestrrat-go/option/v2 ---------------------------------------------------------------- @@ -21713,6 +21686,14 @@ separate copyright notices and license terms. Your use of the source code for the these subcomponents is subject to the terms and conditions of the following licenses. +=== + +This distribution (pgsty/mc, shipped as "mcli") is a community-maintained +fork of the MinIO Client, modified by the Silo project +(https://silo.pgsty.com/). It is not affiliated with, endorsed by, or +sponsored by MinIO, Inc. Modifications are Copyright (c) 2025-2026 PGSTY +and are licensed under the GNU AGPL v3.0 or later. + ================================================================ github.com/minio/md5-simd @@ -28982,7 +28963,7 @@ https://go.etcd.io/etcd/api/v3 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29190,7 +29171,7 @@ https://go.etcd.io/etcd/client/pkg/v3 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29398,7 +29379,7 @@ https://go.etcd.io/etcd/client/v3 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index a09d75e14..15d03aadf 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM golang:1.26.5-alpine AS build +FROM golang:1.27.0-alpine AS build ARG TARGETARCH diff --git a/Makefile b/Makefile index 96cfe0162..529913f85 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go) GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) -GOLANGCI_VERSION ?= v2.11.3 +GOLANGCI_VERSION ?= v2.13.1 VERSION ?= $(shell git describe --tags) REPO ?= docker.io/pgsty @@ -48,7 +48,7 @@ credits: ## regenerate CREDITS from the licenses of Go modules linked into the b check-gen: ## check for updated autogenerated files @go generate ./... >/dev/null - @go mod tidy -compat=1.26 + @go mod tidy -compat=1.27 @env bash $(PWD)/buildscripts/gen-credits.sh @changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum CREDITS); \ if [ -n "$$changed" ]; then \ diff --git a/buildscripts/checkdeps.sh b/buildscripts/checkdeps.sh index ed4f666ea..4fc385a29 100755 --- a/buildscripts/checkdeps.sh +++ b/buildscripts/checkdeps.sh @@ -7,7 +7,7 @@ _init() { ## Minimum required versions for build dependencies GIT_VERSION="1.0" - GO_VERSION="1.16" + GO_VERSION="1.27.0" OSX_VERSION="10.8" KNAME=$(uname -s) ARCH=$(uname -m) diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index afa12e505..b50665717 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -1825,9 +1825,20 @@ "cmd:cmd:field:CompletePart.PartNumber", "cmd:cmd:field:CompletePart.Size", "cmd:cmd:field:ConsoleLogger.Enable", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32C", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC64NVME", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA1", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA256", "cmd:cmd:field:CopyObjectPartResponse.ETag", "cmd:cmd:field:CopyObjectPartResponse.LastModified", "cmd:cmd:field:CopyObjectPartResponse.XMLName", + "cmd:cmd:field:CopyObjectResponse.ChecksumCRC32", + "cmd:cmd:field:CopyObjectResponse.ChecksumCRC32C", + "cmd:cmd:field:CopyObjectResponse.ChecksumCRC64NVME", + "cmd:cmd:field:CopyObjectResponse.ChecksumSHA1", + "cmd:cmd:field:CopyObjectResponse.ChecksumSHA256", + "cmd:cmd:field:CopyObjectResponse.ChecksumType", "cmd:cmd:field:CopyObjectResponse.ETag", "cmd:cmd:field:CopyObjectResponse.LastModified", "cmd:cmd:field:CopyObjectResponse.XMLName", @@ -6939,7 +6950,9 @@ "internal/config/notify:notify:func:SetNotifyRedis", "internal/config/notify:notify:func:SetNotifyWebhook", "internal/config/notify:notify:func:TestSubSysNotificationTargets", + "internal/config/notify:notify:method:LegacyDatabaseTargetError.Error", "internal/config/notify:notify:type:Config", + "internal/config/notify:notify:type:LegacyDatabaseTargetError", "internal/config/notify:notify:var:DefaultAMQPKVS", "internal/config/notify:notify:var:DefaultESKVS", "internal/config/notify:notify:var:DefaultKafkaKVS", diff --git a/cmd/admin-handlers-users.go b/cmd/admin-handlers-users.go index 8530046b9..eb032672d 100644 --- a/cmd/admin-handlers-users.go +++ b/cmd/admin-handlers-users.go @@ -859,7 +859,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re var sp *policy.Policy if len(updateReq.NewPolicy) > 0 { - sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy)) + sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy)) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -1729,7 +1729,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request return } - iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes)) + iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes)) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -2981,7 +2981,7 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth. var sp *policy.Policy if len(createReq.Policy) > 0 { - sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy)) + sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy)) if err != nil { return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err) } diff --git a/cmd/admin-handlers-users_test.go b/cmd/admin-handlers-users_test.go index 828264583..350edf823 100644 --- a/cmd/admin-handlers-users_test.go +++ b/cmd/admin-handlers-users_test.go @@ -204,6 +204,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) { suite.TestUserCreate(c) suite.TestUserPolicyEscalationBug(c) suite.TestPolicyCreate(c) + suite.TestServiceAccountBareARNPolicyRejected(c) suite.TestCannedPolicies(c) suite.TestGroupAddRemove(c) suite.TestServiceAccountOpsByAdmin(c) @@ -600,6 +601,20 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) { c.Fatalf("invalid policy creation success") } + for i, resource := range []string{"arn:aws:s3:::", "*arn:aws:s3:::"} { + barePolicyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Deny", + "Action": ["s3:GetObject"], + "Resource": ["%s"] + }] +}`, resource) + if err = s.adm.AddCannedPolicy(ctx, fmt.Sprintf("%s-bare-%d", policy, i), barePolicyBytes); err == nil { + c.Fatalf("bare ARN policy creation succeeded for %q", resource) + } + } + // 3. Create a user, associate policy and verify access accessKey, secretKey := mustGenerateCredentials(c) err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) @@ -653,6 +668,51 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) { } } +func (s *TestSuiteIAM) TestServiceAccountBareARNPolicyRejected(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + barePolicy := []byte(`{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:GetObject"], + "NotResource": ["arn:aws:s3:::"] + }] +}`) + if _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: globalActiveCred.AccessKey, + Policy: barePolicy, + }); err == nil { + c.Fatal("service account creation accepted a bare ARN policy") + } + + validPolicy := []byte(`{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::*"] + }] +}`) + credentials, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: globalActiveCred.AccessKey, + Policy: validPolicy, + }) + if err != nil { + c.Fatalf("service account creation rejected an explicit resource: %v", err) + } + defer func() { + _ = s.adm.DeleteServiceAccount(ctx, credentials.AccessKey) + }() + + if err = s.adm.UpdateServiceAccount(ctx, credentials.AccessKey, madmin.UpdateServiceAccountReq{ + NewPolicy: barePolicy, + }); err == nil { + c.Fatal("service account update accepted a bare ARN policy") + } +} + func (s *TestSuiteIAM) TestCannedPolicies(c *check) { ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) defer cancel() diff --git a/cmd/api-response.go b/cmd/api-response.go index 9ed172458..c6ad7b07e 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -27,7 +27,6 @@ import ( "path" "strconv" "strings" - "time" "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/crypto" @@ -380,6 +379,13 @@ type CopyObjectResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` LastModified string // time string of format "2006-01-02T15:04:05.000Z" ETag string // md5sum of the copied object. + + ChecksumCRC32 string `xml:",omitempty"` + ChecksumCRC32C string `xml:",omitempty"` + ChecksumSHA1 string `xml:",omitempty"` + ChecksumSHA256 string `xml:",omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` + ChecksumType string `xml:",omitempty"` } // CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object @@ -387,6 +393,12 @@ type CopyObjectPartResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"` LastModified string // time string of format "2006-01-02T15:04:05.000Z" ETag string // md5sum of the copied object part. + + ChecksumCRC32 string `xml:",omitempty"` + ChecksumCRC32C string `xml:",omitempty"` + ChecksumSHA1 string `xml:",omitempty"` + ChecksumSHA256 string `xml:",omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` } // Initiator inherit from Owner struct, fields are same @@ -764,19 +776,31 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode) -// generates CopyObjectResponse from etag and lastModified time. -func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse { +// generates CopyObjectResponse from the committed object information. +func generateCopyObjectResponse(oi ObjectInfo, h http.Header) CopyObjectResponse { + cs, _ := oi.decryptChecksums(0, h) return CopyObjectResponse{ - ETag: "\"" + etag + "\"", - LastModified: amztime.ISO8601Format(lastModified.UTC()), + ETag: "\"" + oi.ETag + "\"", + LastModified: amztime.ISO8601Format(oi.ModTime.UTC()), + ChecksumCRC32: cs[hash.ChecksumCRC32.String()], + ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()], + ChecksumSHA1: cs[hash.ChecksumSHA1.String()], + ChecksumSHA256: cs[hash.ChecksumSHA256.String()], + ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()], + ChecksumType: cs[xhttp.AmzChecksumType], } } -// generates CopyObjectPartResponse from etag and lastModified time. -func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse { +// generates CopyObjectPartResponse from the uploaded part information. +func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse { return CopyObjectPartResponse{ - ETag: "\"" + etag + "\"", - LastModified: amztime.ISO8601Format(lastModified.UTC()), + ETag: "\"" + partInfo.ETag + "\"", + LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()), + ChecksumCRC32: partInfo.ChecksumCRC32, + ChecksumCRC32C: partInfo.ChecksumCRC32C, + ChecksumSHA1: partInfo.ChecksumSHA1, + ChecksumSHA256: partInfo.ChecksumSHA256, + ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME, } } diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index d0ad85144..33a0ebda5 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -198,7 +198,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec byPassSet, r, cred, owner) // Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set. if !byPassSet { - if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) { + if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) { return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID} } } @@ -209,7 +209,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec case objectlock.RetCompliance: // Compliance retention mode cannot be changed or shortened. // https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes - if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) { + if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) { return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID} } apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name, diff --git a/cmd/common-main.go b/cmd/common-main.go index 18aad7346..7f6ead797 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -36,6 +36,7 @@ import ( "strings" "syscall" "time" + "unicode" "github.com/dustin/go-humanize" fcolor "github.com/fatih/color" @@ -540,6 +541,34 @@ func (e envKV) String() string { return fmt.Sprintf("%s=%s", e.Key, e.Value) } +func isValidEnvName(name string) bool { + if name == "" || !isEnvNameStart(name[0]) { + return false + } + for i := 1; i < len(name); i++ { + if !isEnvNameStart(name[i]) && (name[i] < '0' || name[i] > '9') { + return false + } + } + return true +} + +func isEnvNameStart(ch byte) bool { + return ch == '_' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z' +} + +func trimExportPrefix(envEntry string) string { + rest, ok := strings.CutPrefix(envEntry, "export") + if !ok || rest == "" { + return envEntry + } + trimmed := strings.TrimLeftFunc(rest, unicode.IsSpace) + if len(trimmed) == len(rest) { + return envEntry + } + return trimmed +} + func parsEnvEntry(envEntry string) (envKV, error) { envEntry = strings.TrimSpace(envEntry) if envEntry == "" { @@ -554,13 +583,19 @@ func parsEnvEntry(envEntry string) (envKV, error) { Skip: true, }, nil } - envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2) + envTokens := strings.SplitN(trimExportPrefix(envEntry), config.EnvSeparator, 2) if len(envTokens) != 2 { - return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry) + return envKV{}, errors.New("missing '='") } - key := envTokens[0] - val := envTokens[1] + key := strings.TrimSpace(envTokens[0]) + val := strings.TrimSpace(envTokens[1]) + if !isValidEnvName(key) { + return envKV{}, fmt.Errorf("invalid environment variable name %q", key) + } + if strings.IndexByte(val, 0) >= 0 { + return envKV{}, errors.New("environment variable value contains NUL") + } // Remove quotes from the value if found if len(val) >= 2 { @@ -587,10 +622,12 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) { defer f.Close() var ekvs []envKV scanner := bufio.NewScanner(f) + lineNo := 0 for scanner.Scan() { + lineNo++ ekv, err := parsEnvEntry(scanner.Text()) if err != nil { - return nil, err + return nil, fmt.Errorf("%s:%d: %w", envConfigFile, lineNo, err) } if ekv.Skip { // Skips empty lines @@ -599,7 +636,7 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) { ekvs = append(ekvs, ekv) } if err = scanner.Err(); err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", envConfigFile, err) } return ekvs, nil } @@ -666,12 +703,15 @@ func loadEnvVarsFromFiles() { } if env.IsSet(config.EnvConfigEnvFile) { - ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, "")) + envConfigFile := env.Get(config.EnvConfigEnvFile, "") + ekvs, err := minioEnvironFromFile(envConfigFile) if err != nil && !os.IsNotExist(err) { logger.Fatal(err, "Unable to read the config environment file") } for _, ekv := range ekvs { - os.Setenv(ekv.Key, ekv.Value) + if err := os.Setenv(ekv.Key, ekv.Value); err != nil { + logger.Fatal(err, "Unable to set %s from config environment file %s", ekv.Key, envConfigFile) + } } } } diff --git a/cmd/common-main_test.go b/cmd/common-main_test.go index 9757267d2..341ddba49 100644 --- a/cmd/common-main_test.go +++ b/cmd/common-main_test.go @@ -19,8 +19,10 @@ package cmd import ( "errors" + "fmt" "os" "reflect" + "strings" "testing" ) @@ -181,3 +183,162 @@ MINIO_ROOT_PASSWORD=minio123`, }) } } + +func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) { + testCases := []struct { + name string + content string + want []envKV + errLine int + errContains string + errExcludes string + }{ + { + name: "spaces and tabs around separator", + content: "MINIO_ROOT_USER = minio\nMINIO_ROOT_PASSWORD\t=\tminio123", + want: []envKV{ + {Key: "MINIO_ROOT_USER", Value: "minio"}, + {Key: "MINIO_ROOT_PASSWORD", Value: "minio123"}, + }, + }, + { + name: "export tab and quoted spaces", + content: "export\tMINIO_ROOT_USER = \" minio user \"\nexport MINIO_ROOT_PASSWORD = ' minio secret '", + want: []envKV{ + {Key: "MINIO_ROOT_USER", Value: " minio user "}, + {Key: "MINIO_ROOT_PASSWORD", Value: " minio secret "}, + }, + }, + { + name: "export Unicode whitespace", + content: "export\u00a0MINIO_ROOT_USER=value", + want: []envKV{ + {Key: "MINIO_ROOT_USER", Value: "value"}, + }, + }, + { + name: "export is only a standalone prefix", + content: "export=value\nexportFOO=bar", + want: []envKV{ + {Key: "export", Value: "value"}, + {Key: "exportFOO", Value: "bar"}, + }, + }, + { + name: "unquoted whitespace empty value and additional separators", + content: "UNQUOTED = value \nEMPTY =\nTOKEN = scheme://user:password@example.com?a=b", + want: []envKV{ + {Key: "UNQUOTED", Value: "value"}, + {Key: "EMPTY", Value: ""}, + {Key: "TOKEN", Value: "scheme://user:password@example.com?a=b"}, + }, + }, + { + name: "valid underscore and digits", + content: "_VALID_2=value", + want: []envKV{ + {Key: "_VALID_2", Value: "value"}, + }, + }, + { + name: "missing separator redacts the line", + content: "MINIO_ROOT_PASSWORD=valid\nsuper-secret-without-equals", + errLine: 2, + errContains: "missing '='", + errExcludes: "super-secret-without-equals", + }, + { + name: "empty name", + content: "=empty-name-secret", + errLine: 1, + errContains: `invalid environment variable name ""`, + errExcludes: "empty-name-secret", + }, + { + name: "digit leading name", + content: "1MINIO_ROOT_USER=digit-leading-secret", + errLine: 1, + errContains: `invalid environment variable name "1MINIO_ROOT_USER"`, + errExcludes: "digit-leading-secret", + }, + { + name: "hyphenated name", + content: "MINIO-ROOT-USER=hyphen-secret", + errLine: 1, + errContains: `invalid environment variable name "MINIO-ROOT-USER"`, + errExcludes: "hyphen-secret", + }, + { + name: "whitespace in name", + content: "MINIO ROOT USER=whitespace-secret", + errLine: 1, + errContains: `invalid environment variable name "MINIO ROOT USER"`, + errExcludes: "whitespace-secret", + }, + { + name: "NUL in name", + content: "MINIO\x00ROOT=nul-name-secret", + errLine: 1, + errContains: "invalid environment variable name", + errExcludes: "nul-name-secret", + }, + { + name: "NUL in value", + content: "MINIO_ROOT_USER=before\x00nul-value-secret", + errLine: 1, + errContains: "environment variable value contains NUL", + errExcludes: "nul-value-secret", + }, + { + name: "diagnostic has file and line but no value", + content: "MINIO_ROOT_USER=valid\nBAD KEY=super-secret-value", + errLine: 2, + errContains: `invalid environment variable name "BAD KEY"`, + errExcludes: "super-secret-value", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + tmpfile, err := os.CreateTemp(t.TempDir(), "testfile") + if err != nil { + t.Fatal(err) + } + if _, err = tmpfile.WriteString(testCase.content); err != nil { + t.Fatal(err) + } + if err = tmpfile.Close(); err != nil { + t.Fatal(err) + } + + got, err := minioEnvironFromFile(tmpfile.Name()) + if testCase.errContains == "" { + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, testCase.want) { + t.Errorf("expected %v, got %v", testCase.want, got) + } + return + } + + if err == nil { + t.Fatal("expected an error") + } + errText := err.Error() + location := fmt.Sprintf("%s:%d:", tmpfile.Name(), testCase.errLine) + if !strings.Contains(errText, location) { + t.Errorf("expected error to contain %q, got %q", location, errText) + } + if !strings.Contains(errText, testCase.errContains) { + t.Errorf("expected error to contain %q, got %q", testCase.errContains, errText) + } + if testCase.errExcludes != "" && strings.Contains(errText, testCase.errExcludes) { + t.Errorf("expected error to redact %q, got %q", testCase.errExcludes, errText) + } + if got != nil { + t.Errorf("expected no entries on parse error, got %v", got) + } + }) + } +} diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 0cceb1859..170279ab5 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -167,7 +167,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C notify.SetNotifyMQTT(newCfg, k, args) } for k, args := range cfg.Notify.MySQL { - notify.SetNotifyMySQL(newCfg, k, args) + if err := notify.SetNotifyMySQL(newCfg, k, args); err != nil { + return nil, err + } } for k, args := range cfg.Notify.NATS { notify.SetNotifyNATS(newCfg, k, args) @@ -176,7 +178,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C notify.SetNotifyNSQ(newCfg, k, args) } for k, args := range cfg.Notify.PostgreSQL { - notify.SetNotifyPostgres(newCfg, k, args) + if err := notify.SetNotifyPostgres(newCfg, k, args); err != nil { + return nil, err + } } for k, args := range cfg.Notify.Redis { notify.SetNotifyRedis(newCfg, k, args) diff --git a/cmd/config-migrate_test.go b/cmd/config-migrate_test.go new file mode 100644 index 000000000..76a616d9b --- /dev/null +++ b/cmd/config-migrate_test.go @@ -0,0 +1,264 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/notify" + "github.com/minio/minio/internal/event/target" +) + +func installLegacyConfigFile(t *testing.T, configure func(*serverConfigV33)) (string, []byte) { + t.Helper() + + cfg := &serverConfigV33{ + Version: "33", + Notify: notify.NewConfig(), + } + configure(cfg) + + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + + oldConfigDir := globalConfigDir + globalConfigDir = &ConfigDir{path: t.TempDir()} + t.Cleanup(func() { globalConfigDir = oldConfigDir }) + + configFile := getConfigFile() + if err = os.WriteFile(configFile, data, 0o600); err != nil { + t.Fatal(err) + } + return configFile, data +} + +func assertLegacyMigrationError(t *testing.T, err error, subsystem, name, key, secret string) { + t.Helper() + var targetErr *notify.LegacyDatabaseTargetError + if !errors.As(err, &targetErr) { + t.Fatalf("error = %v, want *notify.LegacyDatabaseTargetError", err) + } + msg := err.Error() + for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not contain %q", msg, want) + } + } + if strings.Contains(msg, secret) { + t.Errorf("migration error leaks database password %q: %s", secret, msg) + } +} + +func TestReadConfigWithoutMigrateRejectsLegacyDatabaseTargets(t *testing.T) { + tests := []struct { + name string + subsystem string + key string + secret string + configure func(*serverConfigV33) + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + secret: "postgres-migration-secret", + configure: func(cfg *serverConfigV33) { + cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{ + Enable: true, + Port: "5432", + Username: "legacy-user", + Password: "postgres-migration-secret", + Database: "events", + } + }, + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + secret: "mysql-migration-secret", + configure: func(cfg *serverConfigV33) { + cfg.Notify.MySQL["archive"] = target.MySQLArgs{ + Enable: true, + Port: "3306", + User: "legacy-user", + Password: "mysql-migration-secret", + Database: "events", + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + configFile, original := installLegacyConfigFile(t, test.configure) + got, err := readConfigWithoutMigrate(t.Context(), nil) + if got != nil { + t.Fatalf("config = %v, want nil on failed migration", got) + } + assertLegacyMigrationError(t, err, test.subsystem, "archive", test.key, test.secret) + + after, readErr := os.ReadFile(configFile) + if readErr != nil { + t.Fatal(readErr) + } + if !bytes.Equal(after, original) { + t.Fatal("failed migration rewrote the legacy source config") + } + if _, statErr := os.Stat(configFile + ".old"); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("failed migration created a backup/persistence artifact: %v", statErr) + } + }) + } +} + +func TestReadConfigWithoutMigrateMigratesCanonicalDatabaseTargets(t *testing.T) { + const ( + postgresConnection = "host=postgres.example port=5432 dbname=events user=app password=secret sslmode=disable" + mysqlDSN = "app:secret@tcp(mysql.example:3306)/events?parseTime=true" + discardedLegacyValue = "discarded-legacy-value" + ) + installLegacyConfigFile(t, func(cfg *serverConfigV33) { + cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{ + Enable: true, + Format: "namespace", + ConnectionString: postgresConnection, + Table: "events", + Port: discardedLegacyValue, + Username: discardedLegacyValue, + Password: discardedLegacyValue, + Database: discardedLegacyValue, + } + cfg.Notify.MySQL["archive"] = target.MySQLArgs{ + Enable: true, + Format: "namespace", + DSN: mysqlDSN, + Table: "events", + Port: discardedLegacyValue, + User: discardedLegacyValue, + Password: discardedLegacyValue, + Database: discardedLegacyValue, + } + }) + + got, err := readConfigWithoutMigrate(t.Context(), nil) + if err != nil { + t.Fatalf("readConfigWithoutMigrate: %v", err) + } + tests := []struct { + subsystem string + key string + want string + discarded string + }{ + {config.NotifyPostgresSubSys, target.PostgresConnectionString, postgresConnection, discardedLegacyValue}, + {config.NotifyMySQLSubSys, target.MySQLDSNString, mysqlDSN, discardedLegacyValue}, + } + for _, test := range tests { + kvs := got[test.subsystem]["archive"] + if value := kvs.Get(test.key); value != test.want { + t.Errorf("%s %s = %q, want %q", test.subsystem, test.key, value, test.want) + } + if err := config.CheckValidKeys(test.subsystem+config.SubSystemSeparator+"archive", kvs, notify.DefaultNotificationKVS[test.subsystem]); err != nil { + t.Errorf("migrated %s target failed key validation: %v", test.subsystem, err) + } + for _, key := range []string{"host", "port", "username", "password", "database"} { + if _, ok := kvs.Lookup(key); ok { + t.Errorf("migrated %s target contains legacy key %q", test.subsystem, key) + } + } + for _, kv := range kvs { + if strings.Contains(kv.Value, test.discarded) { + t.Errorf("migrated %s target contains discarded legacy value in %q", test.subsystem, kv.Key) + } + } + } + + postgresTargets, err := notify.GetNotifyPostgres(got[config.NotifyPostgresSubSys]) + if err != nil { + t.Fatalf("GetNotifyPostgres: %v", err) + } + if value := postgresTargets["archive"].ConnectionString; value != postgresConnection { + t.Errorf("Postgres connection string = %q, want %q", value, postgresConnection) + } + mysqlTargets, err := notify.GetNotifyMySQL(got[config.NotifyMySQLSubSys]) + if err != nil { + t.Fatalf("GetNotifyMySQL: %v", err) + } + if value := mysqlTargets["archive"].DSN; value != mysqlDSN { + t.Errorf("MySQL DSN = %q, want %q", value, mysqlDSN) + } +} + +func TestInitConfigSubsystemReturnsLegacyDatabaseTargetError(t *testing.T) { + obj, fsDir, err := prepareFS(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = obj.Shutdown(context.Background()) + _ = os.RemoveAll(fsDir) + }) + + const secret = "startup-migration-secret" + installLegacyConfigFile(t, func(cfg *serverConfigV33) { + cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{ + Enable: true, + Port: "5432", + Username: "legacy-user", + Password: secret, + Database: "events", + } + }) + + globalServerConfigMu.RLock() + var before config.Config + if globalServerConfig != nil { + before = globalServerConfig.Clone() + } + globalServerConfigMu.RUnlock() + + err = initConfigSubsystem(t.Context(), obj) + assertLegacyMigrationError(t, err, config.NotifyPostgresSubSys, "archive", target.PostgresConnectionString, secret) + if configRetriableErrors(err) { + t.Fatal("legacy database migration error must be startup-fatal, not retriable") + } + if !fatalServerConfigError(err) { + t.Fatal("legacy database migration error must abort server startup") + } + + globalServerConfigMu.RLock() + var after config.Config + if globalServerConfig != nil { + after = globalServerConfig.Clone() + } + globalServerConfigMu.RUnlock() + if !reflect.DeepEqual(after, before) { + t.Fatal("failed migration activated a partial server configuration") + } +} diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index 5752daae0..48ec09b2c 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -326,7 +326,7 @@ func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool { if cycles <= 1 { return cycles == 1 } - return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles + return uint32(xxhash.Sum64String(string(h))>>32)%cycles == cycle%cycles } // addChild will add a child based on its hash. diff --git a/cmd/erasure-multipart-upload-checksum_test.go b/cmd/erasure-multipart-upload-checksum_test.go new file mode 100644 index 000000000..dd10a2026 --- /dev/null +++ b/cmd/erasure-multipart-upload-checksum_test.go @@ -0,0 +1,596 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/xml" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +func uploadPartHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, object, uploadID string, partNumber int, data []byte, headers map[string]string, +) (string, *httptest.ResponseRecorder) { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(partNumber)), + int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build UploadPart request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("UploadPart failed: %d %s", rec.Code, rec.Body.String()) + } + return canonicalizeETag(rec.Header()[xhttp.ETag][0]), rec +} + +func listPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, object, uploadID string, headers map[string]string, +) ListPartsResponse { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodGet, + getListMultipartURLWithParams("", bucket, object, uploadID, "1000", "", ""), + 0, nil, creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build ListParts request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("ListParts failed: %d %s", rec.Code, rec.Body.String()) + } + var response ListPartsResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("failed to decode ListParts response: %v", err) + } + return response +} + +func partChecksum(typ hash.ChecksumType, part Part) string { + switch typ.Base() { + case hash.ChecksumCRC32: + return part.ChecksumCRC32 + case hash.ChecksumCRC32C: + return part.ChecksumCRC32C + case hash.ChecksumSHA1: + return part.ChecksumSHA1 + case hash.ChecksumSHA256: + return part.ChecksumSHA256 + case hash.ChecksumCRC64NVME: + return part.ChecksumCRC64NVME + default: + return "" + } +} + +func copyPartChecksum(typ hash.ChecksumType, response CopyObjectPartResponse) string { + switch typ.Base() { + case hash.ChecksumCRC32: + return response.ChecksumCRC32 + case hash.ChecksumCRC32C: + return response.ChecksumCRC32C + case hash.ChecksumSHA1: + return response.ChecksumSHA1 + case hash.ChecksumSHA256: + return response.ChecksumSHA256 + case hash.ChecksumCRC64NVME: + return response.ChecksumCRC64NVME + default: + return "" + } +} + +func completePartWithChecksum(typ hash.ChecksumType, partNumber int, etag, checksum string) CompletePart { + part := CompletePart{PartNumber: partNumber, ETag: etag} + switch typ.Base() { + case hash.ChecksumCRC32: + part.ChecksumCRC32 = checksum + case hash.ChecksumCRC32C: + part.ChecksumCRC32C = checksum + case hash.ChecksumSHA1: + part.ChecksumSHA1 = checksum + case hash.ChecksumSHA256: + part.ChecksumSHA256 = checksum + case hash.ChecksumCRC64NVME: + part.ChecksumCRC64NVME = checksum + } + return part +} + +func completePartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, object, uploadID string, parts []CompletePart, headers map[string]string, +) *httptest.ResponseRecorder { + t.Helper() + body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts}) + if err != nil { + t.Fatalf("failed to encode CompleteMultipartUpload request: %v", err) + } + req, err := newTestSignedRequestV4(http.MethodPost, + getCompleteMultipartUploadURL("", bucket, object, uploadID), + int64(len(body)), bytes.NewReader(body), creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build CompleteMultipartUpload request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + return rec +} + +func copyPartWithoutChecksumHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, source, object, uploadID, sourceRange string, headers map[string]string, +) CopyObjectPartResponse { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, + getCopyObjectPartURL("", bucket, object, uploadID, "1"), + 0, nil, creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build UploadPartCopy request: %v", err) + } + req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source)) + if sourceRange != "" { + req.Header.Set(xhttp.AmzCopySourceRange, sourceRange) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("UploadPartCopy failed: %d %s", rec.Code, rec.Body.String()) + } + var response CopyObjectPartResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("failed to decode UploadPartCopy response: %v", err) + } + return response +} + +// TestAPIUploadPartServerSideChecksum exercises the data transformations that +// made installing a checksum hasher in the object layer unsafe. The checksum +// must always cover logical plaintext, regardless of compression or encryption. +func TestAPIUploadPartServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIUploadPartServerSideChecksum, + []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"}) +} + +func testAPIUploadPartServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + typ := hash.ChecksumCRC32 + data := bytes.Repeat([]byte("multipart-checksum-plaintext-"), 48*1024) + want := mustChecksum(t, typ, data) + + t.Run("upload", func(t *testing.T) { + object := "checksums/upload" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + etag, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, nil) + if got := rec.Header().Get(typ.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want { + t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want) + } + + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + + oi, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err) + } + checksums, _ := oi.decryptChecksums(0, nil) + if got := checksums[typ.String()]; got != want { + t.Fatalf("%s: stored checksum %q, want plaintext checksum %q", instanceType, got, want) + } + }) + + t.Run("copy", func(t *testing.T) { + source := "checksums/source" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + object := "checksums/copy" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials, + bucketName, source, object, uploadID, "", nil) + if got := copyPartChecksum(typ, response); got != want { + t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want) + } + + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want { + t.Fatalf("%s: copied ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want) + } + + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(response.ETag)}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) +} + +func TestAPIUploadPartServerSideChecksumAlgorithms(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumAlgorithms, + endpoints: []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"}, + }) +} + +func testAPIUploadPartServerSideChecksumAlgorithms(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + tests := []struct { + typ hash.ChecksumType + objType string + composite bool + }{ + {hash.ChecksumCRC32, xhttp.AmzChecksumTypeFullObject, false}, + {hash.ChecksumCRC32C, xhttp.AmzChecksumTypeFullObject, false}, + {hash.ChecksumCRC64NVME, xhttp.AmzChecksumTypeFullObject, false}, + {hash.ChecksumCRC32, xhttp.AmzChecksumTypeComposite, true}, + {hash.ChecksumSHA1, xhttp.AmzChecksumTypeComposite, true}, + {hash.ChecksumSHA256, xhttp.AmzChecksumTypeComposite, true}, + } + data := bytes.Repeat([]byte("server-side-part-checksum"), 1024) + + for _, test := range tests { + t.Run(test.typ.String()+"/"+test.objType, func(t *testing.T) { + object := "algorithms/" + test.typ.String() + "/" + test.objType + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + test.typ.String(), test.objType) + etag, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, nil) + if got := rec.Header().Get(test.typ.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + want := mustChecksum(t, test.typ, data) + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(test.typ, listed.Parts[0]) != want { + t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want) + } + + part := CompletePart{PartNumber: 1, ETag: etag} + if test.composite { + part = completePartWithChecksum(test.typ, 1, etag, want) + } + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{part}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + } + + t.Run("multi-part/FULL_OBJECT", func(t *testing.T) { + typ := hash.ChecksumCRC32 + parts, full := multipartChecksumTestData() + object := "algorithms/multi-part-full-object" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + etags := make([]string, len(parts)) + for i, data := range parts { + etag, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, i+1, data, nil) + if got := rec.Header().Get(typ.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + etags[i] = etag + } + + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != len(parts) { + t.Fatalf("%s: ListParts returned %d parts, want %d", instanceType, len(listed.Parts), len(parts)) + } + for i, part := range listed.Parts { + if got, want := partChecksum(typ, part), mustChecksum(t, typ, parts[i]); got != want { + t.Fatalf("%s: part %d checksum %q, want %q", instanceType, i+1, got, want) + } + } + + complete := make([]CompletePart, len(etags)) + for i, etag := range etags { + complete[i] = CompletePart{PartNumber: i + 1, ETag: etag} + } + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, complete, + map[string]string{ + typ.Key(): mustChecksum(t, typ, full), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: multi-part CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("zero-length-part", func(t *testing.T) { + typ := hash.ChecksumCRC32 + object := "algorithms/zero-length" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + etag, _ := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, nil, nil) + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, nil) { + t.Fatalf("%s: zero-length ListParts checksum mismatch: %+v", instanceType, listed.Parts) + } + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: zero-length CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("overwrite-part-checksum", func(t *testing.T) { + typ := hash.ChecksumCRC32 + object := "algorithms/overwrite" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + first := []byte("first part contents") + second := []byte("replacement part contents") + uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, first, nil) + etag, _ := uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, second, nil) + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, second) { + t.Fatalf("%s: overwritten ListParts checksum mismatch: %+v", instanceType, listed.Parts) + } + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: overwritten CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("copy/SHA256/COMPOSITE", func(t *testing.T) { + typ := hash.ChecksumSHA256 + source := "algorithms/copy-source" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + object := "algorithms/copy-SHA256" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeComposite) + start, end := 7, len(data)-9 + response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials, + bucketName, source, object, uploadID, "bytes="+strconv.Itoa(start)+"-"+strconv.Itoa(end-1), nil) + want := mustChecksum(t, typ, data[start:end]) + if got := copyPartChecksum(typ, response); got != want { + t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want) + } + + part := completePartWithChecksum(typ, 1, canonicalizeETag(response.ETag), want) + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{part}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) +} + +func TestAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors, + endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"}, + }) +} + +func testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + data := []byte("client checksum must remain authoritative") + + t.Run("correct-value", func(t *testing.T) { + typ := hash.ChecksumCRC32 + object := "errors/correct-value" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + want := mustChecksum(t, typ, data) + _, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, map[string]string{typ.Key(): want}) + if got := rec.Header().Get(typ.Key()); got != want { + t.Fatalf("%s: client checksum response %q, want %q", instanceType, got, want) + } + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want { + t.Fatalf("%s: client checksum ListParts mismatch: %+v", instanceType, listed.Parts) + } + }) + + t.Run("wrong-algorithm", func(t *testing.T) { + object := "errors/wrong-algorithm" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject) + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, object, uploadID, "1"), + int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, + map[string]string{hash.ChecksumSHA256.Key(): mustChecksum(t, hash.ChecksumSHA256, data)}) + if err != nil { + t.Fatalf("failed to build UploadPart request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" { + t.Fatalf("%s: wrong algorithm returned %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("wrong-value", func(t *testing.T) { + object := "errors/wrong-value" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject) + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, object, uploadID, "1"), + int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, + map[string]string{hash.ChecksumCRC32.Key(): mustChecksum(t, hash.ChecksumCRC32, []byte("wrong"))}) + if err != nil { + t.Fatalf("failed to build UploadPart request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "XAmzContentChecksumMismatch" { + t.Fatalf("%s: wrong value returned %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) +} + +func TestAPIUploadPartServerSideChecksumSSEC(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumSSEC, + endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"}, + }) +} + +func testAPIUploadPartServerSideChecksumSSEC(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + globalIsTLS = true + defer func() { globalIsTLS = false }() + + key := bytes.Repeat([]byte{0x2a}, 32) + keyMD5 := md5.Sum(key) + ssecHeaders := map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]), + } + initHeaders := map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm], + xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey], + xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5], + } + object := "checksums/ssec" + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object), + 0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders) + if err != nil { + t.Fatalf("failed to build NewMultipartUpload request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + var initiated InitiateMultipartUploadResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil { + t.Fatalf("failed to decode NewMultipartUpload response: %v", err) + } + + data := bytes.Repeat([]byte("ssec-checksum-plaintext"), 4096) + etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, initiated.UploadID, 1, data, ssecHeaders) + if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + completeHeaders := map[string]string{ + xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm], + xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey], + xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5], + } + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, completeHeaders) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } +} + +func TestAPIUploadPartServerSideChecksumSSES3(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumSSES3, + endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"}, + }) +} + +func testAPIUploadPartServerSideChecksumSSES3(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=") + if err != nil { + t.Fatal(err) + } + GlobalKMS = KMS + defer func() { GlobalKMS = nil }() + + object := "checksums/sse-s3" + initHeaders := map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES, + } + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object), + 0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders) + if err != nil { + t.Fatalf("failed to build NewMultipartUpload request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + var initiated InitiateMultipartUploadResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil { + t.Fatalf("failed to decode NewMultipartUpload response: %v", err) + } + + data := bytes.Repeat([]byte("sse-s3-checksum-plaintext"), 4096) + etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, initiated.UploadID, 1, data, nil) + if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, map[string]string{ + xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } +} diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index f73b7da4f..cfee44df0 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -597,12 +597,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo onlineDisks := er.getDisks() writeQuorum := fi.WriteQuorum(er.defaultWQuorum()) - if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" { - if r.ContentCRCType().String() != cs { + expectedChecksumType, checksumEnabled := multipartChecksumType(fi.Metadata) + if checksumEnabled { + got := r.contentChecksumType() + if !expectedChecksumType.IsSet() || !got.IsSet() || got.Base() != expectedChecksumType { return pi, InvalidArgument{ Bucket: bucket, Object: fi.Name, - Err: fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()), + Err: fmt.Errorf("checksum missing, want %q, got %q", + fi.Metadata[hash.MinIOMultipartChecksum], got.String()), } } } @@ -725,6 +728,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo } } + partChecksums := r.contentChecksum() + if checksumEnabled && partChecksums[expectedChecksumType.String()] == "" { + err := fmt.Errorf("internal error: checksum missing after reading part, want %q", expectedChecksumType.String()) + bugLogIf(ctx, err) + return pi, toObjectErr(err, bucket, object, uploadID) + } + partInfo := ObjectPartInfo{ Number: partID, ETag: md5hex, @@ -732,7 +742,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo ActualSize: actualSize, ModTime: UTCNow(), Index: index, - Checksums: r.ContentCRC(), + Checksums: partChecksums, } partFI, err := partInfo.MarshalMsg(nil) diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 0247822e7..2364c45b9 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1485,11 +1485,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st // over opts.WantChecksum. if opts.WantServerSideChecksumType.IsSet() { serverSideChecksum := r.RawServerSideChecksumResult() - if serverSideChecksum != nil { - fi.Checksum = serverSideChecksum.AppendTo(nil, nil) - if opts.EncryptFn != nil { - fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) - } + if serverSideChecksum == nil || !serverSideChecksum.Valid() || + serverSideChecksum.Type.Base() != opts.WantServerSideChecksumType.Base() { + err := fmt.Errorf("internal error: server-side checksum missing, invalid, or mismatched after reading object, want %q", opts.WantServerSideChecksumType.String()) + bugLogIf(ctx, err) + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + fi.Checksum = serverSideChecksum.AppendTo(nil, nil) + if opts.EncryptFn != nil { + fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) } } else if fi.Checksum == nil && opts.WantChecksum != nil { // Trailing headers checksums should now be filled. diff --git a/cmd/metrics-realtime.go b/cmd/metrics-realtime.go index 4a1d55732..9eb10990f 100644 --- a/cmd/metrics-realtime.go +++ b/cmd/metrics-realtime.go @@ -191,7 +191,9 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM } } + //nolint:staticcheck // Linux implementations can fail; BSD stubs return a constant nil error. st, err := disk.GetDriveStats(d.Major, d.Minor) + //nolint:staticcheck // Keep the shared cross-platform error handling. if err == nil { dm.IOStats = madmin.DiskIOStats{ ReadIOs: st.ReadIOs, diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 5d791ce45..5aefb39b9 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -1038,9 +1038,10 @@ type SealMD5CurrFn func([]byte) []byte // PutObjReader is a type that wraps sio.EncryptReader and // underlying hash.Reader in a struct type PutObjReader struct { - *hash.Reader // actual data stream - rawReader *hash.Reader // original data stream - sealMD5Fn SealMD5CurrFn + *hash.Reader // actual data stream + rawReader *hash.Reader // original data stream used for ETag calculation + checksumReader *hash.Reader // logical plaintext stream used for S3 checksum calculation + sealMD5Fn SealMD5CurrFn } // Size returns the absolute number of bytes the Reader @@ -1093,15 +1094,51 @@ func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto. // NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's // MD5Current method to construct md5sum when requested downstream. func NewPutObjReader(rawReader *hash.Reader) *PutObjReader { - return &PutObjReader{Reader: rawReader, rawReader: rawReader} + return &PutObjReader{Reader: rawReader, rawReader: rawReader, checksumReader: rawReader} +} + +// setChecksumReader sets the logical plaintext reader used for S3 checksums. +// It can differ from rawReader when the storage stream is compressed. +func (p *PutObjReader) setChecksumReader(r *hash.Reader) { + if r != nil { + p.checksumReader = r + } +} + +// contentChecksumType returns the effective client-provided or server-computed +// checksum type for the logical plaintext stream. +func (p *PutObjReader) contentChecksumType() hash.ChecksumType { + if p.checksumReader == nil { + return hash.ChecksumNone + } + if t := p.checksumReader.ContentCRCType(); t.IsSet() { + return t + } + return p.checksumReader.ServerSideChecksumType +} + +// contentChecksum returns the effective checksum for part metadata. A +// client-provided checksum takes precedence; server computation is only a +// fallback when the client omitted one. +func (p *PutObjReader) contentChecksum() map[string]string { + if p.checksumReader == nil { + return nil + } + if checksum := p.checksumReader.ContentCRC(); checksum != nil { + return checksum + } + if checksum := p.checksumReader.ServerSideChecksumResult; checksum != nil && checksum.Valid() { + return map[string]string{checksum.Type.String(): checksum.Encoded} + } + return nil } // RawServerSideChecksumResult returns the ServerSideChecksumResult from the -// underlying rawReader, since the PutObjReader might be encrypted data and -// thus any checksum from that would be incorrect. +// logical plaintext checksum reader, since the PutObjReader might contain +// compressed or encrypted data and thus any checksum from that would be incorrect. func (p *PutObjReader) RawServerSideChecksumResult() *hash.Checksum { - if p.rawReader != nil { - return p.rawReader.ServerSideChecksumResult + if p.checksumReader != nil { + return p.checksumReader.ServerSideChecksumResult } return nil } diff --git a/cmd/object-copy-checksum_test.go b/cmd/object-copy-checksum_test.go new file mode 100644 index 000000000..ad373f171 --- /dev/null +++ b/cmd/object-copy-checksum_test.go @@ -0,0 +1,556 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +func setCopyChecksumCompression(allowEncrypted bool) func() { + globalCompressConfigMu.Lock() + previous := globalCompressConfig + globalCompressConfig.Enabled = true + globalCompressConfig.Extensions = []string{".txt"} + globalCompressConfig.MimeTypes = nil + globalCompressConfig.AllowEncrypted = allowEncrypted + globalCompressConfigMu.Unlock() + + return func() { + globalCompressConfigMu.Lock() + globalCompressConfig = previous + globalCompressConfigMu.Unlock() + } +} + +func copyChecksumRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials, + bucket, source, destination string, headers map[string]string, +) *httptest.ResponseRecorder { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucket, destination), + 0, nil, credentials.AccessKey, credentials.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build CopyObject request: %v", err) + } + req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source)) + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + return rec +} + +func putCopyChecksumSource(t *testing.T, apiRouter http.Handler, credentials auth.Credentials, + bucket, object string, data []byte, headers map[string]string, +) { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucket, object), + int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build PutObject request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("PutObject(%s) failed: %d %s", object, rec.Code, rec.Body.String()) + } +} + +func readCopyChecksumObject(t *testing.T, obj ObjectLayer, bucket, object string, opts ObjectOptions) []byte { + t.Helper() + gr, err := obj.GetObjectNInfo(t.Context(), bucket, object, nil, nil, opts) + if err != nil { + t.Fatalf("GetObjectNInfo(%s) failed: %v", object, err) + } + defer gr.Close() + data, err := io.ReadAll(gr) + if err != nil { + t.Fatalf("reading %s failed: %v", object, err) + } + return data +} + +func assertCopyChecksum(t *testing.T, obj ObjectLayer, bucket, object string, typ hash.ChecksumType, + data []byte, compressed bool, decryptHeaders http.Header, +) ObjectInfo { + t.Helper() + oi, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}) + if err != nil { + t.Fatalf("GetObjectInfo(%s) failed: %v", object, err) + } + if oi.IsCompressed() != compressed { + t.Fatalf("%s compressed=%v, want %v", object, oi.IsCompressed(), compressed) + } + checksums, _ := oi.decryptChecksums(0, decryptHeaders) + if got, want := checksums[typ.String()], mustChecksum(t, typ, data); got != want { + t.Fatalf("%s stored %s checksum %q, want logical object checksum %q (all: %v)", + object, typ.String(), got, want, checksums) + } + if got := checksums[xhttp.AmzChecksumType]; got != xhttp.AmzChecksumTypeFullObject { + t.Fatalf("%s checksum type %q, want %q", object, got, xhttp.AmzChecksumTypeFullObject) + } + return oi +} + +func assertCopyChecksumResponse(t *testing.T, rec *httptest.ResponseRecorder, typ hash.ChecksumType, data []byte) { + t.Helper() + var response CopyObjectResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("unable to decode CopyObjectResult: %v", err) + } + var got string + switch typ.Base() { + case hash.ChecksumCRC32: + got = response.ChecksumCRC32 + case hash.ChecksumCRC32C: + got = response.ChecksumCRC32C + case hash.ChecksumSHA1: + got = response.ChecksumSHA1 + case hash.ChecksumSHA256: + got = response.ChecksumSHA256 + case hash.ChecksumCRC64NVME: + got = response.ChecksumCRC64NVME + } + if want := mustChecksum(t, typ, data); got != want { + t.Fatalf("CopyObjectResult %s checksum %q, want %q: %s", typ.String(), got, want, rec.Body.String()) + } + if response.ChecksumType != xhttp.AmzChecksumTypeFullObject { + t.Fatalf("CopyObjectResult checksum type %q, want %q", response.ChecksumType, xhttp.AmzChecksumTypeFullObject) + } +} + +// TestAPICopyObjectServerSideChecksum verifies that server-computed checksums +// cover the logical object, never the compressed storage stream. +func TestAPICopyObjectServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + for _, versioned := range []bool{false, true} { + name := "unversioned" + if versioned { + name = "versioned" + } + t.Run(name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksum, + endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"}, + makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned}, + }) + }) + } +} + +func testAPICopyObjectServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("copy-object-checksum-plaintext-"), 64*1024) + source := "copy-checksum/source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + + compressedReader, _ := newS2CompressReader(bytes.NewReader(data), int64(len(data)), false) + compressed, err := io.ReadAll(compressedReader) + if closeErr := compressedReader.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatalf("%s: independently compressing test data failed: %v", instanceType, err) + } + + cases := []struct { + name string + typ hash.ChecksumType + explicit bool + extension string + compressed bool + }{ + {name: "compressed/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/CRC32C", typ: hash.ChecksumCRC32C, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/SHA1", typ: hash.ChecksumSHA1, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/SHA256", typ: hash.ChecksumSHA256, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/CRC64NVME", typ: hash.ChecksumCRC64NVME, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/default", typ: hash.ChecksumCRC64NVME, extension: ".txt", compressed: true}, + {name: "plain/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".bin"}, + {name: "plain/default", typ: hash.ChecksumCRC64NVME, extension: ".bin"}, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + headers := map[string]string(nil) + if tc.explicit { + headers = map[string]string{xhttp.AmzChecksumAlgo: tc.typ.String()} + } + destination := "copy-checksum/destination-" + tc.typ.String() + "-" + string(rune('a'+i)) + tc.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, tc.typ, data) + + info := assertCopyChecksum(t, obj, bucketName, destination, tc.typ, data, tc.compressed, nil) + md5sum := md5.Sum(data) + if got, want := info.ETag, hex.EncodeToString(md5sum[:]); got != want { + t.Fatalf("%s: ETag %q, want logical object MD5 %q", instanceType, got, want) + } + if tc.compressed { + logical := mustChecksum(t, tc.typ, data) + if transformed := mustChecksum(t, tc.typ, compressed); logical == transformed { + t.Fatalf("%s: test payload does not distinguish logical and compressed checksum domains", instanceType) + } + } + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: round-trip body differs for %s", instanceType, tc.name) + } + + if tc.name == "compressed/CRC32" { + for _, method := range []string{http.MethodHead, http.MethodGet} { + url := getHeadObjectURL("", bucketName, destination) + if method == http.MethodGet { + url = getGetObjectURL("", bucketName, destination) + } + req, err := newTestSignedRequestV4(method, url, 0, nil, + credentials.AccessKey, credentials.SecretKey, + map[string]string{xhttp.AmzChecksumMode: "ENABLED"}) + if err != nil { + t.Fatalf("failed to build %s request: %v", method, err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK { + t.Fatalf("%s returned %d: %s", method, response.Code, response.Body.String()) + } + if got, want := response.Header().Get(tc.typ.Key()), mustChecksum(t, tc.typ, data); got != want { + t.Fatalf("%s returned checksum %q, want %q", method, got, want) + } + if method == http.MethodGet && !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("GET response body differs") + } + } + } + }) + } +} + +func TestAPICopyObjectServerSideChecksumEncryption(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksumEncryption, + endpoints: []string{"CopyObject", "PutObject", "GetObject"}, + }) +} + +func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("encrypted-copy-checksum-plaintext-"), 48*1024) + source := "copy-checksum/encrypted-source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + + t.Run("SSE-S3", func(t *testing.T) { + secretKey, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=") + if err != nil { + t.Fatal(err) + } + previousKMS := GlobalKMS + GlobalKMS = secretKey + defer func() { GlobalKMS = previousKMS }() + + for _, variant := range []struct { + name string + extension string + compressed bool + }{ + {name: "encrypted-only", extension: ".bin"}, + {name: "compressed-encrypted", extension: ".txt", compressed: true}, + } { + t.Run(variant.name, func(t *testing.T) { + destination := "copy-checksum/sse-s3-" + variant.name + variant.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES, + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-S3 CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: SSE-S3 round-trip body differs", instanceType) + } + }) + } + + encryptedSource := "copy-checksum/sse-s3-source.bin" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, encryptedSource, data, + map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}) + destination := "copy-checksum/sse-s3-source-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, encryptedSource, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-S3 source CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: SSE-S3 source round-trip body differs", instanceType) + } + }) + + t.Run("SSE-C", func(t *testing.T) { + previousTLS := globalIsTLS + globalIsTLS = true + defer func() { globalIsTLS = previousTLS }() + + key := bytes.Repeat([]byte{0x2a}, 32) + keyMD5 := md5.Sum(key) + headers := map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]), + } + decryptHeaders := http.Header{} + for key, value := range headers { + decryptHeaders.Set(key, value) + } + + getHeaders := make(map[string]string, len(headers)) + for key, value := range headers { + if key != xhttp.AmzChecksumAlgo { + getHeaders[key] = value + } + } + for _, variant := range []struct { + name string + extension string + compressed bool + }{ + {name: "encrypted-only", extension: ".bin"}, + {name: "compressed-encrypted", extension: ".txt", compressed: true}, + } { + t.Run(variant.name, func(t *testing.T) { + destination := "copy-checksum/sse-c-" + variant.name + variant.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-C CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, decryptHeaders) + + req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, destination), + 0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders) + if err != nil { + t.Fatalf("failed to build SSE-C GetObject request: %v", err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("%s: SSE-C GetObject returned %d with %d bytes, want 200 with %d bytes", + instanceType, response.Code, response.Body.Len(), len(data)) + } + }) + } + }) +} + +func TestAPICopyObjectServerSideChecksumSourceVariants(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksumSourceVariants, + endpoints: []string{ + "NewMultipart", "PutObjectPart", "CompleteMultipart", "ListObjectParts", + "CopyObject", "PutObject", "HeadObject", "GetObject", + }, + }) +} + +func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("source-variant-plaintext-"), 64*1024) + + t.Run("compressed-source", func(t *testing.T) { + source := "copy-checksum/compressed-source.txt" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, nil) + if info, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{}); err != nil || !info.IsCompressed() { + t.Fatalf("%s: compressed source precondition failed: compressed=%v err=%v", instanceType, info.IsCompressed(), err) + } + destination := "copy-checksum/compressed-source-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, source, map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumSHA256.String(), + xhttp.AmzMetadataDirective: "REPLACE", + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: in-place CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, hash.ChecksumSHA256, data) + assertCopyChecksum(t, obj, bucketName, source, hash.ChecksumSHA256, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, source, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: in-place CopyObject body differs", instanceType) + } + }) + + t.Run("full-checksum-source", func(t *testing.T) { + source := "copy-checksum/full-checksum-source.bin" + want := mustChecksum(t, hash.ChecksumCRC32, data) + putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, + map[string]string{xhttp.AmzChecksumCRC32: want}) + destination := "copy-checksum/full-checksum-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + }) + + t.Run("multipart-composite-source", func(t *testing.T) { + typ := hash.ChecksumCRC32 + parts, full := multipartChecksumTestData() + source := "copy-checksum/multipart-source.bin" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source, + typ.String(), xhttp.AmzChecksumTypeComposite) + etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, source, uploadID, typ, parts) + partChecksums := make([]string, len(parts)) + for i, part := range parts { + partChecksums[i] = mustChecksum(t, typ, part) + } + rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source, uploadID, + etags, partChecksums, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + sourceInfo, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{}) + if err != nil { + t.Fatalf("%s: source GetObjectInfo failed: %v", instanceType, err) + } + if _, multipart := sourceInfo.decryptChecksums(0, nil); !multipart { + t.Fatalf("%s: source checksum is not multipart composite", instanceType) + } + + destination := "copy-checksum/multipart-copy.txt" + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, typ, full) + assertCopyChecksum(t, obj, bucketName, destination, typ, full, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, full) { + t.Fatalf("%s: multipart source round-trip body differs", instanceType) + } + }) + + for _, boundary := range []struct { + name string + data []byte + compressed bool + }{ + {name: "at-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize)}, + {name: "over-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize+1), compressed: true}, + {name: "indexed", data: bytes.Repeat([]byte{'a'}, compMinIndexSize+1), compressed: true}, + {name: "empty", data: nil}, + } { + t.Run(boundary.name, func(t *testing.T) { + source := "copy-checksum/" + boundary.name + "-source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(boundary.data), int64(len(boundary.data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + destination := "copy-checksum/" + boundary.name + "-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, boundary.data) + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, boundary.data, boundary.compressed, nil) + }) + } +} + +func TestPutObjectRejectsMissingServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testPutObjectRejectsMissingServerSideChecksum, + endpoints: []string{"PutObject"}, + }) +} + +func testPutObjectRejectsMissingServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, + _ http.Handler, _ auth.Credentials, t *testing.T, +) { + data := []byte("the object layer must not silently omit a requested checksum") + for _, test := range []struct { + name string + hasherType hash.ChecksumType + }{ + {name: "missing"}, + {name: "mismatched", hasherType: hash.ChecksumCRC32C}, + } { + t.Run(test.name, func(t *testing.T) { + object := "copy-checksum/" + test.name + "-server-side-checksum" + reader := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "") + if test.hasherType.IsSet() { + reader.AddServerSideChecksumHasher(test.hasherType) + } + _, err := obj.PutObject(t.Context(), bucketName, object, reader, + ObjectOptions{WantServerSideChecksumType: hash.ChecksumCRC32}) + if err == nil || !strings.Contains(err.Error(), "server-side checksum") { + t.Fatalf("%s: PutObject error %v, want server-side checksum invariant error", instanceType, err) + } + if _, err = obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}); !isErrObjectNotFound(err) { + t.Fatalf("%s: failed PutObject left an object behind: %v", instanceType, err) + } + }) + } +} diff --git a/cmd/object-copy-metadata_test.go b/cmd/object-copy-metadata_test.go new file mode 100644 index 000000000..43470436d --- /dev/null +++ b/cmd/object-copy-metadata_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "net/http" + "net/http/httptest" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" +) + +func TestAPICopyObjectMetadataOnlyCompression(t *testing.T) { + defer DetectTestLeak(t)() + for _, versioned := range []bool{false, true} { + name := "unversioned" + if versioned { + name = "versioned" + } + t.Run(name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectMetadataOnlyCompression, + endpoints: []string{"CopyObject", "PutObject", "GetObject"}, + makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned}, + }) + }) + } +} + +func testAPICopyObjectMetadataOnlyCompression(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + data := bytes.Repeat([]byte("metadata-only-copy-plaintext-"), 64*1024) + want := mustChecksum(t, hash.ChecksumCRC32, data) + object := "copy-metadata/existing-checksum.txt" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data, + map[string]string{xhttp.AmzChecksumCRC32: want}) + before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil || before.IsCompressed() { + t.Fatalf("%s: invalid metadata-copy precondition: compressed=%v size=%d err=%v", + instanceType, before.IsCompressed(), before.Size, err) + } + + restoreCompression := setCopyChecksumCompression(true) + compressionRestored := false + defer func() { + if !compressionRestored { + restoreCompression() + } + }() + + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, + map[string]string{xhttp.AmzMetadataDirective: "REPLACE"}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, false, nil) + if got := readCopyChecksumObject(t, obj, bucketName, object, ObjectOptions{}); !bytes.Equal(got, data) { + prefix := got + if len(prefix) > 100 { + prefix = prefix[:100] + } + t.Fatalf("%s: metadata-only CopyObject body differs: got %d bytes, want %d, prefix %q", + instanceType, len(got), len(data), prefix) + } + afterMetadataCopy, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if before.VersionID != "" && afterMetadataCopy.VersionID == before.VersionID { + t.Fatalf("%s: versioned metadata-only copy did not create a new version", instanceType) + } + + destination := "copy-metadata/rewritten.txt" + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, object, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: data-rewriting CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + + compressedObject := "copy-metadata/preserve-compressed.txt" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, compressedObject, data, + map[string]string{xhttp.AmzChecksumCRC32: want}) + assertCopyChecksum(t, obj, bucketName, compressedObject, hash.ChecksumCRC32, data, true, nil) + + restoreCompression() + compressionRestored = true + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, compressedObject, compressedObject, + map[string]string{xhttp.AmzMetadataDirective: "REPLACE"}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: compressed metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, compressedObject, hash.ChecksumCRC32, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, compressedObject, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: compressed metadata-only CopyObject body differs", instanceType) + } +} + +func TestAPICopyObjectSSECKeyRotationKeepsCompressionState(t *testing.T) { + defer DetectTestLeak(t)() + for _, versioned := range []bool{false, true} { + name := "unversioned" + if versioned { + name = "versioned" + } + t.Run(name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectSSECKeyRotationKeepsCompressionState, + endpoints: []string{"CopyObject", "PutObject", "GetObject"}, + makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned}, + }) + }) + } +} + +func testAPICopyObjectSSECKeyRotationKeepsCompressionState(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + previousTLS := globalIsTLS + globalIsTLS = true + defer func() { globalIsTLS = previousTLS }() + + data := bytes.Repeat([]byte("key-rotation-plaintext-"), 64*1024) + object := "copy-metadata/key-rotation.txt" + oldKey := bytes.Repeat([]byte{0x11}, 32) + oldMD5 := md5.Sum(oldKey) + newKey := bytes.Repeat([]byte{0x22}, 32) + newMD5 := md5.Sum(newKey) + + putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data, map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]), + }) + before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil || before.IsCompressed() { + t.Fatalf("%s: invalid key-rotation precondition: compressed=%v err=%v", instanceType, before.IsCompressed(), err) + } + + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]), + xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(oldKey), + xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]), + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: key rotation failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + after, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if after.IsCompressed() { + t.Fatalf("%s: metadata-only key rotation stamped compression metadata", instanceType) + } + if before.VersionID != "" && after.VersionID == before.VersionID { + t.Fatalf("%s: versioned key rotation did not create a new version", instanceType) + } + + getHeaders := map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]), + } + req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object), + 0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders) + if err != nil { + t.Fatalf("failed to build GetObject request: %v", err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("%s: post-rotation GetObject returned %d with %d bytes, want 200 with %d bytes: %s", + instanceType, response.Code, response.Body.Len(), len(data), response.Body.String()) + } +} diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 296817ed5..d5e868925 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1122,6 +1122,12 @@ func getRemoteInstanceTransport() http.RoundTripper { return nil } +// federatedInternalAppName is the minio-go application token that +// getRemoteInstanceClient attaches to every legacy federation proxy request. It +// is declared next to its only producer so that the literal keeps its historical +// file attribution in the rebrand compatibility baseline. +const federatedInternalAppName = "minio-federated" + // Returns a minio-go Client configured to access remote host described by destDNSRecord // Applicable only in a federated deployment var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) { @@ -1136,7 +1142,7 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, if err != nil { return nil, err } - core.SetAppInfo("minio-federated", ReleaseTag) + core.SetAppInfo(federatedInternalAppName, ReleaseTag) return core, nil } @@ -1357,6 +1363,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } // no changes in storage-class expected so its a metadataonly operation. var reader io.Reader = gr + sourceCompressMetadata := make(map[string]string, 2) + for _, key := range []string{ + ReservedMetadataPrefix + "compression", + ReservedMetadataPrefix + "actual-size", + } { + if value, ok := srcInfo.UserDefined[key]; ok { + sourceCompressMetadata[key] = value + } + } // Set the actual size to the compressed/decrypted size if encrypted. actualSize, err := srcInfo.GetActualSize() @@ -1386,15 +1401,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10) reader = etag.NewReader(ctx, reader, nil, nil) - wantEncryption := crypto.Requested(r.Header) - s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption) - dstOpts.IndexCB = cb - defer s2c.Close() - reader = etag.Wrap(s2c, reader) - length = -1 } else { - delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") - delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") reader = gr } @@ -1541,6 +1548,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } } + if isDstCompressed { + checksumReader := srcInfo.Reader + wantEncryption := crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(checksumReader, actualSize, wantEncryption) + dstOpts.IndexCB = cb + defer s2c.Close() + reader = etag.Wrap(s2c, checksumReader) + srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // The storage reader consumes compressed data; checksums remain bound to plaintext. + pReader = NewPutObjReader(srcInfo.Reader) + pReader.setChecksumReader(checksumReader) + } + if isTargetEncrypted { var encReader io.Reader kind, _ := crypto.IsRequested(r.Header) @@ -1676,8 +1700,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) } - // Store the preserved compression metadata. - maps.Copy(srcInfo.UserDefined, compressMetadata) + // Compression metadata must describe data that is actually rewritten. + if !srcInfo.metadataOnly || srcInfo.Legacy || dstOpts.WantServerSideChecksumType.IsSet() { + if isDstCompressed { + maps.Copy(srcInfo.UserDefined, compressMetadata) + } else { + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") + } + } else { + maps.Copy(srcInfo.UserDefined, sourceCompressMetadata) + } // We need to preserve the encryption headers set in EncryptRequest, // so we do not want to override them, copy them instead. @@ -1767,9 +1800,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re copyObjectFn := objectAPI.CopyObject + copySrcOpts := srcOpts + if srcInfo.metadataOnly && dstOpts.Versioned && copySrcOpts.VersionID == "" { + copySrcOpts.VersionID = srcInfo.VersionID + } + // Copy source object to destination, if source and destination // object is same then only metadata is updated. - objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -1778,7 +1816,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re origETag := objInfo.ETag objInfo.ETag = getDecryptedETag(r.Header, objInfo, false) - response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime) + response := generateCopyObjectResponse(objInfo, r.Header) encodedSuccessResponse := encodeResponse(response) if dsc := mustReplicate(ctx, dstBucket, dstObject, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, dstOpts)); dsc.ReplicateAny() { diff --git a/cmd/object-multipart-federation-checksum_test.go b/cmd/object-multipart-federation-checksum_test.go new file mode 100644 index 000000000..f7b8f5017 --- /dev/null +++ b/cmd/object-multipart-federation-checksum_test.go @@ -0,0 +1,367 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// Copyright (c) 2025-2026 PGSTY +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "encoding/xml" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + miniogo "github.com/minio/minio-go/v7" + miniocredentials "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" +) + +const federatedTestUserAgent = "MinIO (linux; amd64) minio-go/v7.0.99 minio-federated/RELEASE.TEST" + +func TestAPIFederatedUploadPartChecksumResponse(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedUploadPartChecksumResponse, + endpoints: []string{"PutObjectPart", "NewMultipart"}, + }) +} + +func testAPIFederatedUploadPartChecksumResponse(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + algorithms := []struct { + name string + typ hash.ChecksumType + checksumType string + }{ + {name: "crc32-full-object", typ: hash.ChecksumCRC32, checksumType: xhttp.AmzChecksumTypeFullObject}, + {name: "sha256-composite", typ: hash.ChecksumSHA256, checksumType: xhttp.AmzChecksumTypeComposite}, + } + userAgents := []struct { + name string + ua string + want bool + }{ + {name: "absent"}, + {name: "ordinary-sdk", ua: "aws-sdk-go/1.55.5"}, + {name: "federation", ua: federatedTestUserAgent, want: true}, + {name: "lookalike-prefix", ua: "evil-minio-federated/RELEASE.TEST"}, + {name: "lookalike-suffix", ua: "minio-federated-extra/RELEASE.TEST"}, + {name: "missing-version", ua: "minio-federated"}, + {name: "empty-version", ua: "minio-federated/"}, + } + data := []byte("federated upload part checksum response") + + for _, algorithm := range algorithms { + for _, userAgent := range userAgents { + t.Run(algorithm.name+"/"+userAgent.name, func(t *testing.T) { + object := "federation/response/" + algorithm.name + "/" + userAgent.name + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + algorithm.typ.String(), algorithm.checksumType) + headers := map[string]string{} + if userAgent.ua != "" { + headers["User-Agent"] = userAgent.ua + } + _, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, headers) + + got := rec.Header().Get(algorithm.typ.Key()) + if userAgent.want { + if want := mustChecksum(t, algorithm.typ, data); got != want { + t.Fatalf("%s: checksum %q, want %q", instanceType, got, want) + } + } else if got != "" { + t.Fatalf("%s: ordinary UploadPart exposed server checksum %q", instanceType, got) + } + if got := rec.Header().Get(xhttp.AmzChecksumType); got != "" { + t.Fatalf("%s: UploadPart returned checksum type %q", instanceType, got) + } + }) + } + } +} + +func TestAPIFederatedUploadPartChecksumMinIOGoWire(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedUploadPartChecksumMinIOGoWire, + endpoints: []string{"PutObjectPart", "NewMultipart"}, + }) +} + +func testAPIFederatedUploadPartChecksumMinIOGoWire(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + server := httptest.NewServer(apiRouter) + defer server.Close() + + core, err := miniogo.NewCore(server.Listener.Addr().String(), &miniogo.Options{ + Creds: miniocredentials.NewStaticV4(credentials.AccessKey, credentials.SecretKey, ""), + Secure: false, + Region: globalMinioDefaultRegion, + BucketLookup: miniogo.BucketLookupPath, + }) + if err != nil { + t.Fatalf("%s: create minio-go Core: %v", instanceType, err) + } + core.SetAppInfo("minio-federated", ReleaseTag) + + object := "federation/minio-go-wire" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject) + data := []byte("minio-go must parse the remote computed checksum") + part, err := core.PutObjectPart(t.Context(), bucketName, object, uploadID, 1, + bytes.NewReader(data), int64(len(data)), miniogo.PutObjectPartOptions{}) + if err != nil { + t.Fatalf("%s: minio-go PutObjectPart: %v", instanceType, err) + } + if want := mustChecksum(t, hash.ChecksumCRC32, data); part.ChecksumCRC32 != want { + t.Fatalf("%s: minio-go checksum %q, want %q", instanceType, part.ChecksumCRC32, want) + } + if part.ETag == "" { + t.Fatalf("%s: minio-go returned an empty ETag", instanceType) + } +} + +func TestAPIFederatedUploadPartChecksumConcurrentOverwrite(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedUploadPartChecksumConcurrentOverwrite, + endpoints: []string{"PutObjectPart", "NewMultipart"}, + }) +} + +func testAPIFederatedUploadPartChecksumConcurrentOverwrite(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + object := "federation/concurrent-overwrite" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumSHA256.String(), xhttp.AmzChecksumTypeComposite) + data := [][]byte{ + bytes.Repeat([]byte("first-writer-"), 4096), + bytes.Repeat([]byte("second-writer-"), 4096), + } + reqs := make([]*http.Request, len(data)) + recorders := make([]*httptest.ResponseRecorder, len(data)) + for i := range data { + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, object, uploadID, "1"), + int64(len(data[i])), bytes.NewReader(data[i]), credentials.AccessKey, credentials.SecretKey, + map[string]string{"User-Agent": federatedTestUserAgent}) + if err != nil { + t.Fatalf("%s: build concurrent request %d: %v", instanceType, i, err) + } + reqs[i] = req + recorders[i] = httptest.NewRecorder() + } + + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range reqs { + wg.Add(1) + go func() { + defer wg.Done() + <-start + apiRouter.ServeHTTP(recorders[i], reqs[i]) + }() + } + close(start) + wg.Wait() + + for i, rec := range recorders { + if rec.Code != http.StatusOK { + t.Fatalf("%s: concurrent request %d failed: %d %s", instanceType, i, rec.Code, rec.Body.String()) + } + got := rec.Header().Get(hash.ChecksumSHA256.Key()) + if want := mustChecksum(t, hash.ChecksumSHA256, data[i]); got != want { + t.Fatalf("%s: concurrent request %d checksum %q, want %q", instanceType, i, got, want) + } + // The ETag and the checksum must describe the same write, so a losing + // writer can never publish the winner's checksum next to its own ETag. + etags := rec.Header()[xhttp.ETag] + if len(etags) != 1 { + t.Fatalf("%s: concurrent request %d returned %d ETags", instanceType, i, len(etags)) + } + md5sum := md5.Sum(data[i]) + if want := hex.EncodeToString(md5sum[:]); canonicalizeETag(etags[0]) != want { + t.Fatalf("%s: concurrent request %d ETag %q, want %q", instanceType, i, etags[0], want) + } + } +} + +// federationTestDNS is a minimal dns.Store so a single test process can play +// both federation roles. +type federationTestDNS struct { + records map[string][]dns.SrvRecord +} + +func (f federationTestDNS) Put(string) error { return nil } + +func (f federationTestDNS) Get(bucket string) ([]dns.SrvRecord, error) { + records, ok := f.records[bucket] + if !ok { + return nil, dns.ErrNoEntriesFound + } + return records, nil +} + +func (f federationTestDNS) Delete(string) error { return nil } +func (f federationTestDNS) List() (map[string][]dns.SrvRecord, error) { return f.records, nil } +func (f federationTestDNS) DeleteRecord(dns.SrvRecord) error { return nil } +func (f federationTestDNS) Close() error { return nil } +func (f federationTestDNS) String() string { return "federation-test-dns" } + +// remoteBucketObjectLayer reports one existing bucket as missing so that +// isRemoteCopyRequired takes the legacy federation branch while the same +// process can still serve that bucket as the remote deployment. +type remoteBucketObjectLayer struct { + ObjectLayer + remoteBucket string +} + +func (l remoteBucketObjectLayer) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) { + if bucket == l.remoteBucket { + return BucketInfo{}, toObjectErr(errVolumeNotFound, bucket) + } + return l.ObjectLayer.GetBucketInfo(ctx, bucket, opts) +} + +// TestAPIFederatedCopyObjectPartChecksum drives the legacy etcd federation +// branch of CopyObjectPartHandler end to end: the proxy forwards the copied +// bytes through the real getRemoteInstanceClient and minio-go, a second HTTP +// endpoint serves the real PutObjectPartHandler, and CopyPartResult must carry +// the checksum computed by that exact remote write. +func TestAPIFederatedCopyObjectPartChecksum(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedCopyObjectPartChecksum, + endpoints: []string{ + "CopyObjectPart", "NewMultipart", "PutObjectPart", + "ListObjectParts", "CompleteMultipart", "PutObject", + }, + }) +} + +func testAPIFederatedCopyObjectPartChecksum(objectAPI ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + algorithms := []struct { + name string + typ hash.ChecksumType + checksumType string + }{ + {name: "crc32-full-object", typ: hash.ChecksumCRC32, checksumType: xhttp.AmzChecksumTypeFullObject}, + {name: "sha256-composite", typ: hash.ChecksumSHA256, checksumType: xhttp.AmzChecksumTypeComposite}, + } + + data := bytes.Repeat([]byte("federated-upload-part-copy-"), 1024) + srcObject := "federation/copy-source.bin" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data, nil) + + // The destination bucket really exists so the remote endpoint can serve it; + // only the proxy's own bucket lookup is told that it lives elsewhere. + remoteBucket := getRandomBucketName() + if err := objectAPI.MakeBucket(t.Context(), remoteBucket, MakeBucketOptions{}); err != nil { + t.Fatalf("%s: unable to create the remote bucket: %v", instanceType, err) + } + + remote := httptest.NewServer(apiRouter) + defer remote.Close() + host, port, _ := strings.Cut(remote.Listener.Addr().String(), ":") + + globalObjLayerMutex.Lock() + previousLayer := globalObjectAPI + globalObjectAPI = remoteBucketObjectLayer{ObjectLayer: previousLayer, remoteBucket: remoteBucket} + globalObjLayerMutex.Unlock() + previousDNS, previousFederation, previousIPs := globalDNSConfig, globalBucketFederation, globalDomainIPs + globalDNSConfig = federationTestDNS{records: map[string][]dns.SrvRecord{ + bucketName: {{Host: host, Port: json.Number(port)}}, + remoteBucket: {{Host: host, Port: json.Number(port)}}, + }} + // Every DNS record resolves to this process, so the bucket forwarding + // middleware always serves locally and only the handler proxies. + globalDomainIPs = set.CreateStringSet(remote.Listener.Addr().String()) + globalBucketFederation = true + defer func() { + globalObjLayerMutex.Lock() + globalObjectAPI = previousLayer + globalObjLayerMutex.Unlock() + globalDNSConfig, globalBucketFederation, globalDomainIPs = previousDNS, previousFederation, previousIPs + }() + + for _, algorithm := range algorithms { + t.Run(algorithm.name, func(t *testing.T) { + object := "federation/copy-destination-" + algorithm.name + ".bin" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, remoteBucket, object, + algorithm.typ.String(), algorithm.checksumType) + + req, err := newTestSignedRequestV4(http.MethodPut, + getCopyObjectPartURL("", remoteBucket, object, uploadID, "1"), + 0, nil, credentials.AccessKey, credentials.SecretKey, + map[string]string{xhttp.AmzCopySource: SlashSeparator + pathJoin(bucketName, srcObject)}) + if err != nil { + t.Fatalf("%s: unable to build UploadPartCopy request: %v", instanceType, err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: federated UploadPartCopy failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + + var response CopyObjectPartResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("%s: unable to decode CopyPartResult: %v", instanceType, err) + } + want := mustChecksum(t, algorithm.typ, data) + if got := copyPartChecksum(algorithm.typ, response); got != want { + t.Fatalf("%s: CopyPartResult %s is %q, want %q: %s", + instanceType, algorithm.typ.String(), got, want, rec.Body.String()) + } + + // The persisted part must carry the same value, and the client must be + // able to complete the upload with what CopyPartResult returned. + parts := listPartsHTTP(t, apiRouter, credentials, remoteBucket, object, uploadID, nil) + if len(parts.Parts) != 1 { + t.Fatalf("%s: ListParts returned %d parts, want 1", instanceType, len(parts.Parts)) + } + if got := partChecksum(algorithm.typ, parts.Parts[0]); got != want { + t.Fatalf("%s: persisted part %s is %q, want %q", instanceType, algorithm.typ.String(), got, want) + } + etag := canonicalizeETag(response.ETag) + completed := completePartsHTTP(t, apiRouter, credentials, remoteBucket, object, uploadID, + []CompletePart{completePartWithChecksum(algorithm.typ, 1, etag, want)}, nil) + if completed.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload rejected the federated part: %d %s", + instanceType, completed.Code, completed.Body.String()) + } + }) + } +} diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index fcc6d5b64..2f6a68a94 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -55,6 +55,88 @@ import ( // Multipart objectAPIHandlers +// isFederatedInternalRequest reports whether User-Agent carries the minio-go +// application token attached by getRemoteInstanceClient. +// +// This is only a response-shape hint. User-Agent is not authenticated and must +// never gate authorization, object visibility, or request validation. It is +// safe here because the only effect is returning the checksum of the body the +// caller was already authorized to upload. +func isFederatedInternalRequest(userAgent string) bool { + for _, product := range strings.Fields(userAgent) { + name, version, ok := strings.Cut(product, "/") + if ok && name == federatedInternalAppName && version != "" { + return true + } + } + return false +} + +// partChecksumMap returns the non-empty part checksums in the form expected by +// hash.AddChecksumHeader. x-amz-checksum-type is deliberately excluded because +// UploadPart does not return it and minio-go cannot carry it in ObjectPart. +func partChecksumMap(partInfo PartInfo) map[string]string { + checksums := make(map[string]string, 1) + if partInfo.ChecksumCRC32 != "" { + checksums[hash.ChecksumCRC32.String()] = partInfo.ChecksumCRC32 + } + if partInfo.ChecksumCRC32C != "" { + checksums[hash.ChecksumCRC32C.String()] = partInfo.ChecksumCRC32C + } + if partInfo.ChecksumSHA1 != "" { + checksums[hash.ChecksumSHA1.String()] = partInfo.ChecksumSHA1 + } + if partInfo.ChecksumSHA256 != "" { + checksums[hash.ChecksumSHA256.String()] = partInfo.ChecksumSHA256 + } + if partInfo.ChecksumCRC64NVME != "" { + checksums[hash.ChecksumCRC64NVME.String()] = partInfo.ChecksumCRC64NVME + } + return checksums +} + +// multipartChecksumType returns the base checksum type recorded when a +// multipart upload was created. The boolean reports whether an algorithm was +// recorded at all. +func multipartChecksumType(metadata map[string]string) (hash.ChecksumType, bool) { + algorithm := metadata[hash.MinIOMultipartChecksum] + if algorithm == "" { + return hash.ChecksumNone, false + } + t := hash.NewChecksumType(algorithm, metadata[hash.MinIOMultipartChecksumType]) + if !t.IsSet() { + return t, true + } + return t.Base(), true +} + +// prepareMultipartChecksumReader validates a supplied part checksum algorithm, +// or installs a server-side hasher when the client omitted the optional +// checksum. It must run before compression or encryption can consume reader. +func prepareMultipartChecksumReader(reader *hash.Reader, metadata map[string]string, bucket, object string) error { + want, ok := multipartChecksumType(metadata) + if !ok { + return nil + } + + got := reader.ContentCRCType() + if !got.IsSet() && reader.ServerSideChecksumType.IsSet() { + got = reader.ServerSideChecksumType + } + if !want.IsSet() || (got.IsSet() && got.Base() != want) { + return InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("checksum missing, want %q, got %q", + metadata[hash.MinIOMultipartChecksum], got.String()), + } + } + if !got.IsSet() { + reader.AddServerSideChecksumHasher(want) + } + return nil +} + // NewMultipartUploadHandler - New multipart upload. // Notice: The S3 client can send secret keys in headers for encryption related jobs, // the handler should ensure to remove these keys before sending them to the object layer. @@ -465,7 +547,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt return } - response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified) + response := generateCopyObjectPartResponse(PartInfo{ + ETag: partInfo.ETag, + LastModified: partInfo.LastModified, + ChecksumCRC32: partInfo.ChecksumCRC32, + ChecksumCRC32C: partInfo.ChecksumCRC32C, + ChecksumSHA1: partInfo.ChecksumSHA1, + ChecksumSHA256: partInfo.ChecksumSHA256, + ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME, + }) encodedSuccessResponse := encodeResponse(response) // Write success response. @@ -475,12 +565,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt actualPartSize = length var reader io.Reader = etag.NewReader(ctx, gr, nil, nil) + var checksumReader *hash.Reader mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + if _, ok := multipartChecksumType(mi.UserDefined); ok { + checksumReader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if err = prepareMultipartChecksumReader(checksumReader, mi.UserDefined, dstBucket, dstObject); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + reader = checksumReader + } _, isEncrypted := crypto.IsEncrypted(mi.UserDefined) @@ -512,6 +615,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt rawReader := srcInfo.Reader pReader := NewPutObjReader(rawReader) + pReader.setChecksumReader(checksumReader) var objectEncryptionKey crypto.ObjectKey if isEncrypted { @@ -591,7 +695,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3) } - response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified) + response := generateCopyObjectPartResponse(partInfo) encodedSuccessResponse := encodeResponse(response) // Write success response. @@ -745,6 +849,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http // Read compression metadata preserved in the init multipart for the decision. _, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"] var idxCb func() []byte + var checksumReader *hash.Reader if isCompressed { actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize) if err != nil { @@ -755,6 +860,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } + if err = prepareMultipartChecksumReader(actualReader, mi.UserDefined, bucket, object); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + checksumReader = actualReader // Set compression metrics. wantEncryption := crypto.Requested(r.Header) @@ -791,8 +901,16 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } + if checksumReader == nil { + if err = prepareMultipartChecksumReader(hashReader, mi.UserDefined, bucket, object); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + checksumReader = hashReader + } pReader := NewPutObjReader(hashReader) + pReader.setChecksumReader(checksumReader) _, isEncrypted := crypto.IsEncrypted(mi.UserDefined) _, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus] @@ -918,6 +1036,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http // Therefore, we have to set the ETag directly as map entry. w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""} hash.TransferChecksumHeader(w, r) + if isFederatedInternalRequest(r.UserAgent()) { + // Legacy federation proxies UploadPartCopy through minio-go + // Core.PutObjectPart, which can only recover checksums from response + // headers. Use the PartInfo returned by this exact write so the ETag and + // checksum cannot be mixed with a concurrent overwrite. + hash.AddChecksumHeader(w, partChecksumMap(partInfo)) + } writeSuccessResponseHeadersOnly(w) } diff --git a/cmd/os-readdir_test.go b/cmd/os-readdir_test.go index 5649b5391..a43ad5488 100644 --- a/cmd/os-readdir_test.go +++ b/cmd/os-readdir_test.go @@ -80,8 +80,6 @@ func setupTestReadDirFiles(t *testing.T) (testResults []result) { for i := range 10 { name := fmt.Sprintf("file-%d", i) if err := os.WriteFile(filepath.Join(dir, name), []byte{}, os.ModePerm); err != nil { - // For cleanup, its required to add these entries into test results. - testResults = append(testResults, result{dir, entries}) t.Fatalf("Unable to create file, %s", err) } entries = append(entries, name) @@ -105,8 +103,6 @@ func setupTestReadDirGeneric(t *testing.T) (testResults []result) { for i := range 10 { name := fmt.Sprintf("file-%d", i) if err := os.WriteFile(filepath.Join(dir, "mydir", name), []byte{}, os.ModePerm); err != nil { - // For cleanup, its required to add these entries into test results. - testResults = append(testResults, result{dir, entries}) t.Fatalf("Unable to write file, %s", err) } } @@ -130,8 +126,6 @@ func setupTestReadDirSymlink(t *testing.T) (testResults []result) { name1 := fmt.Sprintf("file-%d", i) name2 := fmt.Sprintf("file-%d", i+10) if err := os.WriteFile(filepath.Join(dir, name1), []byte{}, os.ModePerm); err != nil { - // For cleanup, its required to add these entries into test results. - testResults = append(testResults, result{dir, entries}) t.Fatalf("Unable to create a file, %s", err) } // Symlink will not be added to entries. diff --git a/cmd/server-main.go b/cmd/server-main.go index a547581f7..bedd39043 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -48,6 +48,7 @@ import ( "github.com/minio/minio/internal/color" "github.com/minio/minio/internal/config" "github.com/minio/minio/internal/config/api" + "github.com/minio/minio/internal/config/notify" "github.com/minio/minio/internal/handlers" "github.com/minio/minio/internal/hash/sha256" xhttp "github.com/minio/minio/internal/http" @@ -535,6 +536,15 @@ func configRetriableErrors(err error) bool { notInitialized } +func fatalServerConfigError(err error) bool { + var configErr config.Err + if errors.As(err, &configErr) { + return true + } + var migrationErr *notify.LegacyDatabaseTargetError + return errors.As(err, &migrationErr) +} + func bootstrapTraceMsg(msg string) { info := madmin.TraceInfo{ TraceType: madmin.TraceBootstrap, @@ -632,7 +642,10 @@ func initConfigSubsystem(ctx context.Context, newObject ObjectLayer) error { // Initialize config system. if err := globalConfigSys.Init(newObject); err != nil { - if configRetriableErrors(err) { + var migrationErr *notify.LegacyDatabaseTargetError + // Do not use fatalServerConfigError here: existing config.Err values + // retain the historical log-and-continue behavior at this boundary. + if configRetriableErrors(err) || errors.As(err, &migrationErr) { return fmt.Errorf("Unable to initialize config system: %w", err) } @@ -965,10 +978,9 @@ func serverMain(ctx *cli.Context) { var err error bootstrapTrace("initServerConfig", func() { if err = initServerConfig(GlobalContext, newObject); err != nil { - var cerr config.Err // For any config error, we don't need to drop into safe-mode // instead its a user error and should be fixed by user. - if errors.As(err, &cerr) { + if fatalServerConfigError(err) { logger.FatalIf(err, "Unable to initialize the server") } diff --git a/cmd/server-rlimit.go b/cmd/server-rlimit.go index ecb779e17..2d15ac828 100644 --- a/cmd/server-rlimit.go +++ b/cmd/server-rlimit.go @@ -46,7 +46,9 @@ func oldLinux() bool { func setMaxResources(ctx serverCtxt) (err error) { // Set the Go runtime max threads threshold to 90% of kernel setting. + //nolint:staticcheck // Linux implementations can fail; BSD stubs return a constant nil error. sysMaxThreads, err := sys.GetMaxThreads() + //nolint:staticcheck // Keep the shared cross-platform error handling. if err == nil { minioMaxThreads := (sysMaxThreads * 90) / 100 // Only set max threads if it is greater than the default one diff --git a/cmd/site-replication.go b/cmd/site-replication.go index 0b2f26f76..a166ae2ad 100644 --- a/cmd/site-replication.go +++ b/cmd/site-replication.go @@ -3734,7 +3734,7 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI bms.ExpiryLCConfig = &expLclCfgStr // if all non expiry rules only, ExpiryUpdatedAt would be nil if meta.lifecycleConfig.ExpiryUpdatedAt != nil { - bms.ExpiryLCConfigUpdatedAt = *(meta.lifecycleConfig.ExpiryUpdatedAt) + bms.ExpiryLCConfigUpdatedAt = *meta.lifecycleConfig.ExpiryUpdatedAt } } diff --git a/cmd/xl-storage-format-v2-legacy.go b/cmd/xl-storage-format-v2-legacy.go index ec2132279..9d682487b 100644 --- a/cmd/xl-storage-format-v2-legacy.go +++ b/cmd/xl-storage-format-v2-legacy.go @@ -50,7 +50,7 @@ func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) { err = msgp.ArrayError{Wanted: 4, Got: zb0001} return o, err } - bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:]) + bts, err = msgp.ReadExactBytes(bts, x.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return o, err @@ -145,7 +145,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) err = msgp.ArrayError{Wanted: 5, Got: zb0001} return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + bts, err = msgp.ReadExactBytes(bts, z.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return o, err @@ -155,7 +155,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) err = msgp.WrapError(err, "ModTime") return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:]) + bts, err = msgp.ReadExactBytes(bts, z.Signature[:]) if err != nil { err = msgp.WrapError(err, "Signature") return o, err @@ -195,7 +195,7 @@ func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.ArrayError{Wanted: 5, Got: zb0001} return err } - err = dc.ReadExactBytes((z.VersionID)[:]) + err = dc.ReadExactBytes(z.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return err @@ -205,7 +205,7 @@ func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "ModTime") return err } - err = dc.ReadExactBytes((z.Signature)[:]) + err = dc.ReadExactBytes(z.Signature[:]) if err != nil { err = msgp.WrapError(err, "Signature") return err diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index f3615e187..f50267e25 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -1101,7 +1101,7 @@ func (x *xlMetaV2) loadLegacy(buf []byte) error { return msgp.WrapError(err, "Versions") } if cap(x.versions) >= int(zb0002) { - x.versions = (x.versions)[:zb0002] + x.versions = x.versions[:zb0002] } else { x.versions = make([]xlMetaV2ShallowVersion, zb0002, zb0002+1) } diff --git a/docs/bucket/notifications/README.md b/docs/bucket/notifications/README.md index 3ba5f1009..5df1ce8fc 100644 --- a/docs/bucket/notifications/README.md +++ b/docs/bucket/notifications/README.md @@ -840,12 +840,11 @@ Received a message: {"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jp > database (string) database name (used only if `connection_string` is empty) > ``` > -> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure -> to migrate to only using _connection_string_ option. To migrate, once you have upgraded all the servers use the -> following command to update the existing notification targets. +> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to +> _connection_string_ before starting SILO. On the old server, use the following command to update the target. > > ``` -> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 username=psqluser password=psqlpass database=bucketevents" +> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 user=psqluser password=psqlpass dbname=bucketevents" > ``` > > Please make sure this step is carried out, without this step PostgreSQL notification targets will not work, @@ -973,9 +972,8 @@ key | value > database (string) database name (used only if `dsn_string` is empty) > ``` > -> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure -> to migrate to only using _dsn_string_ option. To migrate, once you have upgraded all the servers use the -> following command to update the existing notification targets. +> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to +> _dsn_string_ before starting SILO. On the old server, use the following command to update the target. > > ``` > mc admin config set mysilo/ notify_mysql[:name] dsn_string="mysqluser:mysqlpass@tcp(localhost:2832)/bucketevents" @@ -1045,7 +1043,7 @@ Before updating the configuration, let's start with `mc admin config get` comman ```sh $ mc admin config get mysilo/ notify_mysql -notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0 +notify_mysql:myinstance enable=off format=namespace dsn_string= table= queue_dir= queue_limit=0 ``` Use `mc admin config set` command to update MySQL notification configuration for the deployment with `dsn_string` parameter: diff --git a/docs/debugging/inspect/export.go b/docs/debugging/inspect/export.go index 016a412f6..293fc6c5a 100644 --- a/docs/debugging/inspect/export.go +++ b/docs/debugging/inspect/export.go @@ -358,7 +358,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) { e = msgp.ArrayError{Wanted: 5, Got: zb0001} return o, e } - bts, e = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + bts, e = msgp.ReadExactBytes(bts, z.VersionID[:]) if e != nil { e = msgp.WrapError(e, "VersionID") return o, e @@ -368,7 +368,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) { e = msgp.WrapError(e, "ModTime") return o, e } - bts, e = msgp.ReadExactBytes(bts, (z.Signature)[:]) + bts, e = msgp.ReadExactBytes(bts, z.Signature[:]) if e != nil { e = msgp.WrapError(e, "Signature") return o, e diff --git a/docs/debugging/xl-meta/main.go b/docs/debugging/xl-meta/main.go index 23c88d580..a30a037ee 100644 --- a/docs/debugging/xl-meta/main.go +++ b/docs/debugging/xl-meta/main.go @@ -745,7 +745,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt err = msgp.ArrayError{Wanted: want, Got: zb0001} return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + bts, err = msgp.ReadExactBytes(bts, z.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return o, err @@ -755,7 +755,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt err = msgp.WrapError(err, "ModTime") return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:]) + bts, err = msgp.ReadExactBytes(bts, z.Signature[:]) if err != nil { err = msgp.WrapError(err, "Signature") return o, err diff --git a/docs/security/advisories.md b/docs/security/advisories.md index 6860ae33e..6aa2df3f9 100644 --- a/docs/security/advisories.md +++ b/docs/security/advisories.md @@ -19,6 +19,7 @@ Entries carry a CVE identifier where one exists. Where none does, they carry a f | `SN-2026-003` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`2f55347f7`](https://github.com/pgsty/minio/commit/2f55347f78352aed8e08866d370c9426c73362cf) | S3/IAM bucket-policy condition values | Yes (policy-dependent) | Prevents raw request entries that spell condition-key names from shadowing or synthesizing internal condition values; confines `s3:signatureAge` to verified SigV4 presigned requests; separates query-only list fields from header-backed `x-amz-*` fields; and stops client request tags from impersonating stored existing-object tags. | The compatible query form remains for storage class and upload tagging on handlers that consume it; an explicitly present header wins, including an empty header. The historical `X-Amz-Tagging` Header mapping remains a client-supplied `RequestObjectTag` source, so use request-tag conditions only on operations that consume tags. Header-only `x-amz-*` policy keys no longer accept query substitutes. `aws:SourceIp` was left following the existing forwarding-header trust model; that model is addressed separately in the next row. See [Condition value sources and precedence](https://silo.pgsty.com/administration/identity-access-management/policy-based-access-control/#condition-value-sources). | | Not a vulnerability | `fe6dc4780` | Client source address (`aws:SourceIp`, audit `remotehost`, event notification `Host`) | N/A -- opt-in hardening | Adds an enforceable forwarded-header trust boundary, `MINIO_API_TRUSTED_PROXIES`. Set to a list of addresses or CIDR blocks, forwarded headers are believed only from those peers and forwarding chains are read right-to-left past listed hops -- which also stops the client-supplied left-most entry that an appending proxy (the stock nginx `$proxy_add_x_forwarded_for` recipe, or HAProxy's added second header line) leaves in place. Set to `none`, no forwarded header is believed at all. This is the guarantee `_MINIO_API_XFF_HEADER=off` never provided: it suppresses `X-Forwarded-For` alone, so `X-Real-IP` and RFC 7239 `Forwarded` remain one-line substitutions for anyone that setting was meant to stop. | **No behaviour change for any existing deployment**, so there is nothing to do on upgrade unless you want the new boundary. Not assigned a CVE: the default matches upstream, and upstream's own position (maintainer response in [discussion #17878](https://github.com/minio/minio/discussions/17878), Aug 2023) is that IP-based restrictions are impractical without reliable source-IP visibility. The gap being closed is that this was never written anywhere an operator would find it -- an `IpAddress` condition is accepted and behaves as though it works. **If you use `IpAddress` or `NotIpAddress` conditions, note that they were not enforceable before this change**, including behind a reverse proxy whose `X-Forwarded-For` recipe appends rather than overwrites. If you do not, the change affects only the accuracy of client addresses in logs. The new variable is opt-in and inert when unset; `_MINIO_API_XFF_HEADER` keeps its exact upstream semantics, and upstream's `TestXFFDisabled` is retained unmodified as the proof. An `IpAddress` condition remains unenforceable by default against a client with direct network access to the API port -- that is the condition the allowlist exists to fix, not a regression introduced here. When enabling the allowlist: it must name proxies, not the subnet they sit in, because entries are skipped while walking the chain, so a range that also covers clients lets those clients forge. Multi-node deployments must include their own node addresses, since MinIO forwards some requests between nodes and a client can force a hop through the `ListObjectsV2` continuation token; prefer the allowlist over `none` on a cluster for that reason. Loopback is always trusted as a peer so FTP and SFTP keep attributing their sessions. A malformed value stops startup, as does one that names no proxy at all (`","`) or one whose `env://` remote could not be read -- `env.Get` discards that error and yields an empty string, which would otherwise read as unset. Whitespace-only remains equivalent to unset. The policy is read after `MINIO_CONFIG_ENV_FILE` is loaded so environment-file deployments are covered; `_MINIO_API_XFF_HEADER` deliberately keeps upstream's earlier read timing, where a value written into an environment file is ignored. `MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES` now shares the same list parser, but is behaviourally untouched: the extraction is pure code motion, verified identical to the previous implementation across every combination of 37 allowlist values and 21 peer addresses. See [Client source address trust](source-address-trust.md). | | `SN-2026-004` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`97b7d2804`](https://github.com/pgsty/minio/commit/97b7d28040d109061c0a46a4c01bfc7800a97cc1) | IAM policy evaluation of bucket-level actions | Yes (policy-dependent) | Withholds twelve sensitive bucket-level writes from an object-only resource pattern. The IAM matcher appended a trailing slash for bucket-level requests (empty object name), so a resource of `arn:aws:s3:::bucket/*` matched `"bucket/"` and authorized bucket-level actions it was never meant to reach -- upstream [minio/minio#20449](https://github.com/minio/minio/issues/20449). The bucket-policy evaluation path never had the slash and was already reference-correct. | **This is an authorization tightening; read this row before upgrading if you write your own bucket-scoped policies.** Withheld from `bucket/*` on `Allow` statements only: `PutBucketPolicy`, `DeleteBucketPolicy`, `PutBucketObjectLockConfiguration`, `PutBucketVersioning`, `PutReplicationConfiguration`, `PutBucketLifecycle`, `DeleteBucket`, `ForceDeleteBucket`, `PutBucketCors`, `DeleteBucketCors`, `PutBucketQOS`, `PutInventoryConfiguration`. Membership was decided by one question -- does reaching this action give the caller something its object-scoped grant does not already give it? -- because the bug only fires when the statement already grants the bucket action, which in practice means `s3:*`, so the affected principal already holds full object CRUD. Only actions that hand out access to others, defeat a protection aimed at write-holders, act under server credentials, outlive the grant, or destroy the bucket entity qualify. **Deliberately not withheld, and asserted by test so re-adding one is a deliberate act**: `ListBucket`, `GetBucketLocation` and the read/list family, `PutBucketTagging`, `PutBucketEncryption`, `PutBucketNotification`, and `CreateBucket` -- so `mc ls`, SDK session setup and ordinary tenant self-service keep working through `bucket/*`. Breaking those is what got upstream's own full fix reverted. **What to change**: add the bare bucket ARN (`arn:aws:s3:::bucket`) alongside `arn:aws:s3:::bucket/*` in any statement that legitimately grants one of the twelve. Built-in canned policies are unaffected (all use `Resource: "*"`). `Deny` statements are untouched, so no bucket lock is ever weakened, and `NotResource` exclusions keep their full reach. The hardening is monotone by construction rather than by argument: the protected path requires **both** the bare and the historical `"bucket/"` form to match, an intersection with the historical decision -- without that, a fixed-width wildcard such as `mybucke?` would match `"mybucket"` while never having matched `"mybucket/"`, and the hardening would have granted a write the buggy matcher refused. `MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH=on` restores the historical behaviour in full; it is read once at startup. Still deferred to a migration-gated release: the read/list family, a startup audit naming affected policies, and a self-explaining denial log. | +| `SN-2026-005` | [`silo-pkg v3.12.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.12.0) and [`eee05a17c`](https://github.com/pgsty/silo/commit/eee05a17c34a07cebb27220d12697be74c8bd617) | IAM named-policy and service-account policy writes | No direct remote exploit; policy-dependent | Rejects S3, S3 Tables, and KMS ARN namespace prefixes that name no resource, including their historical `*arn:...` serialization, in both `Resource` and `NotResource`. A resource-matching `Deny` using such a prefix could silently fail to deny, while an `Allow` with the prefix in `NotResource` could match far more broadly than intended. The guard applies when creating named policies and when creating or updating service-account session policies. | **This is an authorization tightening for new and updated policies.** Existing policies keep loading, matching, importing, and replicating with unchanged runtime behavior, but a policy containing one of these prefixes cannot be submitted unchanged; replace it with the intended concrete resource, or use an explicit wildcard such as `arn:aws:s3:::*` only when all resources are intended. Enabling the strict path also rejects an admin statement that combines `Resource` with `NotResource`, and rejects non-S3 resources on bucket-scoped admin actions. Here “bare ARN prefix” means a namespace with no resource after it (`arn:aws:s3:::`); it is distinct from the valid “bare bucket ARN” in `SN-2026-004` (`arn:aws:s3:::bucket`). IAM import, site-replication receive paths, stored-policy loading, and STS inline policies remain on the permissive compatibility path in this release. | ## Dependency security updates diff --git a/go.mod b/go.mod index 40960c9a3..284a827b6 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,19 @@ module github.com/minio/minio -go 1.26.5 +go 1.27.0 // Use Pigsty's SILO Console v2.1.1 release while preserving upstream import paths. // The pseudo-version pins v2.1.1's commit because the compatible module path has no /v2 suffix. replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a // Use Pigsty's maintained mc fork for Console's embedded client code. -replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76 +replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb // Use Pigsty's maintained SILO package fork while preserving upstream import paths. // This retains the LDAP TLS fix tracked in https://github.com/pgsty/silo/issues/15. -// v3.11.0 follows upstream minio/pkg's 3.11 line and carries the -// minio/minio#20449 bucket-write boundary hardening. -replace github.com/minio/pkg/v3 => github.com/pgsty/silo-pkg/v3 v3.11.0 +// v3.12.0 carries the minio/minio#20449 bucket-write boundary hardening and +// rejects bare ARN prefixes on strict policy-write paths. +replace github.com/minio/pkg/v3 => github.com/pgsty/silo-pkg/v3 v3.12.0 // v22.7.0 does not compile on NetBSD because its unix implementation uses // CLOCK_MONOTONIC, which is unavailable there. Keep the last portable release @@ -37,7 +37,7 @@ require ( github.com/beevik/ntp v1.5.0 github.com/buger/jsonparser v1.1.2 github.com/cespare/xxhash/v2 v2.3.0 - github.com/cheggaaa/pb v1.0.29 + github.com/cheggaaa/pb v1.0.30 github.com/coreos/go-oidc/v3 v3.17.0 github.com/coreos/go-systemd/v22 v22.7.0 github.com/cosnicolaou/pbzip2 v1.0.6 @@ -65,7 +65,7 @@ require ( github.com/klauspost/reedsolomon v1.13.3 github.com/lib/pq v1.10.9 github.com/lithammer/shortuuid/v4 v4.2.0 - github.com/miekg/dns v1.1.72 + github.com/miekg/dns v1.1.73 github.com/minio/cli v1.24.2 github.com/minio/console v1.7.6 github.com/minio/csvparser v1.0.0 @@ -77,7 +77,7 @@ require ( github.com/minio/madmin-go/v3 v3.0.110 github.com/minio/minio-go/v7 v7.0.99 github.com/minio/mux v1.9.2 - github.com/minio/pkg/v3 v3.6.1 + github.com/minio/pkg/v3 v3.12.0 github.com/minio/selfupdate v0.6.0 github.com/minio/simdjson-go v0.4.5 github.com/minio/sio v0.4.3 @@ -90,9 +90,9 @@ require ( github.com/ncw/directio v1.0.5 github.com/nsqio/go-nsq v1.1.0 github.com/philhofer/fwd v1.2.0 - github.com/pierrec/lz4/v4 v4.1.26 + github.com/pierrec/lz4/v4 v4.1.29 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.10 + github.com/pkg/sftp v1.13.11 github.com/pkg/xattr v0.4.12 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 @@ -108,13 +108,13 @@ require ( github.com/valyala/bytebufferpool v1.0.0 github.com/xdg/scram v1.0.5 github.com/zeebo/xxh3 v1.1.0 - go.etcd.io/etcd/api/v3 v3.6.9 - go.etcd.io/etcd/client/v3 v3.6.9 + go.etcd.io/etcd/api/v3 v3.7.1 + go.etcd.io/etcd/client/v3 v3.7.1 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.28.0 go.yaml.in/yaml/v3 v3.0.5 - goftp.io/server/v2 v2.0.2 - golang.org/x/crypto v0.54.0 + goftp.io/server/v2 v2.0.3 + golang.org/x/crypto v0.55.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -210,7 +210,6 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect - github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect @@ -233,9 +232,8 @@ require ( github.com/lestrrat-go/dsig v1.0.0 // indirect github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect - github.com/lestrrat-go/httprc/v3 v3.0.1 // indirect - github.com/lestrrat-go/jwx/v3 v3.0.12 // indirect - github.com/lestrrat-go/option v1.0.1 // indirect + github.com/lestrrat-go/httprc/v3 v3.0.6 // indirect + github.com/lestrrat-go/jwx/v3 v3.0.13 // indirect github.com/lestrrat-go/option/v2 v2.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect @@ -283,12 +281,12 @@ require ( github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/unrolled/secure v1.17.0 // indirect - github.com/valyala/fastjson v1.6.4 // indirect + github.com/valyala/fastjson v1.6.10 // indirect github.com/vbauerster/mpb/v8 v8.12.0 // indirect github.com/xdg/stringprep v1.0.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.6.9 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.7.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect @@ -300,10 +298,11 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.57.0 // indirect - golang.org/x/text v0.40.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect + golang.org/x/mod v0.40.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.49.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/go.sum b/go.sum index 281f052ae..72ba8c3d9 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payR github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= -github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= -github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/cheggaaa/pb v1.0.30 h1:NylhgqJfXx3JVBGx6ywsXuhpz8caSMPmLArXyAv1bwU= +github.com/cheggaaa/pb v1.0.30/go.mod h1:YgTBwa6PqwwDB/2UKdLuuFRNTwEkcCPsA5AmWivrBAg= github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= @@ -411,12 +411,10 @@ github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7 github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/httprc/v3 v3.0.1 h1:3n7Es68YYGZb2Jf+k//llA4FTZMl3yCwIjFIk4ubevI= -github.com/lestrrat-go/httprc/v3 v3.0.1/go.mod h1:2uAvmbXE4Xq8kAUjVrZOq1tZVYYYs5iP62Cmtru00xk= -github.com/lestrrat-go/jwx/v3 v3.0.12 h1:p25r68Y4KrbBdYjIsQweYxq794CtGCzcrc5dGzJIRjg= -github.com/lestrrat-go/jwx/v3 v3.0.12/go.mod h1:HiUSaNmMLXgZ08OmGBaPVvoZQgJVOQphSrGr5zMamS8= -github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= -github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lestrrat-go/httprc/v3 v3.0.6 h1:4FpLQ18KK/ypPbVU3NLWJNRvH3kcYiqKqWfKGqNWxxI= +github.com/lestrrat-go/httprc/v3 v3.0.6/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0= +github.com/lestrrat-go/jwx/v3 v3.0.13 h1:AdHKiPIYeCSnOJtvdpipPg/0SuFh9rdkN+HF3O0VdSk= +github.com/lestrrat-go/jwx/v3 v3.0.13/go.mod h1:2m0PV1A9tM4b/jVLMx8rh6rBl7F6WGb3EG2hufN9OQU= github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -451,8 +449,8 @@ github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= -github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE= +github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ= github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg= github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/colorjson v1.0.8 h1:AS6gEQ1dTRYHmC4xuoodPDRILHP/9Wz5wYUGDQfPLpg= @@ -547,23 +545,23 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzb github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76 h1:UIlUuz0LQKw4QlAljhv7nPDDFC1+n+e0iED7rWZrgZ8= -github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76/go.mod h1:cTbS+9jGR4Qs7xTf5DEhmCTbzcDWrKMs8ZmTUnCU49E= +github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb h1:ixZKprZQpUIVrUlOy6H7oLb0a4B9eq6sLKrt8wNw1lU= +github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb/go.mod h1:cTbS+9jGR4Qs7xTf5DEhmCTbzcDWrKMs8ZmTUnCU49E= github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a h1:JfEQJkBdTwCXSrCv8P0R5tMpzxxhSdUTvg54mHwzxpA= github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a/go.mod h1:7J8wCQsNT5S7GqCHnQqgj0T7Nagp1fklJhfBJI+v0XI= -github.com/pgsty/silo-pkg/v3 v3.11.0 h1:wjN5d+tWD8Twq+e7k/KBBVhnWXC8xTIlfTcnGIKkmjc= -github.com/pgsty/silo-pkg/v3 v3.11.0/go.mod h1:E2AB4oOgfDeb9In1KDBTrn9wzfvr0WzoPkbXW7wbwBQ= +github.com/pgsty/silo-pkg/v3 v3.12.0 h1:1Bjqjb3KCt0oYhBLpH7W/e/5khTUoIgXWA12An1fbUc= +github.com/pgsty/silo-pkg/v3 v3.12.0/go.mod h1:ohHtpAK7kBCffdMT4oqUbtkoZMHxA7znTCPNjrYw86I= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= -github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg= +github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= -github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= +github.com/pkg/sftp v1.13.11 h1:0N92SLTB8JqASJB14ZLHHzFnBV8mG9zw4K7jghEFWuE= +github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hsN0= github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -639,7 +637,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -666,8 +663,8 @@ github.com/unrolled/secure v1.17.0 h1:Io7ifFgo99Bnh0J7+Q+qcMzWM6kaDPCA5FroFZEdbW github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= -github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= +github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE= github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg= github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= @@ -688,12 +685,12 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI= -go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk= -go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA= -go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y= -go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o= -go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM= +go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E= +go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78= +go.etcd.io/etcd/client/pkg/v3 v3.7.1 h1:rKYsj3pRkR0eK3yjT3XOgrhqfmIfj9pzNgxjh7mfFv4= +go.etcd.io/etcd/client/pkg/v3 v3.7.1/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0= +go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg= +go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= @@ -728,8 +725,8 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -goftp.io/server/v2 v2.0.2 h1:tkZpqyXys+vC15W5yGMi8Kzmbv1QSgeKr8qJXBnJbm8= -goftp.io/server/v2 v2.0.2/go.mod h1:Fl1WdcV7fx1pjOWx7jEHb7tsJ8VwE7+xHu6bVJ6r2qg= +goftp.io/server/v2 v2.0.3 h1:iz6Gxj7f2SFQVxrj0s1is+gueE6O9yTc+Ab0vtQ6Zn4= +goftp.io/server/v2 v2.0.3/go.mod h1:Fl1WdcV7fx1pjOWx7jEHb7tsJ8VwE7+xHu6bVJ6r2qg= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -742,15 +739,15 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//DnRYOnBiGJG14hFBhO8= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -764,8 +761,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -822,8 +819,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= @@ -833,8 +830,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/config/notify/legacy.go b/internal/config/notify/legacy.go index d2a1d67f3..21417f0da 100644 --- a/internal/config/notify/legacy.go +++ b/internal/config/notify/legacy.go @@ -26,6 +26,25 @@ import ( "github.com/minio/minio/internal/event/target" ) +// LegacyDatabaseTargetError reports a pre-KV database notification target +// that cannot be migrated safely. It deliberately carries no configuration +// values so credentials cannot escape through startup logs. +type LegacyDatabaseTargetError struct { + subsystem string + target string + connectionKey string + invalid bool +} + +func (e *LegacyDatabaseTargetError) Error() string { + if e.invalid { + return fmt.Sprintf("%s:%s has invalid %s or target settings; fix the target before migrating to SILO", + e.subsystem, e.target, e.connectionKey) + } + return fmt.Sprintf("%s:%s requires %s; discrete database connection fields are not migrated to SILO", + e.subsystem, e.target, e.connectionKey) +} + // SetNotifyKafka - helper for config migration from older config. func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error { if !cfg.Enable { @@ -325,8 +344,21 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg return nil } + if cfg.ConnectionString == "" { + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyPostgresSubSys, + target: psqName, + connectionKey: target.PostgresConnectionString, + } + } + if err := cfg.Validate(); err != nil { - return err + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyPostgresSubSys, + target: psqName, + connectionKey: target.PostgresConnectionString, + invalid: true, + } } s[config.NotifyPostgresSubSys][psqName] = config.KVS{ @@ -346,26 +378,6 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg Key: target.PostgresTable, Value: cfg.Table, }, - config.KV{ - Key: target.PostgresHost, - Value: cfg.Host.String(), - }, - config.KV{ - Key: target.PostgresPort, - Value: cfg.Port, - }, - config.KV{ - Key: target.PostgresUsername, - Value: cfg.Username, - }, - config.KV{ - Key: target.PostgresPassword, - Value: cfg.Password, - }, - config.KV{ - Key: target.PostgresDatabase, - Value: cfg.Database, - }, config.KV{ Key: target.PostgresQueueDir, Value: cfg.QueueDir, @@ -538,8 +550,21 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error return nil } + if cfg.DSN == "" { + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyMySQLSubSys, + target: sqlName, + connectionKey: target.MySQLDSNString, + } + } + if err := cfg.Validate(); err != nil { - return err + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyMySQLSubSys, + target: sqlName, + connectionKey: target.MySQLDSNString, + invalid: true, + } } s[config.NotifyMySQLSubSys][sqlName] = config.KVS{ @@ -559,26 +584,6 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error Key: target.MySQLTable, Value: cfg.Table, }, - config.KV{ - Key: target.MySQLHost, - Value: cfg.Host.String(), - }, - config.KV{ - Key: target.MySQLPort, - Value: cfg.Port, - }, - config.KV{ - Key: target.MySQLUsername, - Value: cfg.User, - }, - config.KV{ - Key: target.MySQLPassword, - Value: cfg.Password, - }, - config.KV{ - Key: target.MySQLDatabase, - Value: cfg.Database, - }, config.KV{ Key: target.MySQLQueueDir, Value: cfg.QueueDir, diff --git a/internal/config/notify/legacy_test.go b/internal/config/notify/legacy_test.go index 50f298d7f..011ad95fa 100644 --- a/internal/config/notify/legacy_test.go +++ b/internal/config/notify/legacy_test.go @@ -18,6 +18,8 @@ package notify import ( + "errors" + "strings" "testing" "github.com/minio/minio/internal/config" @@ -26,6 +28,25 @@ import ( "github.com/rabbitmq/amqp091-go" ) +func assertLegacyDatabaseTargetError(t *testing.T, err error, subsystem, name, key string, secrets ...string) { + t.Helper() + var targetErr *LegacyDatabaseTargetError + if !errors.As(err, &targetErr) { + t.Fatalf("error = %v, want *LegacyDatabaseTargetError", err) + } + msg := err.Error() + for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not contain %q", msg, want) + } + } + for _, secret := range secrets { + if secret != "" && strings.Contains(msg, secret) { + t.Errorf("error leaks configuration value %q: %s", secret, msg) + } + } +} + // T5 (NATS): a config produced by the legacy migration must survive validation // and round-trip back through the parser unchanged. Before the fix the // migration wrote an env var name as a config key, so every migrated NATS @@ -113,3 +134,172 @@ func TestSetNotifyAMQPRoundTrip(t *testing.T) { t.Errorf("Internal = true, want false (immediate must not be written to the internal key)") } } + +func TestSetNotifyDatabaseTargetsRequireConnectionStrings(t *testing.T) { + postgresHost, err := xnet.ParseHost("legacy-postgres.example") + if err != nil { + t.Fatal(err) + } + mysqlHost, err := xnet.ParseURL("legacy-mysql.example") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + subsystem string + key string + set func(config.Config) error + secrets []string + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + set: func(s config.Config) error { + return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{ + Enable: true, + Format: formatNamespace, + Table: "events", + Host: *postgresHost, + Port: "5432", + Username: "legacy-user", + Password: "legacy-postgres-password", + Database: "legacy-database", + }) + }, + secrets: []string{postgresHost.String(), "5432", "legacy-user", "legacy-postgres-password", "legacy-database"}, + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + set: func(s config.Config) error { + return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{ + Enable: true, + Format: formatNamespace, + Table: "events", + Host: *mysqlHost, + Port: "3306", + User: "legacy-user", + Password: "legacy-mysql-password", + Database: "legacy-database", + }) + }, + secrets: []string{mysqlHost.String(), "3306", "legacy-user", "legacy-mysql-password", "legacy-database"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := config.Config{test.subsystem: map[string]config.KVS{}} + err := test.set(s) + assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secrets...) + if _, ok := s[test.subsystem][testTargetName]; ok { + t.Fatal("unsupported target was emitted despite migration error") + } + }) + } +} + +func TestSetNotifyDisabledDatabaseTargetsAreIgnored(t *testing.T) { + s := config.Config{ + config.NotifyPostgresSubSys: map[string]config.KVS{}, + config.NotifyMySQLSubSys: map[string]config.KVS{}, + } + if err := SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{Password: "discarded-postgres-secret"}); err != nil { + t.Fatalf("SetNotifyPostgres: %v", err) + } + if err := SetNotifyMySQL(s, testTargetName, target.MySQLArgs{Password: "discarded-mysql-secret"}); err != nil { + t.Fatalf("SetNotifyMySQL: %v", err) + } + if _, ok := s[config.NotifyPostgresSubSys][testTargetName]; ok { + t.Fatal("disabled Postgres target was emitted") + } + if _, ok := s[config.NotifyMySQLSubSys][testTargetName]; ok { + t.Fatal("disabled MySQL target was emitted") + } +} + +func TestSetNotifyInvalidDatabaseTargetsDoNotLeak(t *testing.T) { + tests := []struct { + name string + subsystem string + key string + secret string + set func(config.Config) error + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + secret: "postgres-dsn-secret", + set: func(s config.Config) error { + return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{ + Enable: true, + Format: formatNamespace, + ConnectionString: "host=db password=postgres-dsn-secret", + }) + }, + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + secret: "mysql-dsn-secret", + set: func(s config.Config) error { + return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{ + Enable: true, + Format: formatNamespace, + DSN: "user:mysql-dsn-secret@tcp(db:3306/events", + Table: "events", + }) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := config.Config{test.subsystem: map[string]config.KVS{}} + err := test.set(s) + assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secret) + }) + } +} + +func TestDatabaseConnectionStringsSurviveKVTokenization(t *testing.T) { + tests := []struct { + name string + subsystem string + key string + input string + want string + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + input: `notify_postgres:dsn connection_string="host=db port=5432 dbname=events user=app password=inside" table="events"`, + want: "host=db port=5432 dbname=events user=app password=inside", + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + input: `notify_mysql:dsn dsn_string="user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside" table="events"`, + want: "user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := config.Config{test.subsystem: map[string]config.KVS{}} + if _, err := s.SetKVS(test.input, DefaultNotificationKVS); err != nil { + t.Fatalf("SetKVS: %v", err) + } + if got := s[test.subsystem]["dsn"].Get(test.key); got != test.want { + t.Errorf("%s = %q, want %q", test.key, got, test.want) + } + }) + } +} diff --git a/internal/config/notify/parse_test.go b/internal/config/notify/parse_test.go index 51e99747f..074461b1e 100644 --- a/internal/config/notify/parse_test.go +++ b/internal/config/notify/parse_test.go @@ -414,21 +414,9 @@ var configPkgConsts = map[string]string{ "Comment": config.Comment, } -// knownUnregisteredWrites records pre-existing instances of the exact defect -// this audit exists to catch: a legacy migration writing config keys that no -// default KVS registers, so the migrated config is rejected on the next load. -// -// These are inherited from upstream and are the same class as issue #39, but -// they are NOT part of the issue #39 fix and were left untouched deliberately. -// The Postgres/MySQL keys below are the pre-connection-string DSN fields; the -// migration still writes them and `password` carries a plaintext database -// password. -// -// This list must only ever shrink. Do not add entries to silence a new gap. -var knownUnregisteredWrites = map[string][]string{ - "SetNotifyPostgres": {"host", "port", "username", "password", "database"}, - "SetNotifyMySQL": {"host", "port", "username", "password", "database"}, -} +// knownUnregisteredWrites is a shrink-only ratchet for inherited migration +// gaps. Do not add entries to silence a new mismatch. +var knownUnregisteredWrites = map[string][]string{} func TestNotifyConfigKeysAreRegistered(t *testing.T) { targetConsts, err := parseTargetPkgStringConsts("../../event/target") diff --git a/internal/disk/directio_unix.go b/internal/disk/directio_unix.go index 883df95d0..a31fc6bd3 100644 --- a/internal/disk/directio_unix.go +++ b/internal/disk/directio_unix.go @@ -43,7 +43,7 @@ func DisableDirectIO(f *os.File) error { if err != nil { return err } - flag &= ^(syscall.O_DIRECT) + flag &= ^syscall.O_DIRECT _, err = unix.FcntlInt(fd, unix.F_SETFL, flag) return err } diff --git a/internal/event/target/nats_contrib_test.go b/internal/event/target/nats_contrib_test.go index 42a5f0609..965e4d9ef 100644 --- a/internal/event/target/nats_contrib_test.go +++ b/internal/event/target/nats_contrib_test.go @@ -35,7 +35,7 @@ func TestNatsConnPlain(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -59,7 +59,7 @@ func TestNatsConnUserPass(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -85,7 +85,7 @@ func TestNatsConnToken(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -116,7 +116,7 @@ func TestNatsConnNKeySeed(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", diff --git a/internal/event/target/nats_tls_contrib_test.go b/internal/event/target/nats_tls_contrib_test.go index 30cf5b46b..120c86da7 100644 --- a/internal/event/target/nats_tls_contrib_test.go +++ b/internal/event/target/nats_tls_contrib_test.go @@ -33,7 +33,7 @@ func TestNatsConnTLSCustomCA(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -56,7 +56,7 @@ func TestNatsConnTLSCustomCAHandshakeFirst(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -80,7 +80,7 @@ func TestNatsConnTLSClientAuthorization(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", diff --git a/internal/grid/connection.go b/internal/grid/connection.go index 576f4229a..ac3651c84 100644 --- a/internal/grid/connection.go +++ b/internal/grid/connection.go @@ -1806,8 +1806,8 @@ func (ww *wsWriter) writeFrame(w io.Writer, f ws.Frame) error { const ( bit0 = 0x80 len7 = int64(125) - len16 = int64(^(uint16(0))) - len64 = int64(^(uint64(0)) >> 1) + len16 = int64(^uint16(0)) + len64 = int64(^uint64(0) >> 1) ) bts := ww.tmp[:] diff --git a/internal/hash/checksum.go b/internal/hash/checksum.go index fbf62fd84..5131087b8 100644 --- a/internal/hash/checksum.go +++ b/internal/hash/checksum.go @@ -247,7 +247,7 @@ func (c ChecksumType) StringFull() string { // FullObjectRequested will return if the checksum type indicates full object checksum was requested. func (c ChecksumType) FullObjectRequested() bool { - return c&(ChecksumFullObject) == ChecksumFullObject || c.Is(ChecksumCRC64NVME) + return c&ChecksumFullObject == ChecksumFullObject || c.Is(ChecksumCRC64NVME) } // IsMultipartComposite returns true if the checksum is multipart and full object was not requested. diff --git a/internal/ioutil/wait_pipe.go b/internal/ioutil/wait_pipe.go index 67f490ba4..d8e3eab68 100644 --- a/internal/ioutil/wait_pipe.go +++ b/internal/ioutil/wait_pipe.go @@ -58,10 +58,10 @@ func WaitPipe() (*PipeReader, *PipeWriter) { var wg sync.WaitGroup wg.Add(1) return &PipeReader{ - PipeReader: r, - wait: wg.Wait, - }, &PipeWriter{ - PipeWriter: w, - done: wg.Done, - } + PipeReader: r, + wait: wg.Wait, + }, &PipeWriter{ + PipeWriter: w, + done: wg.Done, + } }