From 43f4bb7ed454539e9513b38566ccc0d5974353b7 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 01:08:46 +0800 Subject: [PATCH 01/13] chore(deps): align the SILO Go dependency stack Standardize the related SILO components on Go 1.27 tooling, etcd 3.7.1, current Go-maintained modules, shared runtime versions, and explicit security and portability pins. Keep the shared package Go 1.26 consumer floor, isolate lint tooling from product dependency selection, and preserve upstream-compatible import paths. --- .github/workflows/vulncheck.yml | 2 +- .golangci.yml | 3 + CREDITS | 2 +- Dockerfile.goreleaser | 2 +- Makefile | 4 +- buildscripts/checkdeps.sh | 2 +- cmd/bucket-object-lock.go | 4 +- cmd/data-usage-cache.go | 2 +- cmd/metrics-realtime.go | 2 + cmd/os-readdir_test.go | 6 -- cmd/server-rlimit.go | 2 + cmd/site-replication.go | 2 +- cmd/xl-storage-format-v2-legacy.go | 10 +-- cmd/xl-storage-format-v2.go | 2 +- docs/debugging/inspect/export.go | 4 +- docs/debugging/xl-meta/main.go | 4 +- go.mod | 40 +++++----- go.sum | 75 +++++++++---------- internal/disk/directio_unix.go | 2 +- internal/event/target/nats_contrib_test.go | 8 +- .../event/target/nats_tls_contrib_test.go | 6 +- internal/grid/connection.go | 4 +- internal/hash/checksum.go | 2 +- internal/ioutil/wait_pipe.go | 12 +-- 24 files changed, 100 insertions(+), 102 deletions(-) diff --git a/.github/workflows/vulncheck.yml b/.github/workflows/vulncheck.yml index 87a4aa7e9..63fc1c634 100644 --- a/.github/workflows/vulncheck.yml +++ b/.github/workflows/vulncheck.yml @@ -29,7 +29,7 @@ jobs: - name: Install govulncheck run: | - go install golang.org/x/vuln/cmd/govulncheck@v1.6.0 + go install golang.org/x/vuln/cmd/govulncheck@v1.7.0 echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}" - name: Run govulncheck diff --git a/.golangci.yml b/.golangci.yml index 0533d7cd9..2f5cfc6aa 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -59,6 +59,9 @@ formatters: exclusions: generated: lax paths: + # gofumpt v0.11.0 and Go 1.27's gofmt disagree on the indentation of + # multiple composite literals returned from a single statement. + - internal/ioutil/wait_pipe\.go$ - third_party$ - builtin$ - examples$ diff --git a/CREDITS b/CREDITS index 256a8bf37..110776ccd 100644 --- a/CREDITS +++ b/CREDITS @@ -3396,7 +3396,7 @@ SOFTWARE. github.com/cheggaaa/pb https://github.com/cheggaaa/pb ---------------------------------------------------------------- -Copyright (c) 2012-2015, Sergey Cherepanov +Copyright (c) 2012-2024, Sergey Cherepanov All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index a09d75e14..15d03aadf 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,4 +1,4 @@ -FROM golang:1.26.5-alpine AS build +FROM golang:1.27.0-alpine AS build ARG TARGETARCH diff --git a/Makefile b/Makefile index 96cfe0162..529913f85 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go) GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) -GOLANGCI_VERSION ?= v2.11.3 +GOLANGCI_VERSION ?= v2.13.1 VERSION ?= $(shell git describe --tags) REPO ?= docker.io/pgsty @@ -48,7 +48,7 @@ credits: ## regenerate CREDITS from the licenses of Go modules linked into the b check-gen: ## check for updated autogenerated files @go generate ./... >/dev/null - @go mod tidy -compat=1.26 + @go mod tidy -compat=1.27 @env bash $(PWD)/buildscripts/gen-credits.sh @changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum CREDITS); \ if [ -n "$$changed" ]; then \ diff --git a/buildscripts/checkdeps.sh b/buildscripts/checkdeps.sh index ed4f666ea..4fc385a29 100755 --- a/buildscripts/checkdeps.sh +++ b/buildscripts/checkdeps.sh @@ -7,7 +7,7 @@ _init() { ## Minimum required versions for build dependencies GIT_VERSION="1.0" - GO_VERSION="1.16" + GO_VERSION="1.27.0" OSX_VERSION="10.8" KNAME=$(uname -s) ARCH=$(uname -m) diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index d0ad85144..33a0ebda5 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -198,7 +198,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec byPassSet, r, cred, owner) // Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set. if !byPassSet { - if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) { + if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) { return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID} } } @@ -209,7 +209,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec case objectlock.RetCompliance: // Compliance retention mode cannot be changed or shortened. // https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes - if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) { + if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) { return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID} } apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name, diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index 5752daae0..48ec09b2c 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -326,7 +326,7 @@ func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool { if cycles <= 1 { return cycles == 1 } - return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles + return uint32(xxhash.Sum64String(string(h))>>32)%cycles == cycle%cycles } // addChild will add a child based on its hash. diff --git a/cmd/metrics-realtime.go b/cmd/metrics-realtime.go index 4a1d55732..9eb10990f 100644 --- a/cmd/metrics-realtime.go +++ b/cmd/metrics-realtime.go @@ -191,7 +191,9 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM } } + //nolint:staticcheck // Linux implementations can fail; BSD stubs return a constant nil error. st, err := disk.GetDriveStats(d.Major, d.Minor) + //nolint:staticcheck // Keep the shared cross-platform error handling. if err == nil { dm.IOStats = madmin.DiskIOStats{ ReadIOs: st.ReadIOs, diff --git a/cmd/os-readdir_test.go b/cmd/os-readdir_test.go index 5649b5391..a43ad5488 100644 --- a/cmd/os-readdir_test.go +++ b/cmd/os-readdir_test.go @@ -80,8 +80,6 @@ func setupTestReadDirFiles(t *testing.T) (testResults []result) { for i := range 10 { name := fmt.Sprintf("file-%d", i) if err := os.WriteFile(filepath.Join(dir, name), []byte{}, os.ModePerm); err != nil { - // For cleanup, its required to add these entries into test results. - testResults = append(testResults, result{dir, entries}) t.Fatalf("Unable to create file, %s", err) } entries = append(entries, name) @@ -105,8 +103,6 @@ func setupTestReadDirGeneric(t *testing.T) (testResults []result) { for i := range 10 { name := fmt.Sprintf("file-%d", i) if err := os.WriteFile(filepath.Join(dir, "mydir", name), []byte{}, os.ModePerm); err != nil { - // For cleanup, its required to add these entries into test results. - testResults = append(testResults, result{dir, entries}) t.Fatalf("Unable to write file, %s", err) } } @@ -130,8 +126,6 @@ func setupTestReadDirSymlink(t *testing.T) (testResults []result) { name1 := fmt.Sprintf("file-%d", i) name2 := fmt.Sprintf("file-%d", i+10) if err := os.WriteFile(filepath.Join(dir, name1), []byte{}, os.ModePerm); err != nil { - // For cleanup, its required to add these entries into test results. - testResults = append(testResults, result{dir, entries}) t.Fatalf("Unable to create a file, %s", err) } // Symlink will not be added to entries. diff --git a/cmd/server-rlimit.go b/cmd/server-rlimit.go index ecb779e17..2d15ac828 100644 --- a/cmd/server-rlimit.go +++ b/cmd/server-rlimit.go @@ -46,7 +46,9 @@ func oldLinux() bool { func setMaxResources(ctx serverCtxt) (err error) { // Set the Go runtime max threads threshold to 90% of kernel setting. + //nolint:staticcheck // Linux implementations can fail; BSD stubs return a constant nil error. sysMaxThreads, err := sys.GetMaxThreads() + //nolint:staticcheck // Keep the shared cross-platform error handling. if err == nil { minioMaxThreads := (sysMaxThreads * 90) / 100 // Only set max threads if it is greater than the default one diff --git a/cmd/site-replication.go b/cmd/site-replication.go index 0b2f26f76..a166ae2ad 100644 --- a/cmd/site-replication.go +++ b/cmd/site-replication.go @@ -3734,7 +3734,7 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI bms.ExpiryLCConfig = &expLclCfgStr // if all non expiry rules only, ExpiryUpdatedAt would be nil if meta.lifecycleConfig.ExpiryUpdatedAt != nil { - bms.ExpiryLCConfigUpdatedAt = *(meta.lifecycleConfig.ExpiryUpdatedAt) + bms.ExpiryLCConfigUpdatedAt = *meta.lifecycleConfig.ExpiryUpdatedAt } } diff --git a/cmd/xl-storage-format-v2-legacy.go b/cmd/xl-storage-format-v2-legacy.go index ec2132279..9d682487b 100644 --- a/cmd/xl-storage-format-v2-legacy.go +++ b/cmd/xl-storage-format-v2-legacy.go @@ -50,7 +50,7 @@ func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) { err = msgp.ArrayError{Wanted: 4, Got: zb0001} return o, err } - bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:]) + bts, err = msgp.ReadExactBytes(bts, x.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return o, err @@ -145,7 +145,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) err = msgp.ArrayError{Wanted: 5, Got: zb0001} return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + bts, err = msgp.ReadExactBytes(bts, z.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return o, err @@ -155,7 +155,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) err = msgp.WrapError(err, "ModTime") return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:]) + bts, err = msgp.ReadExactBytes(bts, z.Signature[:]) if err != nil { err = msgp.WrapError(err, "Signature") return o, err @@ -195,7 +195,7 @@ func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.ArrayError{Wanted: 5, Got: zb0001} return err } - err = dc.ReadExactBytes((z.VersionID)[:]) + err = dc.ReadExactBytes(z.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return err @@ -205,7 +205,7 @@ func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "ModTime") return err } - err = dc.ReadExactBytes((z.Signature)[:]) + err = dc.ReadExactBytes(z.Signature[:]) if err != nil { err = msgp.WrapError(err, "Signature") return err diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index f3615e187..f50267e25 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -1101,7 +1101,7 @@ func (x *xlMetaV2) loadLegacy(buf []byte) error { return msgp.WrapError(err, "Versions") } if cap(x.versions) >= int(zb0002) { - x.versions = (x.versions)[:zb0002] + x.versions = x.versions[:zb0002] } else { x.versions = make([]xlMetaV2ShallowVersion, zb0002, zb0002+1) } diff --git a/docs/debugging/inspect/export.go b/docs/debugging/inspect/export.go index 016a412f6..293fc6c5a 100644 --- a/docs/debugging/inspect/export.go +++ b/docs/debugging/inspect/export.go @@ -358,7 +358,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) { e = msgp.ArrayError{Wanted: 5, Got: zb0001} return o, e } - bts, e = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + bts, e = msgp.ReadExactBytes(bts, z.VersionID[:]) if e != nil { e = msgp.WrapError(e, "VersionID") return o, e @@ -368,7 +368,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) { e = msgp.WrapError(e, "ModTime") return o, e } - bts, e = msgp.ReadExactBytes(bts, (z.Signature)[:]) + bts, e = msgp.ReadExactBytes(bts, z.Signature[:]) if e != nil { e = msgp.WrapError(e, "Signature") return o, e diff --git a/docs/debugging/xl-meta/main.go b/docs/debugging/xl-meta/main.go index 23c88d580..a30a037ee 100644 --- a/docs/debugging/xl-meta/main.go +++ b/docs/debugging/xl-meta/main.go @@ -745,7 +745,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt err = msgp.ArrayError{Wanted: want, Got: zb0001} return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + bts, err = msgp.ReadExactBytes(bts, z.VersionID[:]) if err != nil { err = msgp.WrapError(err, "VersionID") return o, err @@ -755,7 +755,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt err = msgp.WrapError(err, "ModTime") return o, err } - bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:]) + bts, err = msgp.ReadExactBytes(bts, z.Signature[:]) if err != nil { err = msgp.WrapError(err, "Signature") return o, err diff --git a/go.mod b/go.mod index 40960c9a3..2e027f488 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/minio/minio -go 1.26.5 +go 1.27.0 // Use Pigsty's SILO Console v2.1.1 release while preserving upstream import paths. // The pseudo-version pins v2.1.1's commit because the compatible module path has no /v2 suffix. replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a // Use Pigsty's maintained mc fork for Console's embedded client code. -replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76 +replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb // Use Pigsty's maintained SILO package fork while preserving upstream import paths. // This retains the LDAP TLS fix tracked in https://github.com/pgsty/silo/issues/15. @@ -37,7 +37,7 @@ require ( github.com/beevik/ntp v1.5.0 github.com/buger/jsonparser v1.1.2 github.com/cespare/xxhash/v2 v2.3.0 - github.com/cheggaaa/pb v1.0.29 + github.com/cheggaaa/pb v1.0.30 github.com/coreos/go-oidc/v3 v3.17.0 github.com/coreos/go-systemd/v22 v22.7.0 github.com/cosnicolaou/pbzip2 v1.0.6 @@ -65,7 +65,7 @@ require ( github.com/klauspost/reedsolomon v1.13.3 github.com/lib/pq v1.10.9 github.com/lithammer/shortuuid/v4 v4.2.0 - github.com/miekg/dns v1.1.72 + github.com/miekg/dns v1.1.73 github.com/minio/cli v1.24.2 github.com/minio/console v1.7.6 github.com/minio/csvparser v1.0.0 @@ -77,7 +77,7 @@ require ( github.com/minio/madmin-go/v3 v3.0.110 github.com/minio/minio-go/v7 v7.0.99 github.com/minio/mux v1.9.2 - github.com/minio/pkg/v3 v3.6.1 + github.com/minio/pkg/v3 v3.11.0 github.com/minio/selfupdate v0.6.0 github.com/minio/simdjson-go v0.4.5 github.com/minio/sio v0.4.3 @@ -90,9 +90,9 @@ require ( github.com/ncw/directio v1.0.5 github.com/nsqio/go-nsq v1.1.0 github.com/philhofer/fwd v1.2.0 - github.com/pierrec/lz4/v4 v4.1.26 + github.com/pierrec/lz4/v4 v4.1.29 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.10 + github.com/pkg/sftp v1.13.11 github.com/pkg/xattr v0.4.12 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/client_model v0.6.2 @@ -108,13 +108,13 @@ require ( github.com/valyala/bytebufferpool v1.0.0 github.com/xdg/scram v1.0.5 github.com/zeebo/xxh3 v1.1.0 - go.etcd.io/etcd/api/v3 v3.6.9 - go.etcd.io/etcd/client/v3 v3.6.9 + go.etcd.io/etcd/api/v3 v3.7.1 + go.etcd.io/etcd/client/v3 v3.7.1 go.uber.org/atomic v1.11.0 go.uber.org/zap v1.28.0 go.yaml.in/yaml/v3 v3.0.5 - goftp.io/server/v2 v2.0.2 - golang.org/x/crypto v0.54.0 + goftp.io/server/v2 v2.0.3 + golang.org/x/crypto v0.55.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -233,9 +233,8 @@ require ( github.com/lestrrat-go/dsig v1.0.0 // indirect github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect - github.com/lestrrat-go/httprc/v3 v3.0.1 // indirect - github.com/lestrrat-go/jwx/v3 v3.0.12 // indirect - github.com/lestrrat-go/option v1.0.1 // indirect + github.com/lestrrat-go/httprc/v3 v3.0.6 // indirect + github.com/lestrrat-go/jwx/v3 v3.0.13 // indirect github.com/lestrrat-go/option/v2 v2.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect @@ -283,12 +282,12 @@ require ( github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/unrolled/secure v1.17.0 // indirect - github.com/valyala/fastjson v1.6.4 // indirect + github.com/valyala/fastjson v1.6.10 // indirect github.com/vbauerster/mpb/v8 v8.12.0 // indirect github.com/xdg/stringprep v1.0.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.6.9 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.7.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect @@ -300,10 +299,11 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - golang.org/x/mod v0.37.0 // indirect - golang.org/x/net v0.57.0 // indirect - golang.org/x/text v0.40.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect + golang.org/x/mod v0.40.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/text v0.41.0 // indirect + golang.org/x/tools v0.49.0 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/go.sum b/go.sum index 281f052ae..82c73ff4e 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,8 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payR github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= -github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= -github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/cheggaaa/pb v1.0.30 h1:NylhgqJfXx3JVBGx6ywsXuhpz8caSMPmLArXyAv1bwU= +github.com/cheggaaa/pb v1.0.30/go.mod h1:YgTBwa6PqwwDB/2UKdLuuFRNTwEkcCPsA5AmWivrBAg= github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= @@ -411,12 +411,10 @@ github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7 github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/httprc/v3 v3.0.1 h1:3n7Es68YYGZb2Jf+k//llA4FTZMl3yCwIjFIk4ubevI= -github.com/lestrrat-go/httprc/v3 v3.0.1/go.mod h1:2uAvmbXE4Xq8kAUjVrZOq1tZVYYYs5iP62Cmtru00xk= -github.com/lestrrat-go/jwx/v3 v3.0.12 h1:p25r68Y4KrbBdYjIsQweYxq794CtGCzcrc5dGzJIRjg= -github.com/lestrrat-go/jwx/v3 v3.0.12/go.mod h1:HiUSaNmMLXgZ08OmGBaPVvoZQgJVOQphSrGr5zMamS8= -github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= -github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lestrrat-go/httprc/v3 v3.0.6 h1:4FpLQ18KK/ypPbVU3NLWJNRvH3kcYiqKqWfKGqNWxxI= +github.com/lestrrat-go/httprc/v3 v3.0.6/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0= +github.com/lestrrat-go/jwx/v3 v3.0.13 h1:AdHKiPIYeCSnOJtvdpipPg/0SuFh9rdkN+HF3O0VdSk= +github.com/lestrrat-go/jwx/v3 v3.0.13/go.mod h1:2m0PV1A9tM4b/jVLMx8rh6rBl7F6WGb3EG2hufN9OQU= github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -451,8 +449,8 @@ github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= -github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE= +github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ= github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg= github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/colorjson v1.0.8 h1:AS6gEQ1dTRYHmC4xuoodPDRILHP/9Wz5wYUGDQfPLpg= @@ -547,23 +545,23 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzb github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76 h1:UIlUuz0LQKw4QlAljhv7nPDDFC1+n+e0iED7rWZrgZ8= -github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76/go.mod h1:cTbS+9jGR4Qs7xTf5DEhmCTbzcDWrKMs8ZmTUnCU49E= +github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb h1:ixZKprZQpUIVrUlOy6H7oLb0a4B9eq6sLKrt8wNw1lU= +github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb/go.mod h1:cTbS+9jGR4Qs7xTf5DEhmCTbzcDWrKMs8ZmTUnCU49E= github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a h1:JfEQJkBdTwCXSrCv8P0R5tMpzxxhSdUTvg54mHwzxpA= github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a/go.mod h1:7J8wCQsNT5S7GqCHnQqgj0T7Nagp1fklJhfBJI+v0XI= github.com/pgsty/silo-pkg/v3 v3.11.0 h1:wjN5d+tWD8Twq+e7k/KBBVhnWXC8xTIlfTcnGIKkmjc= github.com/pgsty/silo-pkg/v3 v3.11.0/go.mod h1:E2AB4oOgfDeb9In1KDBTrn9wzfvr0WzoPkbXW7wbwBQ= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= -github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg= +github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= -github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= +github.com/pkg/sftp v1.13.11 h1:0N92SLTB8JqASJB14ZLHHzFnBV8mG9zw4K7jghEFWuE= +github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hsN0= github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM= github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -639,7 +637,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -666,8 +663,8 @@ github.com/unrolled/secure v1.17.0 h1:Io7ifFgo99Bnh0J7+Q+qcMzWM6kaDPCA5FroFZEdbW github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= -github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4= +github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE= github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg= github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= @@ -688,12 +685,12 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI= -go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk= -go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA= -go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y= -go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o= -go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM= +go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E= +go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78= +go.etcd.io/etcd/client/pkg/v3 v3.7.1 h1:rKYsj3pRkR0eK3yjT3XOgrhqfmIfj9pzNgxjh7mfFv4= +go.etcd.io/etcd/client/pkg/v3 v3.7.1/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0= +go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg= +go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= @@ -728,8 +725,8 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -goftp.io/server/v2 v2.0.2 h1:tkZpqyXys+vC15W5yGMi8Kzmbv1QSgeKr8qJXBnJbm8= -goftp.io/server/v2 v2.0.2/go.mod h1:Fl1WdcV7fx1pjOWx7jEHb7tsJ8VwE7+xHu6bVJ6r2qg= +goftp.io/server/v2 v2.0.3 h1:iz6Gxj7f2SFQVxrj0s1is+gueE6O9yTc+Ab0vtQ6Zn4= +goftp.io/server/v2 v2.0.3/go.mod h1:Fl1WdcV7fx1pjOWx7jEHb7tsJ8VwE7+xHu6bVJ6r2qg= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -742,15 +739,15 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//DnRYOnBiGJG14hFBhO8= +golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= -golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -764,8 +761,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -822,8 +819,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= @@ -833,8 +830,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/disk/directio_unix.go b/internal/disk/directio_unix.go index 883df95d0..a31fc6bd3 100644 --- a/internal/disk/directio_unix.go +++ b/internal/disk/directio_unix.go @@ -43,7 +43,7 @@ func DisableDirectIO(f *os.File) error { if err != nil { return err } - flag &= ^(syscall.O_DIRECT) + flag &= ^syscall.O_DIRECT _, err = unix.FcntlInt(fd, unix.F_SETFL, flag) return err } diff --git a/internal/event/target/nats_contrib_test.go b/internal/event/target/nats_contrib_test.go index 42a5f0609..965e4d9ef 100644 --- a/internal/event/target/nats_contrib_test.go +++ b/internal/event/target/nats_contrib_test.go @@ -35,7 +35,7 @@ func TestNatsConnPlain(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -59,7 +59,7 @@ func TestNatsConnUserPass(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -85,7 +85,7 @@ func TestNatsConnToken(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -116,7 +116,7 @@ func TestNatsConnNKeySeed(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", diff --git a/internal/event/target/nats_tls_contrib_test.go b/internal/event/target/nats_tls_contrib_test.go index 30cf5b46b..120c86da7 100644 --- a/internal/event/target/nats_tls_contrib_test.go +++ b/internal/event/target/nats_tls_contrib_test.go @@ -33,7 +33,7 @@ func TestNatsConnTLSCustomCA(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -56,7 +56,7 @@ func TestNatsConnTLSCustomCAHandshakeFirst(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", @@ -80,7 +80,7 @@ func TestNatsConnTLSClientAuthorization(t *testing.T) { Enable: true, Address: xnet.Host{ Name: "localhost", - Port: (xnet.Port(opts.Port)), + Port: xnet.Port(opts.Port), IsPortSet: true, }, Subject: "test", diff --git a/internal/grid/connection.go b/internal/grid/connection.go index 576f4229a..ac3651c84 100644 --- a/internal/grid/connection.go +++ b/internal/grid/connection.go @@ -1806,8 +1806,8 @@ func (ww *wsWriter) writeFrame(w io.Writer, f ws.Frame) error { const ( bit0 = 0x80 len7 = int64(125) - len16 = int64(^(uint16(0))) - len64 = int64(^(uint64(0)) >> 1) + len16 = int64(^uint16(0)) + len64 = int64(^uint64(0) >> 1) ) bts := ww.tmp[:] diff --git a/internal/hash/checksum.go b/internal/hash/checksum.go index fbf62fd84..5131087b8 100644 --- a/internal/hash/checksum.go +++ b/internal/hash/checksum.go @@ -247,7 +247,7 @@ func (c ChecksumType) StringFull() string { // FullObjectRequested will return if the checksum type indicates full object checksum was requested. func (c ChecksumType) FullObjectRequested() bool { - return c&(ChecksumFullObject) == ChecksumFullObject || c.Is(ChecksumCRC64NVME) + return c&ChecksumFullObject == ChecksumFullObject || c.Is(ChecksumCRC64NVME) } // IsMultipartComposite returns true if the checksum is multipart and full object was not requested. diff --git a/internal/ioutil/wait_pipe.go b/internal/ioutil/wait_pipe.go index 67f490ba4..d8e3eab68 100644 --- a/internal/ioutil/wait_pipe.go +++ b/internal/ioutil/wait_pipe.go @@ -58,10 +58,10 @@ func WaitPipe() (*PipeReader, *PipeWriter) { var wg sync.WaitGroup wg.Add(1) return &PipeReader{ - PipeReader: r, - wait: wg.Wait, - }, &PipeWriter{ - PipeWriter: w, - done: wg.Done, - } + PipeReader: r, + wait: wg.Wait, + }, &PipeWriter{ + PipeWriter: w, + done: wg.Done, + } } From f1ba68358265efd2cd937e9a9929c1df1a49301c Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 02:22:20 +0800 Subject: [PATCH 02/13] fix: require DSNs for legacy database notifications Reject pre-KV PostgreSQL and MySQL targets that lack a canonical connection string, propagate the typed migration error to the fatal startup boundary, and stop emitting unregistered discrete connection keys.\n\nCloses the implementation for #53; release and issue closure remain separate gates. --- cmd/config-migrate.go | 8 +- cmd/config-migrate_test.go | 264 ++++++++++++++++++++++++++ cmd/server-main.go | 18 +- internal/config/notify/legacy.go | 89 +++++---- internal/config/notify/legacy_test.go | 190 ++++++++++++++++++ internal/config/notify/parse_test.go | 18 +- 6 files changed, 525 insertions(+), 62 deletions(-) create mode 100644 cmd/config-migrate_test.go diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go index 0cceb1859..170279ab5 100644 --- a/cmd/config-migrate.go +++ b/cmd/config-migrate.go @@ -167,7 +167,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C notify.SetNotifyMQTT(newCfg, k, args) } for k, args := range cfg.Notify.MySQL { - notify.SetNotifyMySQL(newCfg, k, args) + if err := notify.SetNotifyMySQL(newCfg, k, args); err != nil { + return nil, err + } } for k, args := range cfg.Notify.NATS { notify.SetNotifyNATS(newCfg, k, args) @@ -176,7 +178,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C notify.SetNotifyNSQ(newCfg, k, args) } for k, args := range cfg.Notify.PostgreSQL { - notify.SetNotifyPostgres(newCfg, k, args) + if err := notify.SetNotifyPostgres(newCfg, k, args); err != nil { + return nil, err + } } for k, args := range cfg.Notify.Redis { notify.SetNotifyRedis(newCfg, k, args) diff --git a/cmd/config-migrate_test.go b/cmd/config-migrate_test.go new file mode 100644 index 000000000..76a616d9b --- /dev/null +++ b/cmd/config-migrate_test.go @@ -0,0 +1,264 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/notify" + "github.com/minio/minio/internal/event/target" +) + +func installLegacyConfigFile(t *testing.T, configure func(*serverConfigV33)) (string, []byte) { + t.Helper() + + cfg := &serverConfigV33{ + Version: "33", + Notify: notify.NewConfig(), + } + configure(cfg) + + data, err := json.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + + oldConfigDir := globalConfigDir + globalConfigDir = &ConfigDir{path: t.TempDir()} + t.Cleanup(func() { globalConfigDir = oldConfigDir }) + + configFile := getConfigFile() + if err = os.WriteFile(configFile, data, 0o600); err != nil { + t.Fatal(err) + } + return configFile, data +} + +func assertLegacyMigrationError(t *testing.T, err error, subsystem, name, key, secret string) { + t.Helper() + var targetErr *notify.LegacyDatabaseTargetError + if !errors.As(err, &targetErr) { + t.Fatalf("error = %v, want *notify.LegacyDatabaseTargetError", err) + } + msg := err.Error() + for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not contain %q", msg, want) + } + } + if strings.Contains(msg, secret) { + t.Errorf("migration error leaks database password %q: %s", secret, msg) + } +} + +func TestReadConfigWithoutMigrateRejectsLegacyDatabaseTargets(t *testing.T) { + tests := []struct { + name string + subsystem string + key string + secret string + configure func(*serverConfigV33) + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + secret: "postgres-migration-secret", + configure: func(cfg *serverConfigV33) { + cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{ + Enable: true, + Port: "5432", + Username: "legacy-user", + Password: "postgres-migration-secret", + Database: "events", + } + }, + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + secret: "mysql-migration-secret", + configure: func(cfg *serverConfigV33) { + cfg.Notify.MySQL["archive"] = target.MySQLArgs{ + Enable: true, + Port: "3306", + User: "legacy-user", + Password: "mysql-migration-secret", + Database: "events", + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + configFile, original := installLegacyConfigFile(t, test.configure) + got, err := readConfigWithoutMigrate(t.Context(), nil) + if got != nil { + t.Fatalf("config = %v, want nil on failed migration", got) + } + assertLegacyMigrationError(t, err, test.subsystem, "archive", test.key, test.secret) + + after, readErr := os.ReadFile(configFile) + if readErr != nil { + t.Fatal(readErr) + } + if !bytes.Equal(after, original) { + t.Fatal("failed migration rewrote the legacy source config") + } + if _, statErr := os.Stat(configFile + ".old"); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("failed migration created a backup/persistence artifact: %v", statErr) + } + }) + } +} + +func TestReadConfigWithoutMigrateMigratesCanonicalDatabaseTargets(t *testing.T) { + const ( + postgresConnection = "host=postgres.example port=5432 dbname=events user=app password=secret sslmode=disable" + mysqlDSN = "app:secret@tcp(mysql.example:3306)/events?parseTime=true" + discardedLegacyValue = "discarded-legacy-value" + ) + installLegacyConfigFile(t, func(cfg *serverConfigV33) { + cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{ + Enable: true, + Format: "namespace", + ConnectionString: postgresConnection, + Table: "events", + Port: discardedLegacyValue, + Username: discardedLegacyValue, + Password: discardedLegacyValue, + Database: discardedLegacyValue, + } + cfg.Notify.MySQL["archive"] = target.MySQLArgs{ + Enable: true, + Format: "namespace", + DSN: mysqlDSN, + Table: "events", + Port: discardedLegacyValue, + User: discardedLegacyValue, + Password: discardedLegacyValue, + Database: discardedLegacyValue, + } + }) + + got, err := readConfigWithoutMigrate(t.Context(), nil) + if err != nil { + t.Fatalf("readConfigWithoutMigrate: %v", err) + } + tests := []struct { + subsystem string + key string + want string + discarded string + }{ + {config.NotifyPostgresSubSys, target.PostgresConnectionString, postgresConnection, discardedLegacyValue}, + {config.NotifyMySQLSubSys, target.MySQLDSNString, mysqlDSN, discardedLegacyValue}, + } + for _, test := range tests { + kvs := got[test.subsystem]["archive"] + if value := kvs.Get(test.key); value != test.want { + t.Errorf("%s %s = %q, want %q", test.subsystem, test.key, value, test.want) + } + if err := config.CheckValidKeys(test.subsystem+config.SubSystemSeparator+"archive", kvs, notify.DefaultNotificationKVS[test.subsystem]); err != nil { + t.Errorf("migrated %s target failed key validation: %v", test.subsystem, err) + } + for _, key := range []string{"host", "port", "username", "password", "database"} { + if _, ok := kvs.Lookup(key); ok { + t.Errorf("migrated %s target contains legacy key %q", test.subsystem, key) + } + } + for _, kv := range kvs { + if strings.Contains(kv.Value, test.discarded) { + t.Errorf("migrated %s target contains discarded legacy value in %q", test.subsystem, kv.Key) + } + } + } + + postgresTargets, err := notify.GetNotifyPostgres(got[config.NotifyPostgresSubSys]) + if err != nil { + t.Fatalf("GetNotifyPostgres: %v", err) + } + if value := postgresTargets["archive"].ConnectionString; value != postgresConnection { + t.Errorf("Postgres connection string = %q, want %q", value, postgresConnection) + } + mysqlTargets, err := notify.GetNotifyMySQL(got[config.NotifyMySQLSubSys]) + if err != nil { + t.Fatalf("GetNotifyMySQL: %v", err) + } + if value := mysqlTargets["archive"].DSN; value != mysqlDSN { + t.Errorf("MySQL DSN = %q, want %q", value, mysqlDSN) + } +} + +func TestInitConfigSubsystemReturnsLegacyDatabaseTargetError(t *testing.T) { + obj, fsDir, err := prepareFS(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = obj.Shutdown(context.Background()) + _ = os.RemoveAll(fsDir) + }) + + const secret = "startup-migration-secret" + installLegacyConfigFile(t, func(cfg *serverConfigV33) { + cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{ + Enable: true, + Port: "5432", + Username: "legacy-user", + Password: secret, + Database: "events", + } + }) + + globalServerConfigMu.RLock() + var before config.Config + if globalServerConfig != nil { + before = globalServerConfig.Clone() + } + globalServerConfigMu.RUnlock() + + err = initConfigSubsystem(t.Context(), obj) + assertLegacyMigrationError(t, err, config.NotifyPostgresSubSys, "archive", target.PostgresConnectionString, secret) + if configRetriableErrors(err) { + t.Fatal("legacy database migration error must be startup-fatal, not retriable") + } + if !fatalServerConfigError(err) { + t.Fatal("legacy database migration error must abort server startup") + } + + globalServerConfigMu.RLock() + var after config.Config + if globalServerConfig != nil { + after = globalServerConfig.Clone() + } + globalServerConfigMu.RUnlock() + if !reflect.DeepEqual(after, before) { + t.Fatal("failed migration activated a partial server configuration") + } +} diff --git a/cmd/server-main.go b/cmd/server-main.go index a547581f7..bedd39043 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -48,6 +48,7 @@ import ( "github.com/minio/minio/internal/color" "github.com/minio/minio/internal/config" "github.com/minio/minio/internal/config/api" + "github.com/minio/minio/internal/config/notify" "github.com/minio/minio/internal/handlers" "github.com/minio/minio/internal/hash/sha256" xhttp "github.com/minio/minio/internal/http" @@ -535,6 +536,15 @@ func configRetriableErrors(err error) bool { notInitialized } +func fatalServerConfigError(err error) bool { + var configErr config.Err + if errors.As(err, &configErr) { + return true + } + var migrationErr *notify.LegacyDatabaseTargetError + return errors.As(err, &migrationErr) +} + func bootstrapTraceMsg(msg string) { info := madmin.TraceInfo{ TraceType: madmin.TraceBootstrap, @@ -632,7 +642,10 @@ func initConfigSubsystem(ctx context.Context, newObject ObjectLayer) error { // Initialize config system. if err := globalConfigSys.Init(newObject); err != nil { - if configRetriableErrors(err) { + var migrationErr *notify.LegacyDatabaseTargetError + // Do not use fatalServerConfigError here: existing config.Err values + // retain the historical log-and-continue behavior at this boundary. + if configRetriableErrors(err) || errors.As(err, &migrationErr) { return fmt.Errorf("Unable to initialize config system: %w", err) } @@ -965,10 +978,9 @@ func serverMain(ctx *cli.Context) { var err error bootstrapTrace("initServerConfig", func() { if err = initServerConfig(GlobalContext, newObject); err != nil { - var cerr config.Err // For any config error, we don't need to drop into safe-mode // instead its a user error and should be fixed by user. - if errors.As(err, &cerr) { + if fatalServerConfigError(err) { logger.FatalIf(err, "Unable to initialize the server") } diff --git a/internal/config/notify/legacy.go b/internal/config/notify/legacy.go index d2a1d67f3..21417f0da 100644 --- a/internal/config/notify/legacy.go +++ b/internal/config/notify/legacy.go @@ -26,6 +26,25 @@ import ( "github.com/minio/minio/internal/event/target" ) +// LegacyDatabaseTargetError reports a pre-KV database notification target +// that cannot be migrated safely. It deliberately carries no configuration +// values so credentials cannot escape through startup logs. +type LegacyDatabaseTargetError struct { + subsystem string + target string + connectionKey string + invalid bool +} + +func (e *LegacyDatabaseTargetError) Error() string { + if e.invalid { + return fmt.Sprintf("%s:%s has invalid %s or target settings; fix the target before migrating to SILO", + e.subsystem, e.target, e.connectionKey) + } + return fmt.Sprintf("%s:%s requires %s; discrete database connection fields are not migrated to SILO", + e.subsystem, e.target, e.connectionKey) +} + // SetNotifyKafka - helper for config migration from older config. func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error { if !cfg.Enable { @@ -325,8 +344,21 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg return nil } + if cfg.ConnectionString == "" { + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyPostgresSubSys, + target: psqName, + connectionKey: target.PostgresConnectionString, + } + } + if err := cfg.Validate(); err != nil { - return err + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyPostgresSubSys, + target: psqName, + connectionKey: target.PostgresConnectionString, + invalid: true, + } } s[config.NotifyPostgresSubSys][psqName] = config.KVS{ @@ -346,26 +378,6 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg Key: target.PostgresTable, Value: cfg.Table, }, - config.KV{ - Key: target.PostgresHost, - Value: cfg.Host.String(), - }, - config.KV{ - Key: target.PostgresPort, - Value: cfg.Port, - }, - config.KV{ - Key: target.PostgresUsername, - Value: cfg.Username, - }, - config.KV{ - Key: target.PostgresPassword, - Value: cfg.Password, - }, - config.KV{ - Key: target.PostgresDatabase, - Value: cfg.Database, - }, config.KV{ Key: target.PostgresQueueDir, Value: cfg.QueueDir, @@ -538,8 +550,21 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error return nil } + if cfg.DSN == "" { + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyMySQLSubSys, + target: sqlName, + connectionKey: target.MySQLDSNString, + } + } + if err := cfg.Validate(); err != nil { - return err + return &LegacyDatabaseTargetError{ + subsystem: config.NotifyMySQLSubSys, + target: sqlName, + connectionKey: target.MySQLDSNString, + invalid: true, + } } s[config.NotifyMySQLSubSys][sqlName] = config.KVS{ @@ -559,26 +584,6 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error Key: target.MySQLTable, Value: cfg.Table, }, - config.KV{ - Key: target.MySQLHost, - Value: cfg.Host.String(), - }, - config.KV{ - Key: target.MySQLPort, - Value: cfg.Port, - }, - config.KV{ - Key: target.MySQLUsername, - Value: cfg.User, - }, - config.KV{ - Key: target.MySQLPassword, - Value: cfg.Password, - }, - config.KV{ - Key: target.MySQLDatabase, - Value: cfg.Database, - }, config.KV{ Key: target.MySQLQueueDir, Value: cfg.QueueDir, diff --git a/internal/config/notify/legacy_test.go b/internal/config/notify/legacy_test.go index 50f298d7f..011ad95fa 100644 --- a/internal/config/notify/legacy_test.go +++ b/internal/config/notify/legacy_test.go @@ -18,6 +18,8 @@ package notify import ( + "errors" + "strings" "testing" "github.com/minio/minio/internal/config" @@ -26,6 +28,25 @@ import ( "github.com/rabbitmq/amqp091-go" ) +func assertLegacyDatabaseTargetError(t *testing.T, err error, subsystem, name, key string, secrets ...string) { + t.Helper() + var targetErr *LegacyDatabaseTargetError + if !errors.As(err, &targetErr) { + t.Fatalf("error = %v, want *LegacyDatabaseTargetError", err) + } + msg := err.Error() + for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} { + if !strings.Contains(msg, want) { + t.Errorf("error %q does not contain %q", msg, want) + } + } + for _, secret := range secrets { + if secret != "" && strings.Contains(msg, secret) { + t.Errorf("error leaks configuration value %q: %s", secret, msg) + } + } +} + // T5 (NATS): a config produced by the legacy migration must survive validation // and round-trip back through the parser unchanged. Before the fix the // migration wrote an env var name as a config key, so every migrated NATS @@ -113,3 +134,172 @@ func TestSetNotifyAMQPRoundTrip(t *testing.T) { t.Errorf("Internal = true, want false (immediate must not be written to the internal key)") } } + +func TestSetNotifyDatabaseTargetsRequireConnectionStrings(t *testing.T) { + postgresHost, err := xnet.ParseHost("legacy-postgres.example") + if err != nil { + t.Fatal(err) + } + mysqlHost, err := xnet.ParseURL("legacy-mysql.example") + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + subsystem string + key string + set func(config.Config) error + secrets []string + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + set: func(s config.Config) error { + return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{ + Enable: true, + Format: formatNamespace, + Table: "events", + Host: *postgresHost, + Port: "5432", + Username: "legacy-user", + Password: "legacy-postgres-password", + Database: "legacy-database", + }) + }, + secrets: []string{postgresHost.String(), "5432", "legacy-user", "legacy-postgres-password", "legacy-database"}, + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + set: func(s config.Config) error { + return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{ + Enable: true, + Format: formatNamespace, + Table: "events", + Host: *mysqlHost, + Port: "3306", + User: "legacy-user", + Password: "legacy-mysql-password", + Database: "legacy-database", + }) + }, + secrets: []string{mysqlHost.String(), "3306", "legacy-user", "legacy-mysql-password", "legacy-database"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := config.Config{test.subsystem: map[string]config.KVS{}} + err := test.set(s) + assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secrets...) + if _, ok := s[test.subsystem][testTargetName]; ok { + t.Fatal("unsupported target was emitted despite migration error") + } + }) + } +} + +func TestSetNotifyDisabledDatabaseTargetsAreIgnored(t *testing.T) { + s := config.Config{ + config.NotifyPostgresSubSys: map[string]config.KVS{}, + config.NotifyMySQLSubSys: map[string]config.KVS{}, + } + if err := SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{Password: "discarded-postgres-secret"}); err != nil { + t.Fatalf("SetNotifyPostgres: %v", err) + } + if err := SetNotifyMySQL(s, testTargetName, target.MySQLArgs{Password: "discarded-mysql-secret"}); err != nil { + t.Fatalf("SetNotifyMySQL: %v", err) + } + if _, ok := s[config.NotifyPostgresSubSys][testTargetName]; ok { + t.Fatal("disabled Postgres target was emitted") + } + if _, ok := s[config.NotifyMySQLSubSys][testTargetName]; ok { + t.Fatal("disabled MySQL target was emitted") + } +} + +func TestSetNotifyInvalidDatabaseTargetsDoNotLeak(t *testing.T) { + tests := []struct { + name string + subsystem string + key string + secret string + set func(config.Config) error + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + secret: "postgres-dsn-secret", + set: func(s config.Config) error { + return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{ + Enable: true, + Format: formatNamespace, + ConnectionString: "host=db password=postgres-dsn-secret", + }) + }, + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + secret: "mysql-dsn-secret", + set: func(s config.Config) error { + return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{ + Enable: true, + Format: formatNamespace, + DSN: "user:mysql-dsn-secret@tcp(db:3306/events", + Table: "events", + }) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := config.Config{test.subsystem: map[string]config.KVS{}} + err := test.set(s) + assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secret) + }) + } +} + +func TestDatabaseConnectionStringsSurviveKVTokenization(t *testing.T) { + tests := []struct { + name string + subsystem string + key string + input string + want string + }{ + { + name: "postgres", + subsystem: config.NotifyPostgresSubSys, + key: target.PostgresConnectionString, + input: `notify_postgres:dsn connection_string="host=db port=5432 dbname=events user=app password=inside" table="events"`, + want: "host=db port=5432 dbname=events user=app password=inside", + }, + { + name: "mysql", + subsystem: config.NotifyMySQLSubSys, + key: target.MySQLDSNString, + input: `notify_mysql:dsn dsn_string="user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside" table="events"`, + want: "user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + s := config.Config{test.subsystem: map[string]config.KVS{}} + if _, err := s.SetKVS(test.input, DefaultNotificationKVS); err != nil { + t.Fatalf("SetKVS: %v", err) + } + if got := s[test.subsystem]["dsn"].Get(test.key); got != test.want { + t.Errorf("%s = %q, want %q", test.key, got, test.want) + } + }) + } +} diff --git a/internal/config/notify/parse_test.go b/internal/config/notify/parse_test.go index 51e99747f..074461b1e 100644 --- a/internal/config/notify/parse_test.go +++ b/internal/config/notify/parse_test.go @@ -414,21 +414,9 @@ var configPkgConsts = map[string]string{ "Comment": config.Comment, } -// knownUnregisteredWrites records pre-existing instances of the exact defect -// this audit exists to catch: a legacy migration writing config keys that no -// default KVS registers, so the migrated config is rejected on the next load. -// -// These are inherited from upstream and are the same class as issue #39, but -// they are NOT part of the issue #39 fix and were left untouched deliberately. -// The Postgres/MySQL keys below are the pre-connection-string DSN fields; the -// migration still writes them and `password` carries a plaintext database -// password. -// -// This list must only ever shrink. Do not add entries to silence a new gap. -var knownUnregisteredWrites = map[string][]string{ - "SetNotifyPostgres": {"host", "port", "username", "password", "database"}, - "SetNotifyMySQL": {"host", "port", "username", "password", "database"}, -} +// knownUnregisteredWrites is a shrink-only ratchet for inherited migration +// gaps. Do not add entries to silence a new mismatch. +var knownUnregisteredWrites = map[string][]string{} func TestNotifyConfigKeysAreRegistered(t *testing.T) { targetConsts, err := parseTargetPkgStringConsts("../../event/target") From c565987b9c925838a59b91dd632e9ed60a907615 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 02:29:22 +0800 Subject: [PATCH 03/13] docs: align database notification migration guidance --- docs/bucket/notifications/README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/bucket/notifications/README.md b/docs/bucket/notifications/README.md index 3ba5f1009..5df1ce8fc 100644 --- a/docs/bucket/notifications/README.md +++ b/docs/bucket/notifications/README.md @@ -840,12 +840,11 @@ Received a message: {"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jp > database (string) database name (used only if `connection_string` is empty) > ``` > -> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure -> to migrate to only using _connection_string_ option. To migrate, once you have upgraded all the servers use the -> following command to update the existing notification targets. +> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to +> _connection_string_ before starting SILO. On the old server, use the following command to update the target. > > ``` -> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 username=psqluser password=psqlpass database=bucketevents" +> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 user=psqluser password=psqlpass dbname=bucketevents" > ``` > > Please make sure this step is carried out, without this step PostgreSQL notification targets will not work, @@ -973,9 +972,8 @@ key | value > database (string) database name (used only if `dsn_string` is empty) > ``` > -> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure -> to migrate to only using _dsn_string_ option. To migrate, once you have upgraded all the servers use the -> following command to update the existing notification targets. +> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to +> _dsn_string_ before starting SILO. On the old server, use the following command to update the target. > > ``` > mc admin config set mysilo/ notify_mysql[:name] dsn_string="mysqluser:mysqlpass@tcp(localhost:2832)/bucketevents" @@ -1045,7 +1043,7 @@ Before updating the configuration, let's start with `mc admin config get` comman ```sh $ mc admin config get mysilo/ notify_mysql -notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0 +notify_mysql:myinstance enable=off format=namespace dsn_string= table= queue_dir= queue_limit=0 ``` Use `mc admin config set` command to update MySQL notification configuration for the deployment with `dsn_string` parameter: From 68eeb002f695fa320ff9191937426e7ab074f7d7 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 12:20:13 +0800 Subject: [PATCH 04/13] chore: record notification migration compatibility symbols --- buildscripts/rebrand-guard/compat-baseline.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index 63859b1fc..49c90d7b2 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -6938,7 +6938,9 @@ "internal/config/notify:notify:func:SetNotifyRedis", "internal/config/notify:notify:func:SetNotifyWebhook", "internal/config/notify:notify:func:TestSubSysNotificationTargets", + "internal/config/notify:notify:method:LegacyDatabaseTargetError.Error", "internal/config/notify:notify:type:Config", + "internal/config/notify:notify:type:LegacyDatabaseTargetError", "internal/config/notify:notify:var:DefaultAMQPKVS", "internal/config/notify:notify:var:DefaultESKVS", "internal/config/notify:notify:var:DefaultKafkaKVS", From 7fea6d5a5fa9802eea5e13af39d03606b0bea276 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 12:20:52 +0800 Subject: [PATCH 05/13] fix: compute multipart part checksums server-side Accept checksum-enabled UploadPart and UploadPartCopy requests when clients omit optional per-part checksum headers. Compute over the logical plaintext stream, persist the result, and return it from CopyPartResult while preserving client checksum validation.\n\nRefs #46; follow-ups #63 and #64 remain out of scope. --- .../rebrand-guard/compat-baseline.json | 5 + cmd/api-response.go | 19 +- cmd/erasure-multipart-upload-checksum_test.go | 596 ++++++++++++++++++ cmd/erasure-multipart.go | 18 +- cmd/object-api-utils.go | 45 +- cmd/object-multipart-handlers.go | 82 ++- 6 files changed, 751 insertions(+), 14 deletions(-) create mode 100644 cmd/erasure-multipart-upload-checksum_test.go diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index 49c90d7b2..6296c75f1 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -1824,6 +1824,11 @@ "cmd:cmd:field:CompletePart.PartNumber", "cmd:cmd:field:CompletePart.Size", "cmd:cmd:field:ConsoleLogger.Enable", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32C", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC64NVME", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA1", + "cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA256", "cmd:cmd:field:CopyObjectPartResponse.ETag", "cmd:cmd:field:CopyObjectPartResponse.LastModified", "cmd:cmd:field:CopyObjectPartResponse.XMLName", diff --git a/cmd/api-response.go b/cmd/api-response.go index cf25fd980..e7e22582c 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -387,6 +387,12 @@ type CopyObjectPartResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"` LastModified string // time string of format "2006-01-02T15:04:05.000Z" ETag string // md5sum of the copied object part. + + ChecksumCRC32 string `xml:",omitempty"` + ChecksumCRC32C string `xml:",omitempty"` + ChecksumSHA1 string `xml:",omitempty"` + ChecksumSHA256 string `xml:",omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` } // Initiator inherit from Owner struct, fields are same @@ -771,11 +777,16 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR } } -// generates CopyObjectPartResponse from etag and lastModified time. -func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse { +// generates CopyObjectPartResponse from the uploaded part information. +func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse { return CopyObjectPartResponse{ - ETag: "\"" + etag + "\"", - LastModified: amztime.ISO8601Format(lastModified.UTC()), + ETag: "\"" + partInfo.ETag + "\"", + LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()), + ChecksumCRC32: partInfo.ChecksumCRC32, + ChecksumCRC32C: partInfo.ChecksumCRC32C, + ChecksumSHA1: partInfo.ChecksumSHA1, + ChecksumSHA256: partInfo.ChecksumSHA256, + ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME, } } diff --git a/cmd/erasure-multipart-upload-checksum_test.go b/cmd/erasure-multipart-upload-checksum_test.go new file mode 100644 index 000000000..dd10a2026 --- /dev/null +++ b/cmd/erasure-multipart-upload-checksum_test.go @@ -0,0 +1,596 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/xml" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +func uploadPartHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, object, uploadID string, partNumber int, data []byte, headers map[string]string, +) (string, *httptest.ResponseRecorder) { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(partNumber)), + int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build UploadPart request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("UploadPart failed: %d %s", rec.Code, rec.Body.String()) + } + return canonicalizeETag(rec.Header()[xhttp.ETag][0]), rec +} + +func listPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, object, uploadID string, headers map[string]string, +) ListPartsResponse { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodGet, + getListMultipartURLWithParams("", bucket, object, uploadID, "1000", "", ""), + 0, nil, creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build ListParts request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("ListParts failed: %d %s", rec.Code, rec.Body.String()) + } + var response ListPartsResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("failed to decode ListParts response: %v", err) + } + return response +} + +func partChecksum(typ hash.ChecksumType, part Part) string { + switch typ.Base() { + case hash.ChecksumCRC32: + return part.ChecksumCRC32 + case hash.ChecksumCRC32C: + return part.ChecksumCRC32C + case hash.ChecksumSHA1: + return part.ChecksumSHA1 + case hash.ChecksumSHA256: + return part.ChecksumSHA256 + case hash.ChecksumCRC64NVME: + return part.ChecksumCRC64NVME + default: + return "" + } +} + +func copyPartChecksum(typ hash.ChecksumType, response CopyObjectPartResponse) string { + switch typ.Base() { + case hash.ChecksumCRC32: + return response.ChecksumCRC32 + case hash.ChecksumCRC32C: + return response.ChecksumCRC32C + case hash.ChecksumSHA1: + return response.ChecksumSHA1 + case hash.ChecksumSHA256: + return response.ChecksumSHA256 + case hash.ChecksumCRC64NVME: + return response.ChecksumCRC64NVME + default: + return "" + } +} + +func completePartWithChecksum(typ hash.ChecksumType, partNumber int, etag, checksum string) CompletePart { + part := CompletePart{PartNumber: partNumber, ETag: etag} + switch typ.Base() { + case hash.ChecksumCRC32: + part.ChecksumCRC32 = checksum + case hash.ChecksumCRC32C: + part.ChecksumCRC32C = checksum + case hash.ChecksumSHA1: + part.ChecksumSHA1 = checksum + case hash.ChecksumSHA256: + part.ChecksumSHA256 = checksum + case hash.ChecksumCRC64NVME: + part.ChecksumCRC64NVME = checksum + } + return part +} + +func completePartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, object, uploadID string, parts []CompletePart, headers map[string]string, +) *httptest.ResponseRecorder { + t.Helper() + body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts}) + if err != nil { + t.Fatalf("failed to encode CompleteMultipartUpload request: %v", err) + } + req, err := newTestSignedRequestV4(http.MethodPost, + getCompleteMultipartUploadURL("", bucket, object, uploadID), + int64(len(body)), bytes.NewReader(body), creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build CompleteMultipartUpload request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + return rec +} + +func copyPartWithoutChecksumHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials, + bucket, source, object, uploadID, sourceRange string, headers map[string]string, +) CopyObjectPartResponse { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, + getCopyObjectPartURL("", bucket, object, uploadID, "1"), + 0, nil, creds.AccessKey, creds.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build UploadPartCopy request: %v", err) + } + req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source)) + if sourceRange != "" { + req.Header.Set(xhttp.AmzCopySourceRange, sourceRange) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("UploadPartCopy failed: %d %s", rec.Code, rec.Body.String()) + } + var response CopyObjectPartResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("failed to decode UploadPartCopy response: %v", err) + } + return response +} + +// TestAPIUploadPartServerSideChecksum exercises the data transformations that +// made installing a checksum hasher in the object layer unsafe. The checksum +// must always cover logical plaintext, regardless of compression or encryption. +func TestAPIUploadPartServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIUploadPartServerSideChecksum, + []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"}) +} + +func testAPIUploadPartServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + typ := hash.ChecksumCRC32 + data := bytes.Repeat([]byte("multipart-checksum-plaintext-"), 48*1024) + want := mustChecksum(t, typ, data) + + t.Run("upload", func(t *testing.T) { + object := "checksums/upload" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + etag, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, nil) + if got := rec.Header().Get(typ.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want { + t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want) + } + + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + + oi, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err) + } + checksums, _ := oi.decryptChecksums(0, nil) + if got := checksums[typ.String()]; got != want { + t.Fatalf("%s: stored checksum %q, want plaintext checksum %q", instanceType, got, want) + } + }) + + t.Run("copy", func(t *testing.T) { + source := "checksums/source" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + object := "checksums/copy" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials, + bucketName, source, object, uploadID, "", nil) + if got := copyPartChecksum(typ, response); got != want { + t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want) + } + + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want { + t.Fatalf("%s: copied ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want) + } + + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(response.ETag)}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) +} + +func TestAPIUploadPartServerSideChecksumAlgorithms(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumAlgorithms, + endpoints: []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"}, + }) +} + +func testAPIUploadPartServerSideChecksumAlgorithms(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + tests := []struct { + typ hash.ChecksumType + objType string + composite bool + }{ + {hash.ChecksumCRC32, xhttp.AmzChecksumTypeFullObject, false}, + {hash.ChecksumCRC32C, xhttp.AmzChecksumTypeFullObject, false}, + {hash.ChecksumCRC64NVME, xhttp.AmzChecksumTypeFullObject, false}, + {hash.ChecksumCRC32, xhttp.AmzChecksumTypeComposite, true}, + {hash.ChecksumSHA1, xhttp.AmzChecksumTypeComposite, true}, + {hash.ChecksumSHA256, xhttp.AmzChecksumTypeComposite, true}, + } + data := bytes.Repeat([]byte("server-side-part-checksum"), 1024) + + for _, test := range tests { + t.Run(test.typ.String()+"/"+test.objType, func(t *testing.T) { + object := "algorithms/" + test.typ.String() + "/" + test.objType + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + test.typ.String(), test.objType) + etag, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, nil) + if got := rec.Header().Get(test.typ.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + want := mustChecksum(t, test.typ, data) + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(test.typ, listed.Parts[0]) != want { + t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want) + } + + part := CompletePart{PartNumber: 1, ETag: etag} + if test.composite { + part = completePartWithChecksum(test.typ, 1, etag, want) + } + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{part}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + } + + t.Run("multi-part/FULL_OBJECT", func(t *testing.T) { + typ := hash.ChecksumCRC32 + parts, full := multipartChecksumTestData() + object := "algorithms/multi-part-full-object" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + etags := make([]string, len(parts)) + for i, data := range parts { + etag, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, i+1, data, nil) + if got := rec.Header().Get(typ.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + etags[i] = etag + } + + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != len(parts) { + t.Fatalf("%s: ListParts returned %d parts, want %d", instanceType, len(listed.Parts), len(parts)) + } + for i, part := range listed.Parts { + if got, want := partChecksum(typ, part), mustChecksum(t, typ, parts[i]); got != want { + t.Fatalf("%s: part %d checksum %q, want %q", instanceType, i+1, got, want) + } + } + + complete := make([]CompletePart, len(etags)) + for i, etag := range etags { + complete[i] = CompletePart{PartNumber: i + 1, ETag: etag} + } + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, complete, + map[string]string{ + typ.Key(): mustChecksum(t, typ, full), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: multi-part CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("zero-length-part", func(t *testing.T) { + typ := hash.ChecksumCRC32 + object := "algorithms/zero-length" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + etag, _ := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, nil, nil) + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, nil) { + t.Fatalf("%s: zero-length ListParts checksum mismatch: %+v", instanceType, listed.Parts) + } + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: zero-length CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("overwrite-part-checksum", func(t *testing.T) { + typ := hash.ChecksumCRC32 + object := "algorithms/overwrite" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + first := []byte("first part contents") + second := []byte("replacement part contents") + uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, first, nil) + etag, _ := uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, second, nil) + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, second) { + t.Fatalf("%s: overwritten ListParts checksum mismatch: %+v", instanceType, listed.Parts) + } + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: overwritten CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("copy/SHA256/COMPOSITE", func(t *testing.T) { + typ := hash.ChecksumSHA256 + source := "algorithms/copy-source" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + object := "algorithms/copy-SHA256" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeComposite) + start, end := 7, len(data)-9 + response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials, + bucketName, source, object, uploadID, "bytes="+strconv.Itoa(start)+"-"+strconv.Itoa(end-1), nil) + want := mustChecksum(t, typ, data[start:end]) + if got := copyPartChecksum(typ, response); got != want { + t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want) + } + + part := completePartWithChecksum(typ, 1, canonicalizeETag(response.ETag), want) + rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, + []CompletePart{part}, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) +} + +func TestAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors, + endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"}, + }) +} + +func testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + data := []byte("client checksum must remain authoritative") + + t.Run("correct-value", func(t *testing.T) { + typ := hash.ChecksumCRC32 + object := "errors/correct-value" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + typ.String(), xhttp.AmzChecksumTypeFullObject) + want := mustChecksum(t, typ, data) + _, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, map[string]string{typ.Key(): want}) + if got := rec.Header().Get(typ.Key()); got != want { + t.Fatalf("%s: client checksum response %q, want %q", instanceType, got, want) + } + listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil) + if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want { + t.Fatalf("%s: client checksum ListParts mismatch: %+v", instanceType, listed.Parts) + } + }) + + t.Run("wrong-algorithm", func(t *testing.T) { + object := "errors/wrong-algorithm" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject) + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, object, uploadID, "1"), + int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, + map[string]string{hash.ChecksumSHA256.Key(): mustChecksum(t, hash.ChecksumSHA256, data)}) + if err != nil { + t.Fatalf("failed to build UploadPart request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" { + t.Fatalf("%s: wrong algorithm returned %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) + + t.Run("wrong-value", func(t *testing.T) { + object := "errors/wrong-value" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject) + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, object, uploadID, "1"), + int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, + map[string]string{hash.ChecksumCRC32.Key(): mustChecksum(t, hash.ChecksumCRC32, []byte("wrong"))}) + if err != nil { + t.Fatalf("failed to build UploadPart request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "XAmzContentChecksumMismatch" { + t.Fatalf("%s: wrong value returned %d %s", instanceType, rec.Code, rec.Body.String()) + } + }) +} + +func TestAPIUploadPartServerSideChecksumSSEC(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumSSEC, + endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"}, + }) +} + +func testAPIUploadPartServerSideChecksumSSEC(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + globalIsTLS = true + defer func() { globalIsTLS = false }() + + key := bytes.Repeat([]byte{0x2a}, 32) + keyMD5 := md5.Sum(key) + ssecHeaders := map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]), + } + initHeaders := map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm], + xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey], + xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5], + } + object := "checksums/ssec" + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object), + 0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders) + if err != nil { + t.Fatalf("failed to build NewMultipartUpload request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + var initiated InitiateMultipartUploadResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil { + t.Fatalf("failed to decode NewMultipartUpload response: %v", err) + } + + data := bytes.Repeat([]byte("ssec-checksum-plaintext"), 4096) + etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, initiated.UploadID, 1, data, ssecHeaders) + if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + completeHeaders := map[string]string{ + xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm], + xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey], + xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5], + } + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, completeHeaders) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } +} + +func TestAPIUploadPartServerSideChecksumSSES3(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIUploadPartServerSideChecksumSSES3, + endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"}, + }) +} + +func testAPIUploadPartServerSideChecksumSSES3(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=") + if err != nil { + t.Fatal(err) + } + GlobalKMS = KMS + defer func() { GlobalKMS = nil }() + + object := "checksums/sse-s3" + initHeaders := map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES, + } + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object), + 0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders) + if err != nil { + t.Fatalf("failed to build NewMultipartUpload request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + var initiated InitiateMultipartUploadResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil { + t.Fatalf("failed to decode NewMultipartUpload response: %v", err) + } + + data := bytes.Repeat([]byte("sse-s3-checksum-plaintext"), 4096) + etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, initiated.UploadID, 1, data, nil) + if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" { + t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got) + } + + rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID, + []CompletePart{{PartNumber: 1, ETag: etag}}, map[string]string{ + xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data), + xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject, + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } +} diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index f73b7da4f..cfee44df0 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -597,12 +597,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo onlineDisks := er.getDisks() writeQuorum := fi.WriteQuorum(er.defaultWQuorum()) - if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" { - if r.ContentCRCType().String() != cs { + expectedChecksumType, checksumEnabled := multipartChecksumType(fi.Metadata) + if checksumEnabled { + got := r.contentChecksumType() + if !expectedChecksumType.IsSet() || !got.IsSet() || got.Base() != expectedChecksumType { return pi, InvalidArgument{ Bucket: bucket, Object: fi.Name, - Err: fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()), + Err: fmt.Errorf("checksum missing, want %q, got %q", + fi.Metadata[hash.MinIOMultipartChecksum], got.String()), } } } @@ -725,6 +728,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo } } + partChecksums := r.contentChecksum() + if checksumEnabled && partChecksums[expectedChecksumType.String()] == "" { + err := fmt.Errorf("internal error: checksum missing after reading part, want %q", expectedChecksumType.String()) + bugLogIf(ctx, err) + return pi, toObjectErr(err, bucket, object, uploadID) + } + partInfo := ObjectPartInfo{ Number: partID, ETag: md5hex, @@ -732,7 +742,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo ActualSize: actualSize, ModTime: UTCNow(), Index: index, - Checksums: r.ContentCRC(), + Checksums: partChecksums, } partFI, err := partInfo.MarshalMsg(nil) diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index 5d791ce45..e3f016eef 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -1038,9 +1038,10 @@ type SealMD5CurrFn func([]byte) []byte // PutObjReader is a type that wraps sio.EncryptReader and // underlying hash.Reader in a struct type PutObjReader struct { - *hash.Reader // actual data stream - rawReader *hash.Reader // original data stream - sealMD5Fn SealMD5CurrFn + *hash.Reader // actual data stream + rawReader *hash.Reader // original data stream used for ETag calculation + checksumReader *hash.Reader // logical plaintext stream used for S3 checksum calculation + sealMD5Fn SealMD5CurrFn } // Size returns the absolute number of bytes the Reader @@ -1093,7 +1094,43 @@ func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto. // NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's // MD5Current method to construct md5sum when requested downstream. func NewPutObjReader(rawReader *hash.Reader) *PutObjReader { - return &PutObjReader{Reader: rawReader, rawReader: rawReader} + return &PutObjReader{Reader: rawReader, rawReader: rawReader, checksumReader: rawReader} +} + +// setChecksumReader sets the logical plaintext reader used for S3 checksums. +// It can differ from rawReader when the storage stream is compressed. +func (p *PutObjReader) setChecksumReader(r *hash.Reader) { + if r != nil { + p.checksumReader = r + } +} + +// contentChecksumType returns the effective client-provided or server-computed +// checksum type for the logical plaintext stream. +func (p *PutObjReader) contentChecksumType() hash.ChecksumType { + if p.checksumReader == nil { + return hash.ChecksumNone + } + if t := p.checksumReader.ContentCRCType(); t.IsSet() { + return t + } + return p.checksumReader.ServerSideChecksumType +} + +// contentChecksum returns the effective checksum for part metadata. A +// client-provided checksum takes precedence; server computation is only a +// fallback when the client omitted one. +func (p *PutObjReader) contentChecksum() map[string]string { + if p.checksumReader == nil { + return nil + } + if checksum := p.checksumReader.ContentCRC(); checksum != nil { + return checksum + } + if checksum := p.checksumReader.ServerSideChecksumResult; checksum != nil && checksum.Valid() { + return map[string]string{checksum.Type.String(): checksum.Encoded} + } + return nil } // RawServerSideChecksumResult returns the ServerSideChecksumResult from the diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index fcc6d5b64..03791f499 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -55,6 +55,48 @@ import ( // Multipart objectAPIHandlers +// multipartChecksumType returns the base checksum type recorded when a +// multipart upload was created. The boolean reports whether an algorithm was +// recorded at all. +func multipartChecksumType(metadata map[string]string) (hash.ChecksumType, bool) { + algorithm := metadata[hash.MinIOMultipartChecksum] + if algorithm == "" { + return hash.ChecksumNone, false + } + t := hash.NewChecksumType(algorithm, metadata[hash.MinIOMultipartChecksumType]) + if !t.IsSet() { + return t, true + } + return t.Base(), true +} + +// prepareMultipartChecksumReader validates a supplied part checksum algorithm, +// or installs a server-side hasher when the client omitted the optional +// checksum. It must run before compression or encryption can consume reader. +func prepareMultipartChecksumReader(reader *hash.Reader, metadata map[string]string, bucket, object string) error { + want, ok := multipartChecksumType(metadata) + if !ok { + return nil + } + + got := reader.ContentCRCType() + if !got.IsSet() && reader.ServerSideChecksumType.IsSet() { + got = reader.ServerSideChecksumType + } + if !want.IsSet() || (got.IsSet() && got.Base() != want) { + return InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("checksum missing, want %q, got %q", + metadata[hash.MinIOMultipartChecksum], got.String()), + } + } + if !got.IsSet() { + reader.AddServerSideChecksumHasher(want) + } + return nil +} + // NewMultipartUploadHandler - New multipart upload. // Notice: The S3 client can send secret keys in headers for encryption related jobs, // the handler should ensure to remove these keys before sending them to the object layer. @@ -465,7 +507,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt return } - response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified) + response := generateCopyObjectPartResponse(PartInfo{ + ETag: partInfo.ETag, + LastModified: partInfo.LastModified, + ChecksumCRC32: partInfo.ChecksumCRC32, + ChecksumCRC32C: partInfo.ChecksumCRC32C, + ChecksumSHA1: partInfo.ChecksumSHA1, + ChecksumSHA256: partInfo.ChecksumSHA256, + ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME, + }) encodedSuccessResponse := encodeResponse(response) // Write success response. @@ -475,12 +525,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt actualPartSize = length var reader io.Reader = etag.NewReader(ctx, gr, nil, nil) + var checksumReader *hash.Reader mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + if _, ok := multipartChecksumType(mi.UserDefined); ok { + checksumReader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if err = prepareMultipartChecksumReader(checksumReader, mi.UserDefined, dstBucket, dstObject); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + reader = checksumReader + } _, isEncrypted := crypto.IsEncrypted(mi.UserDefined) @@ -512,6 +575,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt rawReader := srcInfo.Reader pReader := NewPutObjReader(rawReader) + pReader.setChecksumReader(checksumReader) var objectEncryptionKey crypto.ObjectKey if isEncrypted { @@ -591,7 +655,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3) } - response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified) + response := generateCopyObjectPartResponse(partInfo) encodedSuccessResponse := encodeResponse(response) // Write success response. @@ -745,6 +809,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http // Read compression metadata preserved in the init multipart for the decision. _, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"] var idxCb func() []byte + var checksumReader *hash.Reader if isCompressed { actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize) if err != nil { @@ -755,6 +820,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } + if err = prepareMultipartChecksumReader(actualReader, mi.UserDefined, bucket, object); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + checksumReader = actualReader // Set compression metrics. wantEncryption := crypto.Requested(r.Header) @@ -791,8 +861,16 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) return } + if checksumReader == nil { + if err = prepareMultipartChecksumReader(hashReader, mi.UserDefined, bucket, object); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + checksumReader = hashReader + } pReader := NewPutObjReader(hashReader) + pReader.setChecksumReader(checksumReader) _, isEncrypted := crypto.IsEncrypted(mi.UserDefined) _, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus] From 6b0998157c24a9ad802e42c7a04333d2d402ddf1 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 13:43:44 +0800 Subject: [PATCH 06/13] fix: harden config environment file parsing Trim whitespace around assignments, preserve whitespace inside matching quotes, validate portable variable names, and report redacted file-and-line diagnostics. Check config-file Setenv failures instead of silently ignoring invalid entries.\n\nFixes #65 --- cmd/common-main.go | 56 ++++++++++++-- cmd/common-main_test.go | 161 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 8 deletions(-) diff --git a/cmd/common-main.go b/cmd/common-main.go index 18aad7346..7f6ead797 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -36,6 +36,7 @@ import ( "strings" "syscall" "time" + "unicode" "github.com/dustin/go-humanize" fcolor "github.com/fatih/color" @@ -540,6 +541,34 @@ func (e envKV) String() string { return fmt.Sprintf("%s=%s", e.Key, e.Value) } +func isValidEnvName(name string) bool { + if name == "" || !isEnvNameStart(name[0]) { + return false + } + for i := 1; i < len(name); i++ { + if !isEnvNameStart(name[i]) && (name[i] < '0' || name[i] > '9') { + return false + } + } + return true +} + +func isEnvNameStart(ch byte) bool { + return ch == '_' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z' +} + +func trimExportPrefix(envEntry string) string { + rest, ok := strings.CutPrefix(envEntry, "export") + if !ok || rest == "" { + return envEntry + } + trimmed := strings.TrimLeftFunc(rest, unicode.IsSpace) + if len(trimmed) == len(rest) { + return envEntry + } + return trimmed +} + func parsEnvEntry(envEntry string) (envKV, error) { envEntry = strings.TrimSpace(envEntry) if envEntry == "" { @@ -554,13 +583,19 @@ func parsEnvEntry(envEntry string) (envKV, error) { Skip: true, }, nil } - envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2) + envTokens := strings.SplitN(trimExportPrefix(envEntry), config.EnvSeparator, 2) if len(envTokens) != 2 { - return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry) + return envKV{}, errors.New("missing '='") } - key := envTokens[0] - val := envTokens[1] + key := strings.TrimSpace(envTokens[0]) + val := strings.TrimSpace(envTokens[1]) + if !isValidEnvName(key) { + return envKV{}, fmt.Errorf("invalid environment variable name %q", key) + } + if strings.IndexByte(val, 0) >= 0 { + return envKV{}, errors.New("environment variable value contains NUL") + } // Remove quotes from the value if found if len(val) >= 2 { @@ -587,10 +622,12 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) { defer f.Close() var ekvs []envKV scanner := bufio.NewScanner(f) + lineNo := 0 for scanner.Scan() { + lineNo++ ekv, err := parsEnvEntry(scanner.Text()) if err != nil { - return nil, err + return nil, fmt.Errorf("%s:%d: %w", envConfigFile, lineNo, err) } if ekv.Skip { // Skips empty lines @@ -599,7 +636,7 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) { ekvs = append(ekvs, ekv) } if err = scanner.Err(); err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", envConfigFile, err) } return ekvs, nil } @@ -666,12 +703,15 @@ func loadEnvVarsFromFiles() { } if env.IsSet(config.EnvConfigEnvFile) { - ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, "")) + envConfigFile := env.Get(config.EnvConfigEnvFile, "") + ekvs, err := minioEnvironFromFile(envConfigFile) if err != nil && !os.IsNotExist(err) { logger.Fatal(err, "Unable to read the config environment file") } for _, ekv := range ekvs { - os.Setenv(ekv.Key, ekv.Value) + if err := os.Setenv(ekv.Key, ekv.Value); err != nil { + logger.Fatal(err, "Unable to set %s from config environment file %s", ekv.Key, envConfigFile) + } } } } diff --git a/cmd/common-main_test.go b/cmd/common-main_test.go index 9757267d2..c13fa7a8e 100644 --- a/cmd/common-main_test.go +++ b/cmd/common-main_test.go @@ -19,8 +19,10 @@ package cmd import ( "errors" + "fmt" "os" "reflect" + "strings" "testing" ) @@ -181,3 +183,162 @@ MINIO_ROOT_PASSWORD=minio123`, }) } } + +func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) { + testCases := []struct { + name string + content string + want []envKV + errLine int + errContains string + errExcludes string + }{ + { + name: "spaces and tabs around separator", + content: "MINIO_ROOT_USER = minio\nMINIO_ROOT_PASSWORD\t=\tminio123", + want: []envKV{ + {Key: "MINIO_ROOT_USER", Value: "minio"}, + {Key: "MINIO_ROOT_PASSWORD", Value: "minio123"}, + }, + }, + { + name: "export tab and quoted spaces", + content: "export\tMINIO_ROOT_USER = \" minio user \"\nexport MINIO_ROOT_PASSWORD = ' minio secret '", + want: []envKV{ + {Key: "MINIO_ROOT_USER", Value: " minio user "}, + {Key: "MINIO_ROOT_PASSWORD", Value: " minio secret "}, + }, + }, + { + name: "export Unicode whitespace", + content: "export\u00a0MINIO_ROOT_USER=value", + want: []envKV{ + {Key: "MINIO_ROOT_USER", Value: "value"}, + }, + }, + { + name: "export is only a standalone prefix", + content: "export=value\nexportFOO=bar", + want: []envKV{ + {Key: "export", Value: "value"}, + {Key: "exportFOO", Value: "bar"}, + }, + }, + { + name: "unquoted whitespace empty value and additional separators", + content: "UNQUOTED = value \nEMPTY =\nTOKEN = scheme://user:password@example.com?a=b", + want: []envKV{ + {Key: "UNQUOTED", Value: "value"}, + {Key: "EMPTY", Value: ""}, + {Key: "TOKEN", Value: "scheme://user:password@example.com?a=b"}, + }, + }, + { + name: "valid underscore and digits", + content: "_MINIO_2=value", + want: []envKV{ + {Key: "_MINIO_2", Value: "value"}, + }, + }, + { + name: "missing separator redacts the line", + content: "MINIO_ROOT_PASSWORD=valid\nsuper-secret-without-equals", + errLine: 2, + errContains: "missing '='", + errExcludes: "super-secret-without-equals", + }, + { + name: "empty name", + content: "=empty-name-secret", + errLine: 1, + errContains: `invalid environment variable name ""`, + errExcludes: "empty-name-secret", + }, + { + name: "digit leading name", + content: "1MINIO_ROOT_USER=digit-leading-secret", + errLine: 1, + errContains: `invalid environment variable name "1MINIO_ROOT_USER"`, + errExcludes: "digit-leading-secret", + }, + { + name: "hyphenated name", + content: "MINIO-ROOT-USER=hyphen-secret", + errLine: 1, + errContains: `invalid environment variable name "MINIO-ROOT-USER"`, + errExcludes: "hyphen-secret", + }, + { + name: "whitespace in name", + content: "MINIO ROOT USER=whitespace-secret", + errLine: 1, + errContains: `invalid environment variable name "MINIO ROOT USER"`, + errExcludes: "whitespace-secret", + }, + { + name: "NUL in name", + content: "MINIO\x00ROOT=nul-name-secret", + errLine: 1, + errContains: "invalid environment variable name", + errExcludes: "nul-name-secret", + }, + { + name: "NUL in value", + content: "MINIO_ROOT_USER=before\x00nul-value-secret", + errLine: 1, + errContains: "environment variable value contains NUL", + errExcludes: "nul-value-secret", + }, + { + name: "diagnostic has file and line but no value", + content: "MINIO_ROOT_USER=valid\nBAD KEY=super-secret-value", + errLine: 2, + errContains: `invalid environment variable name "BAD KEY"`, + errExcludes: "super-secret-value", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + tmpfile, err := os.CreateTemp(t.TempDir(), "testfile") + if err != nil { + t.Fatal(err) + } + if _, err = tmpfile.WriteString(testCase.content); err != nil { + t.Fatal(err) + } + if err = tmpfile.Close(); err != nil { + t.Fatal(err) + } + + got, err := minioEnvironFromFile(tmpfile.Name()) + if testCase.errContains == "" { + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, testCase.want) { + t.Errorf("expected %v, got %v", testCase.want, got) + } + return + } + + if err == nil { + t.Fatal("expected an error") + } + errText := err.Error() + location := fmt.Sprintf("%s:%d:", tmpfile.Name(), testCase.errLine) + if !strings.Contains(errText, location) { + t.Errorf("expected error to contain %q, got %q", location, errText) + } + if !strings.Contains(errText, testCase.errContains) { + t.Errorf("expected error to contain %q, got %q", testCase.errContains, errText) + } + if testCase.errExcludes != "" && strings.Contains(errText, testCase.errExcludes) { + t.Errorf("expected error to redact %q, got %q", testCase.errExcludes, errText) + } + if got != nil { + t.Errorf("expected no entries on parse error, got %v", got) + } + }) + } +} From 45eb2e423d297ad4acae79800604be2a2bba92b3 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 15:03:45 +0800 Subject: [PATCH 07/13] fix(ci): align compatibility fixtures and generated credits --- CREDITS | 41 +++++++++++------------------------------ cmd/common-main_test.go | 4 ++-- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/CREDITS b/CREDITS index 110776ccd..4dc7f36c9 100644 --- a/CREDITS +++ b/CREDITS @@ -16641,33 +16641,6 @@ SOFTWARE. ================================================================ -github.com/lestrrat-go/option -https://github.com/lestrrat-go/option ----------------------------------------------------------------- -MIT License - -Copyright (c) 2021 lestrrat-go - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -================================================================ - github.com/lestrrat-go/option/v2 https://github.com/lestrrat-go/option/v2 ---------------------------------------------------------------- @@ -21713,6 +21686,14 @@ separate copyright notices and license terms. Your use of the source code for the these subcomponents is subject to the terms and conditions of the following licenses. +=== + +This distribution (pgsty/mc, shipped as "mcli") is a community-maintained +fork of the MinIO Client, modified by the Silo project +(https://silo.pgsty.com/). It is not affiliated with, endorsed by, or +sponsored by MinIO, Inc. Modifications are Copyright (c) 2025-2026 PGSTY +and are licensed under the GNU AGPL v3.0 or later. + ================================================================ github.com/minio/md5-simd @@ -28982,7 +28963,7 @@ https://go.etcd.io/etcd/api/v3 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29190,7 +29171,7 @@ https://go.etcd.io/etcd/client/pkg/v3 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29398,7 +29379,7 @@ https://go.etcd.io/etcd/client/v3 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 The etcd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cmd/common-main_test.go b/cmd/common-main_test.go index c13fa7a8e..341ddba49 100644 --- a/cmd/common-main_test.go +++ b/cmd/common-main_test.go @@ -235,9 +235,9 @@ func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) { }, { name: "valid underscore and digits", - content: "_MINIO_2=value", + content: "_VALID_2=value", want: []envKV{ - {Key: "_MINIO_2", Value: "value"}, + {Key: "_VALID_2", Value: "value"}, }, }, { From eee05a17c34a07cebb27220d12697be74c8bd617 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 17:59:13 +0800 Subject: [PATCH 08/13] Reject bare ARN policies on admin writes Use silo-pkg v3.12 strict validation when creating named policies and when creating or updating service-account session policies. Keep stored policy loads, IAM import, site replication, and STS inline policies on the permissive compatibility path. --- cmd/admin-handlers-users.go | 6 ++-- cmd/admin-handlers-users_test.go | 60 ++++++++++++++++++++++++++++++++ go.mod | 9 +++-- go.sum | 4 +-- 4 files changed, 69 insertions(+), 10 deletions(-) diff --git a/cmd/admin-handlers-users.go b/cmd/admin-handlers-users.go index 8530046b9..eb032672d 100644 --- a/cmd/admin-handlers-users.go +++ b/cmd/admin-handlers-users.go @@ -859,7 +859,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re var sp *policy.Policy if len(updateReq.NewPolicy) > 0 { - sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy)) + sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy)) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -1729,7 +1729,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request return } - iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes)) + iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes)) if err != nil { writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) return @@ -2981,7 +2981,7 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth. var sp *policy.Policy if len(createReq.Policy) > 0 { - sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy)) + sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy)) if err != nil { return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err) } diff --git a/cmd/admin-handlers-users_test.go b/cmd/admin-handlers-users_test.go index 828264583..350edf823 100644 --- a/cmd/admin-handlers-users_test.go +++ b/cmd/admin-handlers-users_test.go @@ -204,6 +204,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) { suite.TestUserCreate(c) suite.TestUserPolicyEscalationBug(c) suite.TestPolicyCreate(c) + suite.TestServiceAccountBareARNPolicyRejected(c) suite.TestCannedPolicies(c) suite.TestGroupAddRemove(c) suite.TestServiceAccountOpsByAdmin(c) @@ -600,6 +601,20 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) { c.Fatalf("invalid policy creation success") } + for i, resource := range []string{"arn:aws:s3:::", "*arn:aws:s3:::"} { + barePolicyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Deny", + "Action": ["s3:GetObject"], + "Resource": ["%s"] + }] +}`, resource) + if err = s.adm.AddCannedPolicy(ctx, fmt.Sprintf("%s-bare-%d", policy, i), barePolicyBytes); err == nil { + c.Fatalf("bare ARN policy creation succeeded for %q", resource) + } + } + // 3. Create a user, associate policy and verify access accessKey, secretKey := mustGenerateCredentials(c) err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) @@ -653,6 +668,51 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) { } } +func (s *TestSuiteIAM) TestServiceAccountBareARNPolicyRejected(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + barePolicy := []byte(`{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:GetObject"], + "NotResource": ["arn:aws:s3:::"] + }] +}`) + if _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: globalActiveCred.AccessKey, + Policy: barePolicy, + }); err == nil { + c.Fatal("service account creation accepted a bare ARN policy") + } + + validPolicy := []byte(`{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": ["arn:aws:s3:::*"] + }] +}`) + credentials, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: globalActiveCred.AccessKey, + Policy: validPolicy, + }) + if err != nil { + c.Fatalf("service account creation rejected an explicit resource: %v", err) + } + defer func() { + _ = s.adm.DeleteServiceAccount(ctx, credentials.AccessKey) + }() + + if err = s.adm.UpdateServiceAccount(ctx, credentials.AccessKey, madmin.UpdateServiceAccountReq{ + NewPolicy: barePolicy, + }); err == nil { + c.Fatal("service account update accepted a bare ARN policy") + } +} + func (s *TestSuiteIAM) TestCannedPolicies(c *check) { ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) defer cancel() diff --git a/go.mod b/go.mod index 2e027f488..284a827b6 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,9 @@ replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260806055018-b0021fd // Use Pigsty's maintained SILO package fork while preserving upstream import paths. // This retains the LDAP TLS fix tracked in https://github.com/pgsty/silo/issues/15. -// v3.11.0 follows upstream minio/pkg's 3.11 line and carries the -// minio/minio#20449 bucket-write boundary hardening. -replace github.com/minio/pkg/v3 => github.com/pgsty/silo-pkg/v3 v3.11.0 +// v3.12.0 carries the minio/minio#20449 bucket-write boundary hardening and +// rejects bare ARN prefixes on strict policy-write paths. +replace github.com/minio/pkg/v3 => github.com/pgsty/silo-pkg/v3 v3.12.0 // v22.7.0 does not compile on NetBSD because its unix implementation uses // CLOCK_MONOTONIC, which is unavailable there. Keep the last portable release @@ -77,7 +77,7 @@ require ( github.com/minio/madmin-go/v3 v3.0.110 github.com/minio/minio-go/v7 v7.0.99 github.com/minio/mux v1.9.2 - github.com/minio/pkg/v3 v3.11.0 + github.com/minio/pkg/v3 v3.12.0 github.com/minio/selfupdate v0.6.0 github.com/minio/simdjson-go v0.4.5 github.com/minio/sio v0.4.3 @@ -210,7 +210,6 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect github.com/googleapis/gax-go/v2 v2.22.0 // indirect - github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect diff --git a/go.sum b/go.sum index 82c73ff4e..72ba8c3d9 100644 --- a/go.sum +++ b/go.sum @@ -549,8 +549,8 @@ github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb h1:ixZKprZQpUIVrUlOy6H7oL github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb/go.mod h1:cTbS+9jGR4Qs7xTf5DEhmCTbzcDWrKMs8ZmTUnCU49E= github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a h1:JfEQJkBdTwCXSrCv8P0R5tMpzxxhSdUTvg54mHwzxpA= github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a/go.mod h1:7J8wCQsNT5S7GqCHnQqgj0T7Nagp1fklJhfBJI+v0XI= -github.com/pgsty/silo-pkg/v3 v3.11.0 h1:wjN5d+tWD8Twq+e7k/KBBVhnWXC8xTIlfTcnGIKkmjc= -github.com/pgsty/silo-pkg/v3 v3.11.0/go.mod h1:E2AB4oOgfDeb9In1KDBTrn9wzfvr0WzoPkbXW7wbwBQ= +github.com/pgsty/silo-pkg/v3 v3.12.0 h1:1Bjqjb3KCt0oYhBLpH7W/e/5khTUoIgXWA12An1fbUc= +github.com/pgsty/silo-pkg/v3 v3.12.0/go.mod h1:ohHtpAK7kBCffdMT4oqUbtkoZMHxA7znTCPNjrYw86I= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg= From 56c67dacf13cb3c5c1c59e64afec77ab145fb7f4 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 18:01:22 +0800 Subject: [PATCH 09/13] Document bare ARN policy hardening Record the new strict named-policy and service-account write behavior, the additional admin-policy validation it activates, the compatibility boundaries retained for stored/imported/replicated policies, and the required manual rewrite for legacy bare ARN prefixes. --- docs/security/advisories.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/security/advisories.md b/docs/security/advisories.md index 6860ae33e..6aa2df3f9 100644 --- a/docs/security/advisories.md +++ b/docs/security/advisories.md @@ -19,6 +19,7 @@ Entries carry a CVE identifier where one exists. Where none does, they carry a f | `SN-2026-003` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`2f55347f7`](https://github.com/pgsty/minio/commit/2f55347f78352aed8e08866d370c9426c73362cf) | S3/IAM bucket-policy condition values | Yes (policy-dependent) | Prevents raw request entries that spell condition-key names from shadowing or synthesizing internal condition values; confines `s3:signatureAge` to verified SigV4 presigned requests; separates query-only list fields from header-backed `x-amz-*` fields; and stops client request tags from impersonating stored existing-object tags. | The compatible query form remains for storage class and upload tagging on handlers that consume it; an explicitly present header wins, including an empty header. The historical `X-Amz-Tagging` Header mapping remains a client-supplied `RequestObjectTag` source, so use request-tag conditions only on operations that consume tags. Header-only `x-amz-*` policy keys no longer accept query substitutes. `aws:SourceIp` was left following the existing forwarding-header trust model; that model is addressed separately in the next row. See [Condition value sources and precedence](https://silo.pgsty.com/administration/identity-access-management/policy-based-access-control/#condition-value-sources). | | Not a vulnerability | `fe6dc4780` | Client source address (`aws:SourceIp`, audit `remotehost`, event notification `Host`) | N/A -- opt-in hardening | Adds an enforceable forwarded-header trust boundary, `MINIO_API_TRUSTED_PROXIES`. Set to a list of addresses or CIDR blocks, forwarded headers are believed only from those peers and forwarding chains are read right-to-left past listed hops -- which also stops the client-supplied left-most entry that an appending proxy (the stock nginx `$proxy_add_x_forwarded_for` recipe, or HAProxy's added second header line) leaves in place. Set to `none`, no forwarded header is believed at all. This is the guarantee `_MINIO_API_XFF_HEADER=off` never provided: it suppresses `X-Forwarded-For` alone, so `X-Real-IP` and RFC 7239 `Forwarded` remain one-line substitutions for anyone that setting was meant to stop. | **No behaviour change for any existing deployment**, so there is nothing to do on upgrade unless you want the new boundary. Not assigned a CVE: the default matches upstream, and upstream's own position (maintainer response in [discussion #17878](https://github.com/minio/minio/discussions/17878), Aug 2023) is that IP-based restrictions are impractical without reliable source-IP visibility. The gap being closed is that this was never written anywhere an operator would find it -- an `IpAddress` condition is accepted and behaves as though it works. **If you use `IpAddress` or `NotIpAddress` conditions, note that they were not enforceable before this change**, including behind a reverse proxy whose `X-Forwarded-For` recipe appends rather than overwrites. If you do not, the change affects only the accuracy of client addresses in logs. The new variable is opt-in and inert when unset; `_MINIO_API_XFF_HEADER` keeps its exact upstream semantics, and upstream's `TestXFFDisabled` is retained unmodified as the proof. An `IpAddress` condition remains unenforceable by default against a client with direct network access to the API port -- that is the condition the allowlist exists to fix, not a regression introduced here. When enabling the allowlist: it must name proxies, not the subnet they sit in, because entries are skipped while walking the chain, so a range that also covers clients lets those clients forge. Multi-node deployments must include their own node addresses, since MinIO forwards some requests between nodes and a client can force a hop through the `ListObjectsV2` continuation token; prefer the allowlist over `none` on a cluster for that reason. Loopback is always trusted as a peer so FTP and SFTP keep attributing their sessions. A malformed value stops startup, as does one that names no proxy at all (`","`) or one whose `env://` remote could not be read -- `env.Get` discards that error and yields an empty string, which would otherwise read as unset. Whitespace-only remains equivalent to unset. The policy is read after `MINIO_CONFIG_ENV_FILE` is loaded so environment-file deployments are covered; `_MINIO_API_XFF_HEADER` deliberately keeps upstream's earlier read timing, where a value written into an environment file is ignored. `MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES` now shares the same list parser, but is behaviourally untouched: the extraction is pure code motion, verified identical to the previous implementation across every combination of 37 allowlist values and 21 peer addresses. See [Client source address trust](source-address-trust.md). | | `SN-2026-004` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`97b7d2804`](https://github.com/pgsty/minio/commit/97b7d28040d109061c0a46a4c01bfc7800a97cc1) | IAM policy evaluation of bucket-level actions | Yes (policy-dependent) | Withholds twelve sensitive bucket-level writes from an object-only resource pattern. The IAM matcher appended a trailing slash for bucket-level requests (empty object name), so a resource of `arn:aws:s3:::bucket/*` matched `"bucket/"` and authorized bucket-level actions it was never meant to reach -- upstream [minio/minio#20449](https://github.com/minio/minio/issues/20449). The bucket-policy evaluation path never had the slash and was already reference-correct. | **This is an authorization tightening; read this row before upgrading if you write your own bucket-scoped policies.** Withheld from `bucket/*` on `Allow` statements only: `PutBucketPolicy`, `DeleteBucketPolicy`, `PutBucketObjectLockConfiguration`, `PutBucketVersioning`, `PutReplicationConfiguration`, `PutBucketLifecycle`, `DeleteBucket`, `ForceDeleteBucket`, `PutBucketCors`, `DeleteBucketCors`, `PutBucketQOS`, `PutInventoryConfiguration`. Membership was decided by one question -- does reaching this action give the caller something its object-scoped grant does not already give it? -- because the bug only fires when the statement already grants the bucket action, which in practice means `s3:*`, so the affected principal already holds full object CRUD. Only actions that hand out access to others, defeat a protection aimed at write-holders, act under server credentials, outlive the grant, or destroy the bucket entity qualify. **Deliberately not withheld, and asserted by test so re-adding one is a deliberate act**: `ListBucket`, `GetBucketLocation` and the read/list family, `PutBucketTagging`, `PutBucketEncryption`, `PutBucketNotification`, and `CreateBucket` -- so `mc ls`, SDK session setup and ordinary tenant self-service keep working through `bucket/*`. Breaking those is what got upstream's own full fix reverted. **What to change**: add the bare bucket ARN (`arn:aws:s3:::bucket`) alongside `arn:aws:s3:::bucket/*` in any statement that legitimately grants one of the twelve. Built-in canned policies are unaffected (all use `Resource: "*"`). `Deny` statements are untouched, so no bucket lock is ever weakened, and `NotResource` exclusions keep their full reach. The hardening is monotone by construction rather than by argument: the protected path requires **both** the bare and the historical `"bucket/"` form to match, an intersection with the historical decision -- without that, a fixed-width wildcard such as `mybucke?` would match `"mybucket"` while never having matched `"mybucket/"`, and the hardening would have granted a write the buggy matcher refused. `MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH=on` restores the historical behaviour in full; it is read once at startup. Still deferred to a migration-gated release: the read/list family, a startup audit naming affected policies, and a self-explaining denial log. | +| `SN-2026-005` | [`silo-pkg v3.12.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.12.0) and [`eee05a17c`](https://github.com/pgsty/silo/commit/eee05a17c34a07cebb27220d12697be74c8bd617) | IAM named-policy and service-account policy writes | No direct remote exploit; policy-dependent | Rejects S3, S3 Tables, and KMS ARN namespace prefixes that name no resource, including their historical `*arn:...` serialization, in both `Resource` and `NotResource`. A resource-matching `Deny` using such a prefix could silently fail to deny, while an `Allow` with the prefix in `NotResource` could match far more broadly than intended. The guard applies when creating named policies and when creating or updating service-account session policies. | **This is an authorization tightening for new and updated policies.** Existing policies keep loading, matching, importing, and replicating with unchanged runtime behavior, but a policy containing one of these prefixes cannot be submitted unchanged; replace it with the intended concrete resource, or use an explicit wildcard such as `arn:aws:s3:::*` only when all resources are intended. Enabling the strict path also rejects an admin statement that combines `Resource` with `NotResource`, and rejects non-S3 resources on bucket-scoped admin actions. Here “bare ARN prefix” means a namespace with no resource after it (`arn:aws:s3:::`); it is distinct from the valid “bare bucket ARN” in `SN-2026-004` (`arn:aws:s3:::bucket`). IAM import, site-replication receive paths, stored-policy loading, and STS inline policies remain on the permissive compatibility path in this release. | ## Dependency security updates From c0e7159771af022fd5b2416195d276c21da6b52e Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 19:51:27 +0800 Subject: [PATCH 10/13] fix: checksum CopyObject data before compression Signed-off-by: Feng Ruohang --- cmd/erasure-object.go | 14 +- cmd/object-api-utils.go | 8 +- cmd/object-copy-checksum_test.go | 519 +++++++++++++++++++++++++++++++ cmd/object-handlers.go | 23 +- 4 files changed, 549 insertions(+), 15 deletions(-) create mode 100644 cmd/object-copy-checksum_test.go diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 0247822e7..2364c45b9 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1485,11 +1485,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st // over opts.WantChecksum. if opts.WantServerSideChecksumType.IsSet() { serverSideChecksum := r.RawServerSideChecksumResult() - if serverSideChecksum != nil { - fi.Checksum = serverSideChecksum.AppendTo(nil, nil) - if opts.EncryptFn != nil { - fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) - } + if serverSideChecksum == nil || !serverSideChecksum.Valid() || + serverSideChecksum.Type.Base() != opts.WantServerSideChecksumType.Base() { + err := fmt.Errorf("internal error: server-side checksum missing, invalid, or mismatched after reading object, want %q", opts.WantServerSideChecksumType.String()) + bugLogIf(ctx, err) + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + fi.Checksum = serverSideChecksum.AppendTo(nil, nil) + if opts.EncryptFn != nil { + fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) } } else if fi.Checksum == nil && opts.WantChecksum != nil { // Trailing headers checksums should now be filled. diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index e3f016eef..5aefb39b9 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -1134,11 +1134,11 @@ func (p *PutObjReader) contentChecksum() map[string]string { } // RawServerSideChecksumResult returns the ServerSideChecksumResult from the -// underlying rawReader, since the PutObjReader might be encrypted data and -// thus any checksum from that would be incorrect. +// logical plaintext checksum reader, since the PutObjReader might contain +// compressed or encrypted data and thus any checksum from that would be incorrect. func (p *PutObjReader) RawServerSideChecksumResult() *hash.Checksum { - if p.rawReader != nil { - return p.rawReader.ServerSideChecksumResult + if p.checksumReader != nil { + return p.checksumReader.ServerSideChecksumResult } return nil } diff --git a/cmd/object-copy-checksum_test.go b/cmd/object-copy-checksum_test.go new file mode 100644 index 000000000..29f153c84 --- /dev/null +++ b/cmd/object-copy-checksum_test.go @@ -0,0 +1,519 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +func setCopyChecksumCompression(allowEncrypted bool) func() { + globalCompressConfigMu.Lock() + previous := globalCompressConfig + globalCompressConfig.Enabled = true + globalCompressConfig.Extensions = []string{".txt"} + globalCompressConfig.MimeTypes = nil + globalCompressConfig.AllowEncrypted = allowEncrypted + globalCompressConfigMu.Unlock() + + return func() { + globalCompressConfigMu.Lock() + globalCompressConfig = previous + globalCompressConfigMu.Unlock() + } +} + +func copyChecksumRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials, + bucket, source, destination string, headers map[string]string, +) *httptest.ResponseRecorder { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucket, destination), + 0, nil, credentials.AccessKey, credentials.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build CopyObject request: %v", err) + } + req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source)) + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + return rec +} + +func putCopyChecksumSource(t *testing.T, apiRouter http.Handler, credentials auth.Credentials, + bucket, object string, data []byte, headers map[string]string, +) { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucket, object), + int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build PutObject request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("PutObject(%s) failed: %d %s", object, rec.Code, rec.Body.String()) + } +} + +func readCopyChecksumObject(t *testing.T, obj ObjectLayer, bucket, object string, opts ObjectOptions) []byte { + t.Helper() + gr, err := obj.GetObjectNInfo(t.Context(), bucket, object, nil, nil, opts) + if err != nil { + t.Fatalf("GetObjectNInfo(%s) failed: %v", object, err) + } + defer gr.Close() + data, err := io.ReadAll(gr) + if err != nil { + t.Fatalf("reading %s failed: %v", object, err) + } + return data +} + +func assertCopyChecksum(t *testing.T, obj ObjectLayer, bucket, object string, typ hash.ChecksumType, + data []byte, compressed bool, decryptHeaders http.Header, +) ObjectInfo { + t.Helper() + oi, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}) + if err != nil { + t.Fatalf("GetObjectInfo(%s) failed: %v", object, err) + } + if oi.IsCompressed() != compressed { + t.Fatalf("%s compressed=%v, want %v", object, oi.IsCompressed(), compressed) + } + checksums, _ := oi.decryptChecksums(0, decryptHeaders) + if got, want := checksums[typ.String()], mustChecksum(t, typ, data); got != want { + t.Fatalf("%s stored %s checksum %q, want logical object checksum %q (all: %v)", + object, typ.String(), got, want, checksums) + } + if got := checksums[xhttp.AmzChecksumType]; got != xhttp.AmzChecksumTypeFullObject { + t.Fatalf("%s checksum type %q, want %q", object, got, xhttp.AmzChecksumTypeFullObject) + } + return oi +} + +// TestAPICopyObjectServerSideChecksum verifies that server-computed checksums +// cover the logical object, never the compressed storage stream. +func TestAPICopyObjectServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + for _, versioned := range []bool{false, true} { + name := "unversioned" + if versioned { + name = "versioned" + } + t.Run(name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksum, + endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"}, + makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned}, + }) + }) + } +} + +func testAPICopyObjectServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("copy-object-checksum-plaintext-"), 64*1024) + source := "copy-checksum/source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + + compressedReader, _ := newS2CompressReader(bytes.NewReader(data), int64(len(data)), false) + compressed, err := io.ReadAll(compressedReader) + if closeErr := compressedReader.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatalf("%s: independently compressing test data failed: %v", instanceType, err) + } + + cases := []struct { + name string + typ hash.ChecksumType + explicit bool + extension string + compressed bool + }{ + {name: "compressed/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/CRC32C", typ: hash.ChecksumCRC32C, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/SHA1", typ: hash.ChecksumSHA1, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/SHA256", typ: hash.ChecksumSHA256, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/CRC64NVME", typ: hash.ChecksumCRC64NVME, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/default", typ: hash.ChecksumCRC64NVME, extension: ".txt", compressed: true}, + {name: "plain/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".bin"}, + {name: "plain/default", typ: hash.ChecksumCRC64NVME, extension: ".bin"}, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + headers := map[string]string(nil) + if tc.explicit { + headers = map[string]string{xhttp.AmzChecksumAlgo: tc.typ.String()} + } + destination := "copy-checksum/destination-" + tc.typ.String() + "-" + string(rune('a'+i)) + tc.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + + info := assertCopyChecksum(t, obj, bucketName, destination, tc.typ, data, tc.compressed, nil) + md5sum := md5.Sum(data) + if got, want := info.ETag, hex.EncodeToString(md5sum[:]); got != want { + t.Fatalf("%s: ETag %q, want logical object MD5 %q", instanceType, got, want) + } + if tc.compressed { + logical := mustChecksum(t, tc.typ, data) + if transformed := mustChecksum(t, tc.typ, compressed); logical == transformed { + t.Fatalf("%s: test payload does not distinguish logical and compressed checksum domains", instanceType) + } + } + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: round-trip body differs for %s", instanceType, tc.name) + } + + if tc.name == "compressed/CRC32" { + for _, method := range []string{http.MethodHead, http.MethodGet} { + url := getHeadObjectURL("", bucketName, destination) + if method == http.MethodGet { + url = getGetObjectURL("", bucketName, destination) + } + req, err := newTestSignedRequestV4(method, url, 0, nil, + credentials.AccessKey, credentials.SecretKey, + map[string]string{xhttp.AmzChecksumMode: "ENABLED"}) + if err != nil { + t.Fatalf("failed to build %s request: %v", method, err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK { + t.Fatalf("%s returned %d: %s", method, response.Code, response.Body.String()) + } + if got, want := response.Header().Get(tc.typ.Key()), mustChecksum(t, tc.typ, data); got != want { + t.Fatalf("%s returned checksum %q, want %q", method, got, want) + } + if method == http.MethodGet && !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("GET response body differs") + } + } + } + }) + } +} + +func TestAPICopyObjectServerSideChecksumEncryption(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksumEncryption, + endpoints: []string{"CopyObject", "PutObject", "GetObject"}, + }) +} + +func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("encrypted-copy-checksum-plaintext-"), 48*1024) + source := "copy-checksum/encrypted-source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + + t.Run("SSE-S3", func(t *testing.T) { + secretKey, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=") + if err != nil { + t.Fatal(err) + } + previousKMS := GlobalKMS + GlobalKMS = secretKey + defer func() { GlobalKMS = previousKMS }() + + for _, variant := range []struct { + name string + extension string + compressed bool + }{ + {name: "encrypted-only", extension: ".bin"}, + {name: "compressed-encrypted", extension: ".txt", compressed: true}, + } { + t.Run(variant.name, func(t *testing.T) { + destination := "copy-checksum/sse-s3-" + variant.name + variant.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES, + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-S3 CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: SSE-S3 round-trip body differs", instanceType) + } + }) + } + + encryptedSource := "copy-checksum/sse-s3-source.bin" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, encryptedSource, data, + map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}) + destination := "copy-checksum/sse-s3-source-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, encryptedSource, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-S3 source CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: SSE-S3 source round-trip body differs", instanceType) + } + }) + + t.Run("SSE-C", func(t *testing.T) { + previousTLS := globalIsTLS + globalIsTLS = true + defer func() { globalIsTLS = previousTLS }() + + key := bytes.Repeat([]byte{0x2a}, 32) + keyMD5 := md5.Sum(key) + headers := map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]), + } + decryptHeaders := http.Header{} + for key, value := range headers { + decryptHeaders.Set(key, value) + } + + getHeaders := make(map[string]string, len(headers)) + for key, value := range headers { + if key != xhttp.AmzChecksumAlgo { + getHeaders[key] = value + } + } + for _, variant := range []struct { + name string + extension string + compressed bool + }{ + {name: "encrypted-only", extension: ".bin"}, + {name: "compressed-encrypted", extension: ".txt", compressed: true}, + } { + t.Run(variant.name, func(t *testing.T) { + destination := "copy-checksum/sse-c-" + variant.name + variant.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-C CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, decryptHeaders) + + req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, destination), + 0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders) + if err != nil { + t.Fatalf("failed to build SSE-C GetObject request: %v", err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("%s: SSE-C GetObject returned %d with %d bytes, want 200 with %d bytes", + instanceType, response.Code, response.Body.Len(), len(data)) + } + }) + } + }) +} + +func TestAPICopyObjectServerSideChecksumSourceVariants(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksumSourceVariants, + endpoints: []string{ + "NewMultipart", "PutObjectPart", "CompleteMultipart", "ListObjectParts", + "CopyObject", "PutObject", "HeadObject", "GetObject", + }, + }) +} + +func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("source-variant-plaintext-"), 64*1024) + + t.Run("compressed-source", func(t *testing.T) { + source := "copy-checksum/compressed-source.txt" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, nil) + if info, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{}); err != nil || !info.IsCompressed() { + t.Fatalf("%s: compressed source precondition failed: compressed=%v err=%v", instanceType, info.IsCompressed(), err) + } + destination := "copy-checksum/compressed-source-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, source, map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumSHA256.String(), + xhttp.AmzMetadataDirective: "REPLACE", + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: in-place CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, source, hash.ChecksumSHA256, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, source, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: in-place CopyObject body differs", instanceType) + } + }) + + t.Run("full-checksum-source", func(t *testing.T) { + source := "copy-checksum/full-checksum-source.bin" + want := mustChecksum(t, hash.ChecksumCRC32, data) + putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, + map[string]string{xhttp.AmzChecksumCRC32: want}) + destination := "copy-checksum/full-checksum-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + }) + + t.Run("multipart-composite-source", func(t *testing.T) { + typ := hash.ChecksumCRC32 + parts, full := multipartChecksumTestData() + source := "copy-checksum/multipart-source.bin" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source, + typ.String(), xhttp.AmzChecksumTypeComposite) + etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, source, uploadID, typ, parts) + partChecksums := make([]string, len(parts)) + for i, part := range parts { + partChecksums[i] = mustChecksum(t, typ, part) + } + rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source, uploadID, + etags, partChecksums, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + sourceInfo, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{}) + if err != nil { + t.Fatalf("%s: source GetObjectInfo failed: %v", instanceType, err) + } + if _, multipart := sourceInfo.decryptChecksums(0, nil); !multipart { + t.Fatalf("%s: source checksum is not multipart composite", instanceType) + } + + destination := "copy-checksum/multipart-copy.txt" + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, typ, full, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, full) { + t.Fatalf("%s: multipart source round-trip body differs", instanceType) + } + }) + + for _, boundary := range []struct { + name string + data []byte + compressed bool + }{ + {name: "at-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize)}, + {name: "over-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize+1), compressed: true}, + {name: "indexed", data: bytes.Repeat([]byte{'a'}, compMinIndexSize+1), compressed: true}, + {name: "empty", data: nil}, + } { + t.Run(boundary.name, func(t *testing.T) { + source := "copy-checksum/" + boundary.name + "-source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(boundary.data), int64(len(boundary.data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + destination := "copy-checksum/" + boundary.name + "-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, boundary.data, boundary.compressed, nil) + }) + } +} + +func TestPutObjectRejectsMissingServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testPutObjectRejectsMissingServerSideChecksum, + endpoints: []string{"PutObject"}, + }) +} + +func testPutObjectRejectsMissingServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, + _ http.Handler, _ auth.Credentials, t *testing.T, +) { + data := []byte("the object layer must not silently omit a requested checksum") + for _, test := range []struct { + name string + hasherType hash.ChecksumType + }{ + {name: "missing"}, + {name: "mismatched", hasherType: hash.ChecksumCRC32C}, + } { + t.Run(test.name, func(t *testing.T) { + object := "copy-checksum/" + test.name + "-server-side-checksum" + reader := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "") + if test.hasherType.IsSet() { + reader.AddServerSideChecksumHasher(test.hasherType) + } + _, err := obj.PutObject(t.Context(), bucketName, object, reader, + ObjectOptions{WantServerSideChecksumType: hash.ChecksumCRC32}) + if err == nil || !strings.Contains(err.Error(), "server-side checksum") { + t.Fatalf("%s: PutObject error %v, want server-side checksum invariant error", instanceType, err) + } + if _, err = obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}); !isErrObjectNotFound(err) { + t.Fatalf("%s: failed PutObject left an object behind: %v", instanceType, err) + } + }) + } +} diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 296817ed5..18065def8 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1386,12 +1386,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10) reader = etag.NewReader(ctx, reader, nil, nil) - wantEncryption := crypto.Requested(r.Header) - s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption) - dstOpts.IndexCB = cb - defer s2c.Close() - reader = etag.Wrap(s2c, reader) - length = -1 } else { delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") @@ -1541,6 +1535,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } } + if isDstCompressed { + checksumReader := srcInfo.Reader + wantEncryption := crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(checksumReader, actualSize, wantEncryption) + dstOpts.IndexCB = cb + defer s2c.Close() + reader = etag.Wrap(s2c, checksumReader) + srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // The storage reader consumes compressed data; checksums remain bound to plaintext. + pReader = NewPutObjReader(srcInfo.Reader) + pReader.setChecksumReader(checksumReader) + } + if isTargetEncrypted { var encReader io.Reader kind, _ := crypto.IsRequested(r.Header) From 05df6e70d7e6c2e3bd64f1cf34bd59e085ead566 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 20:32:11 +0800 Subject: [PATCH 11/13] fix: preserve transform state on metadata-only copies Signed-off-by: Feng Ruohang --- cmd/object-copy-metadata_test.go | 202 +++++++++++++++++++++++++++++++ cmd/object-handlers.go | 31 ++++- 2 files changed, 228 insertions(+), 5 deletions(-) create mode 100644 cmd/object-copy-metadata_test.go diff --git a/cmd/object-copy-metadata_test.go b/cmd/object-copy-metadata_test.go new file mode 100644 index 000000000..43470436d --- /dev/null +++ b/cmd/object-copy-metadata_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "net/http" + "net/http/httptest" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" +) + +func TestAPICopyObjectMetadataOnlyCompression(t *testing.T) { + defer DetectTestLeak(t)() + for _, versioned := range []bool{false, true} { + name := "unversioned" + if versioned { + name = "versioned" + } + t.Run(name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectMetadataOnlyCompression, + endpoints: []string{"CopyObject", "PutObject", "GetObject"}, + makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned}, + }) + }) + } +} + +func testAPICopyObjectMetadataOnlyCompression(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + data := bytes.Repeat([]byte("metadata-only-copy-plaintext-"), 64*1024) + want := mustChecksum(t, hash.ChecksumCRC32, data) + object := "copy-metadata/existing-checksum.txt" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data, + map[string]string{xhttp.AmzChecksumCRC32: want}) + before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil || before.IsCompressed() { + t.Fatalf("%s: invalid metadata-copy precondition: compressed=%v size=%d err=%v", + instanceType, before.IsCompressed(), before.Size, err) + } + + restoreCompression := setCopyChecksumCompression(true) + compressionRestored := false + defer func() { + if !compressionRestored { + restoreCompression() + } + }() + + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, + map[string]string{xhttp.AmzMetadataDirective: "REPLACE"}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, false, nil) + if got := readCopyChecksumObject(t, obj, bucketName, object, ObjectOptions{}); !bytes.Equal(got, data) { + prefix := got + if len(prefix) > 100 { + prefix = prefix[:100] + } + t.Fatalf("%s: metadata-only CopyObject body differs: got %d bytes, want %d, prefix %q", + instanceType, len(got), len(data), prefix) + } + afterMetadataCopy, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if before.VersionID != "" && afterMetadataCopy.VersionID == before.VersionID { + t.Fatalf("%s: versioned metadata-only copy did not create a new version", instanceType) + } + + destination := "copy-metadata/rewritten.txt" + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, object, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: data-rewriting CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + + compressedObject := "copy-metadata/preserve-compressed.txt" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, compressedObject, data, + map[string]string{xhttp.AmzChecksumCRC32: want}) + assertCopyChecksum(t, obj, bucketName, compressedObject, hash.ChecksumCRC32, data, true, nil) + + restoreCompression() + compressionRestored = true + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, compressedObject, compressedObject, + map[string]string{xhttp.AmzMetadataDirective: "REPLACE"}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: compressed metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, compressedObject, hash.ChecksumCRC32, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, compressedObject, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: compressed metadata-only CopyObject body differs", instanceType) + } +} + +func TestAPICopyObjectSSECKeyRotationKeepsCompressionState(t *testing.T) { + defer DetectTestLeak(t)() + for _, versioned := range []bool{false, true} { + name := "unversioned" + if versioned { + name = "versioned" + } + t.Run(name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectSSECKeyRotationKeepsCompressionState, + endpoints: []string{"CopyObject", "PutObject", "GetObject"}, + makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned}, + }) + }) + } +} + +func testAPICopyObjectSSECKeyRotationKeepsCompressionState(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + previousTLS := globalIsTLS + globalIsTLS = true + defer func() { globalIsTLS = previousTLS }() + + data := bytes.Repeat([]byte("key-rotation-plaintext-"), 64*1024) + object := "copy-metadata/key-rotation.txt" + oldKey := bytes.Repeat([]byte{0x11}, 32) + oldMD5 := md5.Sum(oldKey) + newKey := bytes.Repeat([]byte{0x22}, 32) + newMD5 := md5.Sum(newKey) + + putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data, map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]), + }) + before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil || before.IsCompressed() { + t.Fatalf("%s: invalid key-rotation precondition: compressed=%v err=%v", instanceType, before.IsCompressed(), err) + } + + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]), + xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(oldKey), + xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]), + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: key rotation failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + after, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if after.IsCompressed() { + t.Fatalf("%s: metadata-only key rotation stamped compression metadata", instanceType) + } + if before.VersionID != "" && after.VersionID == before.VersionID { + t.Fatalf("%s: versioned key rotation did not create a new version", instanceType) + } + + getHeaders := map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]), + } + req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object), + 0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders) + if err != nil { + t.Fatalf("failed to build GetObject request: %v", err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("%s: post-rotation GetObject returned %d with %d bytes, want 200 with %d bytes: %s", + instanceType, response.Code, response.Body.Len(), len(data), response.Body.String()) + } +} diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 18065def8..14b3e93f9 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1357,6 +1357,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } // no changes in storage-class expected so its a metadataonly operation. var reader io.Reader = gr + sourceCompressMetadata := make(map[string]string, 2) + for _, key := range []string{ + ReservedMetadataPrefix + "compression", + ReservedMetadataPrefix + "actual-size", + } { + if value, ok := srcInfo.UserDefined[key]; ok { + sourceCompressMetadata[key] = value + } + } // Set the actual size to the compressed/decrypted size if encrypted. actualSize, err := srcInfo.GetActualSize() @@ -1387,8 +1396,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re reader = etag.NewReader(ctx, reader, nil, nil) } else { - delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") - delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") reader = gr } @@ -1687,8 +1694,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) } - // Store the preserved compression metadata. - maps.Copy(srcInfo.UserDefined, compressMetadata) + // Compression metadata must describe data that is actually rewritten. + if !srcInfo.metadataOnly || srcInfo.Legacy || dstOpts.WantServerSideChecksumType.IsSet() { + if isDstCompressed { + maps.Copy(srcInfo.UserDefined, compressMetadata) + } else { + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") + } + } else { + maps.Copy(srcInfo.UserDefined, sourceCompressMetadata) + } // We need to preserve the encryption headers set in EncryptRequest, // so we do not want to override them, copy them instead. @@ -1778,9 +1794,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re copyObjectFn := objectAPI.CopyObject + copySrcOpts := srcOpts + if srcInfo.metadataOnly && dstOpts.Versioned && copySrcOpts.VersionID == "" { + copySrcOpts.VersionID = srcInfo.VersionID + } + // Copy source object to destination, if source and destination // object is same then only metadata is updated. - objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return From f2520f3346b7ca3974d57c6bf8b55d5c8fd56b4b Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 20:37:36 +0800 Subject: [PATCH 12/13] fix: return checksums from CopyObject Signed-off-by: Feng Ruohang --- .../rebrand-guard/compat-baseline.json | 6 +++ cmd/api-response.go | 23 +++++++++--- cmd/object-copy-checksum_test.go | 37 +++++++++++++++++++ cmd/object-handlers.go | 2 +- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index 6296c75f1..ccacdc502 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -1832,6 +1832,12 @@ "cmd:cmd:field:CopyObjectPartResponse.ETag", "cmd:cmd:field:CopyObjectPartResponse.LastModified", "cmd:cmd:field:CopyObjectPartResponse.XMLName", + "cmd:cmd:field:CopyObjectResponse.ChecksumCRC32", + "cmd:cmd:field:CopyObjectResponse.ChecksumCRC32C", + "cmd:cmd:field:CopyObjectResponse.ChecksumCRC64NVME", + "cmd:cmd:field:CopyObjectResponse.ChecksumSHA1", + "cmd:cmd:field:CopyObjectResponse.ChecksumSHA256", + "cmd:cmd:field:CopyObjectResponse.ChecksumType", "cmd:cmd:field:CopyObjectResponse.ETag", "cmd:cmd:field:CopyObjectResponse.LastModified", "cmd:cmd:field:CopyObjectResponse.XMLName", diff --git a/cmd/api-response.go b/cmd/api-response.go index e7e22582c..ddd5d15e3 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -27,7 +27,6 @@ import ( "path" "strconv" "strings" - "time" "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/crypto" @@ -380,6 +379,13 @@ type CopyObjectResponse struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` LastModified string // time string of format "2006-01-02T15:04:05.000Z" ETag string // md5sum of the copied object. + + ChecksumCRC32 string `xml:",omitempty"` + ChecksumCRC32C string `xml:",omitempty"` + ChecksumSHA1 string `xml:",omitempty"` + ChecksumSHA256 string `xml:",omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` + ChecksumType string `xml:",omitempty"` } // CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object @@ -769,11 +775,18 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode) -// generates CopyObjectResponse from etag and lastModified time. -func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse { +// generates CopyObjectResponse from the committed object information. +func generateCopyObjectResponse(oi ObjectInfo, h http.Header) CopyObjectResponse { + cs, _ := oi.decryptChecksums(0, h) return CopyObjectResponse{ - ETag: "\"" + etag + "\"", - LastModified: amztime.ISO8601Format(lastModified.UTC()), + ETag: "\"" + oi.ETag + "\"", + LastModified: amztime.ISO8601Format(oi.ModTime.UTC()), + ChecksumCRC32: cs[hash.ChecksumCRC32.String()], + ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()], + ChecksumSHA1: cs[hash.ChecksumSHA1.String()], + ChecksumSHA256: cs[hash.ChecksumSHA256.String()], + ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()], + ChecksumType: cs[xhttp.AmzChecksumType], } } diff --git a/cmd/object-copy-checksum_test.go b/cmd/object-copy-checksum_test.go index 29f153c84..ad373f171 100644 --- a/cmd/object-copy-checksum_test.go +++ b/cmd/object-copy-checksum_test.go @@ -22,6 +22,7 @@ import ( "crypto/md5" "encoding/base64" "encoding/hex" + "encoding/xml" "io" "net/http" "net/http/httptest" @@ -117,6 +118,33 @@ func assertCopyChecksum(t *testing.T, obj ObjectLayer, bucket, object string, ty return oi } +func assertCopyChecksumResponse(t *testing.T, rec *httptest.ResponseRecorder, typ hash.ChecksumType, data []byte) { + t.Helper() + var response CopyObjectResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("unable to decode CopyObjectResult: %v", err) + } + var got string + switch typ.Base() { + case hash.ChecksumCRC32: + got = response.ChecksumCRC32 + case hash.ChecksumCRC32C: + got = response.ChecksumCRC32C + case hash.ChecksumSHA1: + got = response.ChecksumSHA1 + case hash.ChecksumSHA256: + got = response.ChecksumSHA256 + case hash.ChecksumCRC64NVME: + got = response.ChecksumCRC64NVME + } + if want := mustChecksum(t, typ, data); got != want { + t.Fatalf("CopyObjectResult %s checksum %q, want %q: %s", typ.String(), got, want, rec.Body.String()) + } + if response.ChecksumType != xhttp.AmzChecksumTypeFullObject { + t.Fatalf("CopyObjectResult checksum type %q, want %q", response.ChecksumType, xhttp.AmzChecksumTypeFullObject) + } +} + // TestAPICopyObjectServerSideChecksum verifies that server-computed checksums // cover the logical object, never the compressed storage stream. func TestAPICopyObjectServerSideChecksum(t *testing.T) { @@ -187,6 +215,7 @@ func testAPICopyObjectServerSideChecksum(obj ObjectLayer, instanceType, bucketNa if rec.Code != http.StatusOK { t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, tc.typ, data) info := assertCopyChecksum(t, obj, bucketName, destination, tc.typ, data, tc.compressed, nil) md5sum := md5.Sum(data) @@ -280,6 +309,7 @@ func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType if rec.Code != http.StatusOK { t.Fatalf("%s: SSE-S3 CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, nil) if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { t.Fatalf("%s: SSE-S3 round-trip body differs", instanceType) @@ -296,6 +326,7 @@ func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType if rec.Code != http.StatusOK { t.Fatalf("%s: SSE-S3 source CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { t.Fatalf("%s: SSE-S3 source round-trip body differs", instanceType) @@ -340,6 +371,7 @@ func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType if rec.Code != http.StatusOK { t.Fatalf("%s: SSE-C CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, decryptHeaders) req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, destination), @@ -390,6 +422,7 @@ func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instance if rec.Code != http.StatusOK { t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, source, map[string]string{ @@ -399,6 +432,7 @@ func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instance if rec.Code != http.StatusOK { t.Fatalf("%s: in-place CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, hash.ChecksumSHA256, data) assertCopyChecksum(t, obj, bucketName, source, hash.ChecksumSHA256, data, true, nil) if got := readCopyChecksumObject(t, obj, bucketName, source, ObjectOptions{}); !bytes.Equal(got, data) { t.Fatalf("%s: in-place CopyObject body differs", instanceType) @@ -415,6 +449,7 @@ func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instance if rec.Code != http.StatusOK { t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data) assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) }) @@ -447,6 +482,7 @@ func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instance if rec.Code != http.StatusOK { t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, typ, full) assertCopyChecksum(t, obj, bucketName, destination, typ, full, true, nil) if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, full) { t.Fatalf("%s: multipart source round-trip body differs", instanceType) @@ -475,6 +511,7 @@ func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instance if rec.Code != http.StatusOK { t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) } + assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, boundary.data) assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, boundary.data, boundary.compressed, nil) }) } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 14b3e93f9..5ed554e62 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1810,7 +1810,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re origETag := objInfo.ETag objInfo.ETag = getDecryptedETag(r.Header, objInfo, false) - response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime) + response := generateCopyObjectResponse(objInfo, r.Header) encodedSuccessResponse := encodeResponse(response) if dsc := mustReplicate(ctx, dstBucket, dstObject, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, dstOpts)); dsc.ReplicateAny() { From 8d76a255c4fed9081cdb1a5f1455524858619029 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 26 Aug 2026 01:06:50 +0800 Subject: [PATCH 13/13] fix: return the remote part checksum to federated UploadPartCopy (#72) The legacy etcd federation branch of CopyObjectPartHandler forwards copied bytes with minio-go Core.PutObjectPart, which can only recover a checksum from response headers. After the server-side part checksum work, the remote computes and persists the checksum, but an AWS-compatible UploadPart response correctly omits a checksum the request did not supply, so the proxy had nothing to put in CopyPartResult. The destination now returns the non-empty checksum fields of the PartInfo produced by that exact write, but only when the request carries the minio-federated application token that getRemoteInstanceClient already attaches. Ordinary UploadPart responses are unchanged, and the checksum type is deliberately not returned because UploadPart does not carry it. The User-Agent is a response-shape hint only: it never gates authorization, visibility or validation, and it can expose nothing beyond the checksum of the body the caller just uploaded. Reading the checksum from the same PartInfo that produced the response ETag also keeps the pair bound to one write, so a concurrent overwrite of the same part number cannot publish another writer's checksum. Tests cover the application token gating matrix including lookalike tokens, the real minio-go response parser, concurrent overwrites of one part number, and an in-process two-deployment probe that drives the federation branch through the real getRemoteInstanceClient into a real PutObjectPartHandler for both FULL_OBJECT and COMPOSITE uploads. Fixes #64 Signed-off-by: Feng Ruohang Co-authored-by: Claude Opus 5 (1M context) --- cmd/object-handlers.go | 8 +- ...ject-multipart-federation-checksum_test.go | 367 ++++++++++++++++++ cmd/object-multipart-handlers.go | 47 +++ 3 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 cmd/object-multipart-federation-checksum_test.go diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 5ed554e62..d5e868925 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1122,6 +1122,12 @@ func getRemoteInstanceTransport() http.RoundTripper { return nil } +// federatedInternalAppName is the minio-go application token that +// getRemoteInstanceClient attaches to every legacy federation proxy request. It +// is declared next to its only producer so that the literal keeps its historical +// file attribution in the rebrand compatibility baseline. +const federatedInternalAppName = "minio-federated" + // Returns a minio-go Client configured to access remote host described by destDNSRecord // Applicable only in a federated deployment var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) { @@ -1136,7 +1142,7 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, if err != nil { return nil, err } - core.SetAppInfo("minio-federated", ReleaseTag) + core.SetAppInfo(federatedInternalAppName, ReleaseTag) return core, nil } diff --git a/cmd/object-multipart-federation-checksum_test.go b/cmd/object-multipart-federation-checksum_test.go new file mode 100644 index 000000000..f7b8f5017 --- /dev/null +++ b/cmd/object-multipart-federation-checksum_test.go @@ -0,0 +1,367 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// Copyright (c) 2025-2026 PGSTY +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "encoding/xml" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + miniogo "github.com/minio/minio-go/v7" + miniocredentials "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" +) + +const federatedTestUserAgent = "MinIO (linux; amd64) minio-go/v7.0.99 minio-federated/RELEASE.TEST" + +func TestAPIFederatedUploadPartChecksumResponse(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedUploadPartChecksumResponse, + endpoints: []string{"PutObjectPart", "NewMultipart"}, + }) +} + +func testAPIFederatedUploadPartChecksumResponse(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + algorithms := []struct { + name string + typ hash.ChecksumType + checksumType string + }{ + {name: "crc32-full-object", typ: hash.ChecksumCRC32, checksumType: xhttp.AmzChecksumTypeFullObject}, + {name: "sha256-composite", typ: hash.ChecksumSHA256, checksumType: xhttp.AmzChecksumTypeComposite}, + } + userAgents := []struct { + name string + ua string + want bool + }{ + {name: "absent"}, + {name: "ordinary-sdk", ua: "aws-sdk-go/1.55.5"}, + {name: "federation", ua: federatedTestUserAgent, want: true}, + {name: "lookalike-prefix", ua: "evil-minio-federated/RELEASE.TEST"}, + {name: "lookalike-suffix", ua: "minio-federated-extra/RELEASE.TEST"}, + {name: "missing-version", ua: "minio-federated"}, + {name: "empty-version", ua: "minio-federated/"}, + } + data := []byte("federated upload part checksum response") + + for _, algorithm := range algorithms { + for _, userAgent := range userAgents { + t.Run(algorithm.name+"/"+userAgent.name, func(t *testing.T) { + object := "federation/response/" + algorithm.name + "/" + userAgent.name + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + algorithm.typ.String(), algorithm.checksumType) + headers := map[string]string{} + if userAgent.ua != "" { + headers["User-Agent"] = userAgent.ua + } + _, rec := uploadPartHTTP(t, apiRouter, credentials, + bucketName, object, uploadID, 1, data, headers) + + got := rec.Header().Get(algorithm.typ.Key()) + if userAgent.want { + if want := mustChecksum(t, algorithm.typ, data); got != want { + t.Fatalf("%s: checksum %q, want %q", instanceType, got, want) + } + } else if got != "" { + t.Fatalf("%s: ordinary UploadPart exposed server checksum %q", instanceType, got) + } + if got := rec.Header().Get(xhttp.AmzChecksumType); got != "" { + t.Fatalf("%s: UploadPart returned checksum type %q", instanceType, got) + } + }) + } + } +} + +func TestAPIFederatedUploadPartChecksumMinIOGoWire(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedUploadPartChecksumMinIOGoWire, + endpoints: []string{"PutObjectPart", "NewMultipart"}, + }) +} + +func testAPIFederatedUploadPartChecksumMinIOGoWire(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + server := httptest.NewServer(apiRouter) + defer server.Close() + + core, err := miniogo.NewCore(server.Listener.Addr().String(), &miniogo.Options{ + Creds: miniocredentials.NewStaticV4(credentials.AccessKey, credentials.SecretKey, ""), + Secure: false, + Region: globalMinioDefaultRegion, + BucketLookup: miniogo.BucketLookupPath, + }) + if err != nil { + t.Fatalf("%s: create minio-go Core: %v", instanceType, err) + } + core.SetAppInfo("minio-federated", ReleaseTag) + + object := "federation/minio-go-wire" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject) + data := []byte("minio-go must parse the remote computed checksum") + part, err := core.PutObjectPart(t.Context(), bucketName, object, uploadID, 1, + bytes.NewReader(data), int64(len(data)), miniogo.PutObjectPartOptions{}) + if err != nil { + t.Fatalf("%s: minio-go PutObjectPart: %v", instanceType, err) + } + if want := mustChecksum(t, hash.ChecksumCRC32, data); part.ChecksumCRC32 != want { + t.Fatalf("%s: minio-go checksum %q, want %q", instanceType, part.ChecksumCRC32, want) + } + if part.ETag == "" { + t.Fatalf("%s: minio-go returned an empty ETag", instanceType) + } +} + +func TestAPIFederatedUploadPartChecksumConcurrentOverwrite(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedUploadPartChecksumConcurrentOverwrite, + endpoints: []string{"PutObjectPart", "NewMultipart"}, + }) +} + +func testAPIFederatedUploadPartChecksumConcurrentOverwrite(_ ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + object := "federation/concurrent-overwrite" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object, + hash.ChecksumSHA256.String(), xhttp.AmzChecksumTypeComposite) + data := [][]byte{ + bytes.Repeat([]byte("first-writer-"), 4096), + bytes.Repeat([]byte("second-writer-"), 4096), + } + reqs := make([]*http.Request, len(data)) + recorders := make([]*httptest.ResponseRecorder, len(data)) + for i := range data { + req, err := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, object, uploadID, "1"), + int64(len(data[i])), bytes.NewReader(data[i]), credentials.AccessKey, credentials.SecretKey, + map[string]string{"User-Agent": federatedTestUserAgent}) + if err != nil { + t.Fatalf("%s: build concurrent request %d: %v", instanceType, i, err) + } + reqs[i] = req + recorders[i] = httptest.NewRecorder() + } + + start := make(chan struct{}) + var wg sync.WaitGroup + for i := range reqs { + wg.Add(1) + go func() { + defer wg.Done() + <-start + apiRouter.ServeHTTP(recorders[i], reqs[i]) + }() + } + close(start) + wg.Wait() + + for i, rec := range recorders { + if rec.Code != http.StatusOK { + t.Fatalf("%s: concurrent request %d failed: %d %s", instanceType, i, rec.Code, rec.Body.String()) + } + got := rec.Header().Get(hash.ChecksumSHA256.Key()) + if want := mustChecksum(t, hash.ChecksumSHA256, data[i]); got != want { + t.Fatalf("%s: concurrent request %d checksum %q, want %q", instanceType, i, got, want) + } + // The ETag and the checksum must describe the same write, so a losing + // writer can never publish the winner's checksum next to its own ETag. + etags := rec.Header()[xhttp.ETag] + if len(etags) != 1 { + t.Fatalf("%s: concurrent request %d returned %d ETags", instanceType, i, len(etags)) + } + md5sum := md5.Sum(data[i]) + if want := hex.EncodeToString(md5sum[:]); canonicalizeETag(etags[0]) != want { + t.Fatalf("%s: concurrent request %d ETag %q, want %q", instanceType, i, etags[0], want) + } + } +} + +// federationTestDNS is a minimal dns.Store so a single test process can play +// both federation roles. +type federationTestDNS struct { + records map[string][]dns.SrvRecord +} + +func (f federationTestDNS) Put(string) error { return nil } + +func (f federationTestDNS) Get(bucket string) ([]dns.SrvRecord, error) { + records, ok := f.records[bucket] + if !ok { + return nil, dns.ErrNoEntriesFound + } + return records, nil +} + +func (f federationTestDNS) Delete(string) error { return nil } +func (f federationTestDNS) List() (map[string][]dns.SrvRecord, error) { return f.records, nil } +func (f federationTestDNS) DeleteRecord(dns.SrvRecord) error { return nil } +func (f federationTestDNS) Close() error { return nil } +func (f federationTestDNS) String() string { return "federation-test-dns" } + +// remoteBucketObjectLayer reports one existing bucket as missing so that +// isRemoteCopyRequired takes the legacy federation branch while the same +// process can still serve that bucket as the remote deployment. +type remoteBucketObjectLayer struct { + ObjectLayer + remoteBucket string +} + +func (l remoteBucketObjectLayer) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) { + if bucket == l.remoteBucket { + return BucketInfo{}, toObjectErr(errVolumeNotFound, bucket) + } + return l.ObjectLayer.GetBucketInfo(ctx, bucket, opts) +} + +// TestAPIFederatedCopyObjectPartChecksum drives the legacy etcd federation +// branch of CopyObjectPartHandler end to end: the proxy forwards the copied +// bytes through the real getRemoteInstanceClient and minio-go, a second HTTP +// endpoint serves the real PutObjectPartHandler, and CopyPartResult must carry +// the checksum computed by that exact remote write. +func TestAPIFederatedCopyObjectPartChecksum(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPIFederatedCopyObjectPartChecksum, + endpoints: []string{ + "CopyObjectPart", "NewMultipart", "PutObjectPart", + "ListObjectParts", "CompleteMultipart", "PutObject", + }, + }) +} + +func testAPIFederatedCopyObjectPartChecksum(objectAPI ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + algorithms := []struct { + name string + typ hash.ChecksumType + checksumType string + }{ + {name: "crc32-full-object", typ: hash.ChecksumCRC32, checksumType: xhttp.AmzChecksumTypeFullObject}, + {name: "sha256-composite", typ: hash.ChecksumSHA256, checksumType: xhttp.AmzChecksumTypeComposite}, + } + + data := bytes.Repeat([]byte("federated-upload-part-copy-"), 1024) + srcObject := "federation/copy-source.bin" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data, nil) + + // The destination bucket really exists so the remote endpoint can serve it; + // only the proxy's own bucket lookup is told that it lives elsewhere. + remoteBucket := getRandomBucketName() + if err := objectAPI.MakeBucket(t.Context(), remoteBucket, MakeBucketOptions{}); err != nil { + t.Fatalf("%s: unable to create the remote bucket: %v", instanceType, err) + } + + remote := httptest.NewServer(apiRouter) + defer remote.Close() + host, port, _ := strings.Cut(remote.Listener.Addr().String(), ":") + + globalObjLayerMutex.Lock() + previousLayer := globalObjectAPI + globalObjectAPI = remoteBucketObjectLayer{ObjectLayer: previousLayer, remoteBucket: remoteBucket} + globalObjLayerMutex.Unlock() + previousDNS, previousFederation, previousIPs := globalDNSConfig, globalBucketFederation, globalDomainIPs + globalDNSConfig = federationTestDNS{records: map[string][]dns.SrvRecord{ + bucketName: {{Host: host, Port: json.Number(port)}}, + remoteBucket: {{Host: host, Port: json.Number(port)}}, + }} + // Every DNS record resolves to this process, so the bucket forwarding + // middleware always serves locally and only the handler proxies. + globalDomainIPs = set.CreateStringSet(remote.Listener.Addr().String()) + globalBucketFederation = true + defer func() { + globalObjLayerMutex.Lock() + globalObjectAPI = previousLayer + globalObjLayerMutex.Unlock() + globalDNSConfig, globalBucketFederation, globalDomainIPs = previousDNS, previousFederation, previousIPs + }() + + for _, algorithm := range algorithms { + t.Run(algorithm.name, func(t *testing.T) { + object := "federation/copy-destination-" + algorithm.name + ".bin" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, remoteBucket, object, + algorithm.typ.String(), algorithm.checksumType) + + req, err := newTestSignedRequestV4(http.MethodPut, + getCopyObjectPartURL("", remoteBucket, object, uploadID, "1"), + 0, nil, credentials.AccessKey, credentials.SecretKey, + map[string]string{xhttp.AmzCopySource: SlashSeparator + pathJoin(bucketName, srcObject)}) + if err != nil { + t.Fatalf("%s: unable to build UploadPartCopy request: %v", instanceType, err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: federated UploadPartCopy failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + + var response CopyObjectPartResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("%s: unable to decode CopyPartResult: %v", instanceType, err) + } + want := mustChecksum(t, algorithm.typ, data) + if got := copyPartChecksum(algorithm.typ, response); got != want { + t.Fatalf("%s: CopyPartResult %s is %q, want %q: %s", + instanceType, algorithm.typ.String(), got, want, rec.Body.String()) + } + + // The persisted part must carry the same value, and the client must be + // able to complete the upload with what CopyPartResult returned. + parts := listPartsHTTP(t, apiRouter, credentials, remoteBucket, object, uploadID, nil) + if len(parts.Parts) != 1 { + t.Fatalf("%s: ListParts returned %d parts, want 1", instanceType, len(parts.Parts)) + } + if got := partChecksum(algorithm.typ, parts.Parts[0]); got != want { + t.Fatalf("%s: persisted part %s is %q, want %q", instanceType, algorithm.typ.String(), got, want) + } + etag := canonicalizeETag(response.ETag) + completed := completePartsHTTP(t, apiRouter, credentials, remoteBucket, object, uploadID, + []CompletePart{completePartWithChecksum(algorithm.typ, 1, etag, want)}, nil) + if completed.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload rejected the federated part: %d %s", + instanceType, completed.Code, completed.Body.String()) + } + }) + } +} diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go index 03791f499..2f6a68a94 100644 --- a/cmd/object-multipart-handlers.go +++ b/cmd/object-multipart-handlers.go @@ -55,6 +55,46 @@ import ( // Multipart objectAPIHandlers +// isFederatedInternalRequest reports whether User-Agent carries the minio-go +// application token attached by getRemoteInstanceClient. +// +// This is only a response-shape hint. User-Agent is not authenticated and must +// never gate authorization, object visibility, or request validation. It is +// safe here because the only effect is returning the checksum of the body the +// caller was already authorized to upload. +func isFederatedInternalRequest(userAgent string) bool { + for _, product := range strings.Fields(userAgent) { + name, version, ok := strings.Cut(product, "/") + if ok && name == federatedInternalAppName && version != "" { + return true + } + } + return false +} + +// partChecksumMap returns the non-empty part checksums in the form expected by +// hash.AddChecksumHeader. x-amz-checksum-type is deliberately excluded because +// UploadPart does not return it and minio-go cannot carry it in ObjectPart. +func partChecksumMap(partInfo PartInfo) map[string]string { + checksums := make(map[string]string, 1) + if partInfo.ChecksumCRC32 != "" { + checksums[hash.ChecksumCRC32.String()] = partInfo.ChecksumCRC32 + } + if partInfo.ChecksumCRC32C != "" { + checksums[hash.ChecksumCRC32C.String()] = partInfo.ChecksumCRC32C + } + if partInfo.ChecksumSHA1 != "" { + checksums[hash.ChecksumSHA1.String()] = partInfo.ChecksumSHA1 + } + if partInfo.ChecksumSHA256 != "" { + checksums[hash.ChecksumSHA256.String()] = partInfo.ChecksumSHA256 + } + if partInfo.ChecksumCRC64NVME != "" { + checksums[hash.ChecksumCRC64NVME.String()] = partInfo.ChecksumCRC64NVME + } + return checksums +} + // multipartChecksumType returns the base checksum type recorded when a // multipart upload was created. The boolean reports whether an algorithm was // recorded at all. @@ -996,6 +1036,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http // Therefore, we have to set the ETag directly as map entry. w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""} hash.TransferChecksumHeader(w, r) + if isFederatedInternalRequest(r.UserAgent()) { + // Legacy federation proxies UploadPartCopy through minio-go + // Core.PutObjectPart, which can only recover checksums from response + // headers. Use the PartInfo returned by this exact write so the ETag and + // checksum cannot be mixed with a concurrent overwrite. + hash.AddChecksumHeader(w, partChecksumMap(partInfo)) + } writeSuccessResponseHeadersOnly(w) }