mirror of
https://github.com/pgsty/minio.git
synced 2026-09-05 18:16:16 +03:00
Merge branch 'main' into feat-add-checksumtype-completemultipartupload-response
This commit is contained in:
@@ -29,7 +29,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Install govulncheck
|
- name: Install govulncheck
|
||||||
run: |
|
run: |
|
||||||
go install golang.org/x/vuln/cmd/govulncheck@v1.6.0
|
go install golang.org/x/vuln/cmd/govulncheck@v1.7.0
|
||||||
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
|
||||||
|
|
||||||
- name: Run govulncheck
|
- name: Run govulncheck
|
||||||
|
|||||||
@@ -59,6 +59,9 @@ formatters:
|
|||||||
exclusions:
|
exclusions:
|
||||||
generated: lax
|
generated: lax
|
||||||
paths:
|
paths:
|
||||||
|
# gofumpt v0.11.0 and Go 1.27's gofmt disagree on the indentation of
|
||||||
|
# multiple composite literals returned from a single statement.
|
||||||
|
- internal/ioutil/wait_pipe\.go$
|
||||||
- third_party$
|
- third_party$
|
||||||
- builtin$
|
- builtin$
|
||||||
- examples$
|
- examples$
|
||||||
|
|||||||
@@ -3396,7 +3396,7 @@ SOFTWARE.
|
|||||||
github.com/cheggaaa/pb
|
github.com/cheggaaa/pb
|
||||||
https://github.com/cheggaaa/pb
|
https://github.com/cheggaaa/pb
|
||||||
----------------------------------------------------------------
|
----------------------------------------------------------------
|
||||||
Copyright (c) 2012-2015, Sergey Cherepanov
|
Copyright (c) 2012-2024, Sergey Cherepanov
|
||||||
All rights reserved.
|
All rights reserved.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||||
@@ -16641,33 +16641,6 @@ SOFTWARE.
|
|||||||
|
|
||||||
================================================================
|
================================================================
|
||||||
|
|
||||||
github.com/lestrrat-go/option
|
|
||||||
https://github.com/lestrrat-go/option
|
|
||||||
----------------------------------------------------------------
|
|
||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2021 lestrrat-go
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
|
|
||||||
================================================================
|
|
||||||
|
|
||||||
github.com/lestrrat-go/option/v2
|
github.com/lestrrat-go/option/v2
|
||||||
https://github.com/lestrrat-go/option/v2
|
https://github.com/lestrrat-go/option/v2
|
||||||
----------------------------------------------------------------
|
----------------------------------------------------------------
|
||||||
@@ -21713,6 +21686,14 @@ separate copyright notices and license terms. Your use of the source
|
|||||||
code for the these subcomponents is subject to the terms and conditions
|
code for the these subcomponents is subject to the terms and conditions
|
||||||
of the following licenses.
|
of the following licenses.
|
||||||
|
|
||||||
|
===
|
||||||
|
|
||||||
|
This distribution (pgsty/mc, shipped as "mcli") is a community-maintained
|
||||||
|
fork of the MinIO Client, modified by the Silo project
|
||||||
|
(https://silo.pgsty.com/). It is not affiliated with, endorsed by, or
|
||||||
|
sponsored by MinIO, Inc. Modifications are Copyright (c) 2025-2026 PGSTY
|
||||||
|
and are licensed under the GNU AGPL v3.0 or later.
|
||||||
|
|
||||||
================================================================
|
================================================================
|
||||||
|
|
||||||
github.com/minio/md5-simd
|
github.com/minio/md5-simd
|
||||||
@@ -28982,7 +28963,7 @@ https://go.etcd.io/etcd/api/v3
|
|||||||
same "printed page" as the copyright notice for easier
|
same "printed page" as the copyright notice for easier
|
||||||
identification within third-party archives.
|
identification within third-party archives.
|
||||||
|
|
||||||
Copyright [yyyy] [name of copyright owner]
|
Copyright 2020 The etcd Authors
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
@@ -29190,7 +29171,7 @@ https://go.etcd.io/etcd/client/pkg/v3
|
|||||||
same "printed page" as the copyright notice for easier
|
same "printed page" as the copyright notice for easier
|
||||||
identification within third-party archives.
|
identification within third-party archives.
|
||||||
|
|
||||||
Copyright [yyyy] [name of copyright owner]
|
Copyright 2020 The etcd Authors
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
@@ -29398,7 +29379,7 @@ https://go.etcd.io/etcd/client/v3
|
|||||||
same "printed page" as the copyright notice for easier
|
same "printed page" as the copyright notice for easier
|
||||||
identification within third-party archives.
|
identification within third-party archives.
|
||||||
|
|
||||||
Copyright [yyyy] [name of copyright owner]
|
Copyright 2020 The etcd Authors
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
FROM golang:1.26.5-alpine AS build
|
FROM golang:1.27.0-alpine AS build
|
||||||
|
|
||||||
ARG TARGETARCH
|
ARG TARGETARCH
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
|||||||
|
|
||||||
GOOS ?= $(shell go env GOOS)
|
GOOS ?= $(shell go env GOOS)
|
||||||
GOARCH ?= $(shell go env GOARCH)
|
GOARCH ?= $(shell go env GOARCH)
|
||||||
GOLANGCI_VERSION ?= v2.11.3
|
GOLANGCI_VERSION ?= v2.13.1
|
||||||
|
|
||||||
VERSION ?= $(shell git describe --tags)
|
VERSION ?= $(shell git describe --tags)
|
||||||
REPO ?= docker.io/pgsty
|
REPO ?= docker.io/pgsty
|
||||||
@@ -48,7 +48,7 @@ credits: ## regenerate CREDITS from the licenses of Go modules linked into the b
|
|||||||
|
|
||||||
check-gen: ## check for updated autogenerated files
|
check-gen: ## check for updated autogenerated files
|
||||||
@go generate ./... >/dev/null
|
@go generate ./... >/dev/null
|
||||||
@go mod tidy -compat=1.26
|
@go mod tidy -compat=1.27
|
||||||
@env bash $(PWD)/buildscripts/gen-credits.sh
|
@env bash $(PWD)/buildscripts/gen-credits.sh
|
||||||
@changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum CREDITS); \
|
@changed=$$(git diff --name-only -- '*_gen.go' '*_gen_test.go' '*_msgp_test.go' '*_string.go' go.mod go.sum CREDITS); \
|
||||||
if [ -n "$$changed" ]; then \
|
if [ -n "$$changed" ]; then \
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ _init() {
|
|||||||
|
|
||||||
## Minimum required versions for build dependencies
|
## Minimum required versions for build dependencies
|
||||||
GIT_VERSION="1.0"
|
GIT_VERSION="1.0"
|
||||||
GO_VERSION="1.16"
|
GO_VERSION="1.27.0"
|
||||||
OSX_VERSION="10.8"
|
OSX_VERSION="10.8"
|
||||||
KNAME=$(uname -s)
|
KNAME=$(uname -s)
|
||||||
ARCH=$(uname -m)
|
ARCH=$(uname -m)
|
||||||
|
|||||||
@@ -1825,9 +1825,20 @@
|
|||||||
"cmd:cmd:field:CompletePart.PartNumber",
|
"cmd:cmd:field:CompletePart.PartNumber",
|
||||||
"cmd:cmd:field:CompletePart.Size",
|
"cmd:cmd:field:CompletePart.Size",
|
||||||
"cmd:cmd:field:ConsoleLogger.Enable",
|
"cmd:cmd:field:ConsoleLogger.Enable",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32C",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC64NVME",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA1",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA256",
|
||||||
"cmd:cmd:field:CopyObjectPartResponse.ETag",
|
"cmd:cmd:field:CopyObjectPartResponse.ETag",
|
||||||
"cmd:cmd:field:CopyObjectPartResponse.LastModified",
|
"cmd:cmd:field:CopyObjectPartResponse.LastModified",
|
||||||
"cmd:cmd:field:CopyObjectPartResponse.XMLName",
|
"cmd:cmd:field:CopyObjectPartResponse.XMLName",
|
||||||
|
"cmd:cmd:field:CopyObjectResponse.ChecksumCRC32",
|
||||||
|
"cmd:cmd:field:CopyObjectResponse.ChecksumCRC32C",
|
||||||
|
"cmd:cmd:field:CopyObjectResponse.ChecksumCRC64NVME",
|
||||||
|
"cmd:cmd:field:CopyObjectResponse.ChecksumSHA1",
|
||||||
|
"cmd:cmd:field:CopyObjectResponse.ChecksumSHA256",
|
||||||
|
"cmd:cmd:field:CopyObjectResponse.ChecksumType",
|
||||||
"cmd:cmd:field:CopyObjectResponse.ETag",
|
"cmd:cmd:field:CopyObjectResponse.ETag",
|
||||||
"cmd:cmd:field:CopyObjectResponse.LastModified",
|
"cmd:cmd:field:CopyObjectResponse.LastModified",
|
||||||
"cmd:cmd:field:CopyObjectResponse.XMLName",
|
"cmd:cmd:field:CopyObjectResponse.XMLName",
|
||||||
@@ -6939,7 +6950,9 @@
|
|||||||
"internal/config/notify:notify:func:SetNotifyRedis",
|
"internal/config/notify:notify:func:SetNotifyRedis",
|
||||||
"internal/config/notify:notify:func:SetNotifyWebhook",
|
"internal/config/notify:notify:func:SetNotifyWebhook",
|
||||||
"internal/config/notify:notify:func:TestSubSysNotificationTargets",
|
"internal/config/notify:notify:func:TestSubSysNotificationTargets",
|
||||||
|
"internal/config/notify:notify:method:LegacyDatabaseTargetError.Error",
|
||||||
"internal/config/notify:notify:type:Config",
|
"internal/config/notify:notify:type:Config",
|
||||||
|
"internal/config/notify:notify:type:LegacyDatabaseTargetError",
|
||||||
"internal/config/notify:notify:var:DefaultAMQPKVS",
|
"internal/config/notify:notify:var:DefaultAMQPKVS",
|
||||||
"internal/config/notify:notify:var:DefaultESKVS",
|
"internal/config/notify:notify:var:DefaultESKVS",
|
||||||
"internal/config/notify:notify:var:DefaultKafkaKVS",
|
"internal/config/notify:notify:var:DefaultKafkaKVS",
|
||||||
|
|||||||
@@ -859,7 +859,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
|||||||
|
|
||||||
var sp *policy.Policy
|
var sp *policy.Policy
|
||||||
if len(updateReq.NewPolicy) > 0 {
|
if len(updateReq.NewPolicy) > 0 {
|
||||||
sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
|
sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -1729,7 +1729,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
|
iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -2981,7 +2981,7 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
|||||||
|
|
||||||
var sp *policy.Policy
|
var sp *policy.Policy
|
||||||
if len(createReq.Policy) > 0 {
|
if len(createReq.Policy) > 0 {
|
||||||
sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
|
sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -204,6 +204,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
|||||||
suite.TestUserCreate(c)
|
suite.TestUserCreate(c)
|
||||||
suite.TestUserPolicyEscalationBug(c)
|
suite.TestUserPolicyEscalationBug(c)
|
||||||
suite.TestPolicyCreate(c)
|
suite.TestPolicyCreate(c)
|
||||||
|
suite.TestServiceAccountBareARNPolicyRejected(c)
|
||||||
suite.TestCannedPolicies(c)
|
suite.TestCannedPolicies(c)
|
||||||
suite.TestGroupAddRemove(c)
|
suite.TestGroupAddRemove(c)
|
||||||
suite.TestServiceAccountOpsByAdmin(c)
|
suite.TestServiceAccountOpsByAdmin(c)
|
||||||
@@ -600,6 +601,20 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
|||||||
c.Fatalf("invalid policy creation success")
|
c.Fatalf("invalid policy creation success")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for i, resource := range []string{"arn:aws:s3:::", "*arn:aws:s3:::"} {
|
||||||
|
barePolicyBytes := fmt.Appendf(nil, `{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Deny",
|
||||||
|
"Action": ["s3:GetObject"],
|
||||||
|
"Resource": ["%s"]
|
||||||
|
}]
|
||||||
|
}`, resource)
|
||||||
|
if err = s.adm.AddCannedPolicy(ctx, fmt.Sprintf("%s-bare-%d", policy, i), barePolicyBytes); err == nil {
|
||||||
|
c.Fatalf("bare ARN policy creation succeeded for %q", resource)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 3. Create a user, associate policy and verify access
|
// 3. Create a user, associate policy and verify access
|
||||||
accessKey, secretKey := mustGenerateCredentials(c)
|
accessKey, secretKey := mustGenerateCredentials(c)
|
||||||
err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
|
err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
|
||||||
@@ -653,6 +668,51 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *TestSuiteIAM) TestServiceAccountBareARNPolicyRejected(c *check) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
barePolicy := []byte(`{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject"],
|
||||||
|
"NotResource": ["arn:aws:s3:::"]
|
||||||
|
}]
|
||||||
|
}`)
|
||||||
|
if _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||||
|
TargetUser: globalActiveCred.AccessKey,
|
||||||
|
Policy: barePolicy,
|
||||||
|
}); err == nil {
|
||||||
|
c.Fatal("service account creation accepted a bare ARN policy")
|
||||||
|
}
|
||||||
|
|
||||||
|
validPolicy := []byte(`{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::*"]
|
||||||
|
}]
|
||||||
|
}`)
|
||||||
|
credentials, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||||
|
TargetUser: globalActiveCred.AccessKey,
|
||||||
|
Policy: validPolicy,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
c.Fatalf("service account creation rejected an explicit resource: %v", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
_ = s.adm.DeleteServiceAccount(ctx, credentials.AccessKey)
|
||||||
|
}()
|
||||||
|
|
||||||
|
if err = s.adm.UpdateServiceAccount(ctx, credentials.AccessKey, madmin.UpdateServiceAccountReq{
|
||||||
|
NewPolicy: barePolicy,
|
||||||
|
}); err == nil {
|
||||||
|
c.Fatal("service account update accepted a bare ARN policy")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|||||||
+33
-9
@@ -27,7 +27,6 @@ import (
|
|||||||
"path"
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/minio/minio/internal/amztime"
|
"github.com/minio/minio/internal/amztime"
|
||||||
"github.com/minio/minio/internal/crypto"
|
"github.com/minio/minio/internal/crypto"
|
||||||
@@ -380,6 +379,13 @@ type CopyObjectResponse struct {
|
|||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||||
ETag string // md5sum of the copied object.
|
ETag string // md5sum of the copied object.
|
||||||
|
|
||||||
|
ChecksumCRC32 string `xml:",omitempty"`
|
||||||
|
ChecksumCRC32C string `xml:",omitempty"`
|
||||||
|
ChecksumSHA1 string `xml:",omitempty"`
|
||||||
|
ChecksumSHA256 string `xml:",omitempty"`
|
||||||
|
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||||
|
ChecksumType string `xml:",omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object
|
// CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object
|
||||||
@@ -387,6 +393,12 @@ type CopyObjectPartResponse struct {
|
|||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
||||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||||
ETag string // md5sum of the copied object part.
|
ETag string // md5sum of the copied object part.
|
||||||
|
|
||||||
|
ChecksumCRC32 string `xml:",omitempty"`
|
||||||
|
ChecksumCRC32C string `xml:",omitempty"`
|
||||||
|
ChecksumSHA1 string `xml:",omitempty"`
|
||||||
|
ChecksumSHA256 string `xml:",omitempty"`
|
||||||
|
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initiator inherit from Owner struct, fields are same
|
// Initiator inherit from Owner struct, fields are same
|
||||||
@@ -764,19 +776,31 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
|||||||
|
|
||||||
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
||||||
|
|
||||||
// generates CopyObjectResponse from etag and lastModified time.
|
// generates CopyObjectResponse from the committed object information.
|
||||||
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
func generateCopyObjectResponse(oi ObjectInfo, h http.Header) CopyObjectResponse {
|
||||||
|
cs, _ := oi.decryptChecksums(0, h)
|
||||||
return CopyObjectResponse{
|
return CopyObjectResponse{
|
||||||
ETag: "\"" + etag + "\"",
|
ETag: "\"" + oi.ETag + "\"",
|
||||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
LastModified: amztime.ISO8601Format(oi.ModTime.UTC()),
|
||||||
|
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||||
|
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||||
|
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||||
|
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||||
|
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
|
||||||
|
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// generates CopyObjectPartResponse from etag and lastModified time.
|
// generates CopyObjectPartResponse from the uploaded part information.
|
||||||
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
|
func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse {
|
||||||
return CopyObjectPartResponse{
|
return CopyObjectPartResponse{
|
||||||
ETag: "\"" + etag + "\"",
|
ETag: "\"" + partInfo.ETag + "\"",
|
||||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()),
|
||||||
|
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||||
|
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||||
|
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||||
|
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||||
|
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -198,7 +198,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
|||||||
byPassSet, r, cred, owner)
|
byPassSet, r, cred, owner)
|
||||||
// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
|
// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
|
||||||
if !byPassSet {
|
if !byPassSet {
|
||||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -209,7 +209,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
|||||||
case objectlock.RetCompliance:
|
case objectlock.RetCompliance:
|
||||||
// Compliance retention mode cannot be changed or shortened.
|
// Compliance retention mode cannot be changed or shortened.
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||||
}
|
}
|
||||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||||
|
|||||||
+48
-8
@@ -36,6 +36,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
fcolor "github.com/fatih/color"
|
fcolor "github.com/fatih/color"
|
||||||
@@ -540,6 +541,34 @@ func (e envKV) String() string {
|
|||||||
return fmt.Sprintf("%s=%s", e.Key, e.Value)
|
return fmt.Sprintf("%s=%s", e.Key, e.Value)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isValidEnvName(name string) bool {
|
||||||
|
if name == "" || !isEnvNameStart(name[0]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := 1; i < len(name); i++ {
|
||||||
|
if !isEnvNameStart(name[i]) && (name[i] < '0' || name[i] > '9') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func isEnvNameStart(ch byte) bool {
|
||||||
|
return ch == '_' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z'
|
||||||
|
}
|
||||||
|
|
||||||
|
func trimExportPrefix(envEntry string) string {
|
||||||
|
rest, ok := strings.CutPrefix(envEntry, "export")
|
||||||
|
if !ok || rest == "" {
|
||||||
|
return envEntry
|
||||||
|
}
|
||||||
|
trimmed := strings.TrimLeftFunc(rest, unicode.IsSpace)
|
||||||
|
if len(trimmed) == len(rest) {
|
||||||
|
return envEntry
|
||||||
|
}
|
||||||
|
return trimmed
|
||||||
|
}
|
||||||
|
|
||||||
func parsEnvEntry(envEntry string) (envKV, error) {
|
func parsEnvEntry(envEntry string) (envKV, error) {
|
||||||
envEntry = strings.TrimSpace(envEntry)
|
envEntry = strings.TrimSpace(envEntry)
|
||||||
if envEntry == "" {
|
if envEntry == "" {
|
||||||
@@ -554,13 +583,19 @@ func parsEnvEntry(envEntry string) (envKV, error) {
|
|||||||
Skip: true,
|
Skip: true,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
|
envTokens := strings.SplitN(trimExportPrefix(envEntry), config.EnvSeparator, 2)
|
||||||
if len(envTokens) != 2 {
|
if len(envTokens) != 2 {
|
||||||
return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
|
return envKV{}, errors.New("missing '='")
|
||||||
}
|
}
|
||||||
|
|
||||||
key := envTokens[0]
|
key := strings.TrimSpace(envTokens[0])
|
||||||
val := envTokens[1]
|
val := strings.TrimSpace(envTokens[1])
|
||||||
|
if !isValidEnvName(key) {
|
||||||
|
return envKV{}, fmt.Errorf("invalid environment variable name %q", key)
|
||||||
|
}
|
||||||
|
if strings.IndexByte(val, 0) >= 0 {
|
||||||
|
return envKV{}, errors.New("environment variable value contains NUL")
|
||||||
|
}
|
||||||
|
|
||||||
// Remove quotes from the value if found
|
// Remove quotes from the value if found
|
||||||
if len(val) >= 2 {
|
if len(val) >= 2 {
|
||||||
@@ -587,10 +622,12 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
|
|||||||
defer f.Close()
|
defer f.Close()
|
||||||
var ekvs []envKV
|
var ekvs []envKV
|
||||||
scanner := bufio.NewScanner(f)
|
scanner := bufio.NewScanner(f)
|
||||||
|
lineNo := 0
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
|
lineNo++
|
||||||
ekv, err := parsEnvEntry(scanner.Text())
|
ekv, err := parsEnvEntry(scanner.Text())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("%s:%d: %w", envConfigFile, lineNo, err)
|
||||||
}
|
}
|
||||||
if ekv.Skip {
|
if ekv.Skip {
|
||||||
// Skips empty lines
|
// Skips empty lines
|
||||||
@@ -599,7 +636,7 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
|
|||||||
ekvs = append(ekvs, ekv)
|
ekvs = append(ekvs, ekv)
|
||||||
}
|
}
|
||||||
if err = scanner.Err(); err != nil {
|
if err = scanner.Err(); err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("%s: %w", envConfigFile, err)
|
||||||
}
|
}
|
||||||
return ekvs, nil
|
return ekvs, nil
|
||||||
}
|
}
|
||||||
@@ -666,12 +703,15 @@ func loadEnvVarsFromFiles() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if env.IsSet(config.EnvConfigEnvFile) {
|
if env.IsSet(config.EnvConfigEnvFile) {
|
||||||
ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
|
envConfigFile := env.Get(config.EnvConfigEnvFile, "")
|
||||||
|
ekvs, err := minioEnvironFromFile(envConfigFile)
|
||||||
if err != nil && !os.IsNotExist(err) {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
logger.Fatal(err, "Unable to read the config environment file")
|
logger.Fatal(err, "Unable to read the config environment file")
|
||||||
}
|
}
|
||||||
for _, ekv := range ekvs {
|
for _, ekv := range ekvs {
|
||||||
os.Setenv(ekv.Key, ekv.Value)
|
if err := os.Setenv(ekv.Key, ekv.Value); err != nil {
|
||||||
|
logger.Fatal(err, "Unable to set %s from config environment file %s", ekv.Key, envConfigFile)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,8 +19,10 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -181,3 +183,162 @@ MINIO_ROOT_PASSWORD=minio123`,
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
want []envKV
|
||||||
|
errLine int
|
||||||
|
errContains string
|
||||||
|
errExcludes string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "spaces and tabs around separator",
|
||||||
|
content: "MINIO_ROOT_USER = minio\nMINIO_ROOT_PASSWORD\t=\tminio123",
|
||||||
|
want: []envKV{
|
||||||
|
{Key: "MINIO_ROOT_USER", Value: "minio"},
|
||||||
|
{Key: "MINIO_ROOT_PASSWORD", Value: "minio123"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "export tab and quoted spaces",
|
||||||
|
content: "export\tMINIO_ROOT_USER = \" minio user \"\nexport MINIO_ROOT_PASSWORD = ' minio secret '",
|
||||||
|
want: []envKV{
|
||||||
|
{Key: "MINIO_ROOT_USER", Value: " minio user "},
|
||||||
|
{Key: "MINIO_ROOT_PASSWORD", Value: " minio secret "},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "export Unicode whitespace",
|
||||||
|
content: "export\u00a0MINIO_ROOT_USER=value",
|
||||||
|
want: []envKV{
|
||||||
|
{Key: "MINIO_ROOT_USER", Value: "value"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "export is only a standalone prefix",
|
||||||
|
content: "export=value\nexportFOO=bar",
|
||||||
|
want: []envKV{
|
||||||
|
{Key: "export", Value: "value"},
|
||||||
|
{Key: "exportFOO", Value: "bar"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unquoted whitespace empty value and additional separators",
|
||||||
|
content: "UNQUOTED = value \nEMPTY =\nTOKEN = scheme://user:password@example.com?a=b",
|
||||||
|
want: []envKV{
|
||||||
|
{Key: "UNQUOTED", Value: "value"},
|
||||||
|
{Key: "EMPTY", Value: ""},
|
||||||
|
{Key: "TOKEN", Value: "scheme://user:password@example.com?a=b"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid underscore and digits",
|
||||||
|
content: "_VALID_2=value",
|
||||||
|
want: []envKV{
|
||||||
|
{Key: "_VALID_2", Value: "value"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing separator redacts the line",
|
||||||
|
content: "MINIO_ROOT_PASSWORD=valid\nsuper-secret-without-equals",
|
||||||
|
errLine: 2,
|
||||||
|
errContains: "missing '='",
|
||||||
|
errExcludes: "super-secret-without-equals",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty name",
|
||||||
|
content: "=empty-name-secret",
|
||||||
|
errLine: 1,
|
||||||
|
errContains: `invalid environment variable name ""`,
|
||||||
|
errExcludes: "empty-name-secret",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "digit leading name",
|
||||||
|
content: "1MINIO_ROOT_USER=digit-leading-secret",
|
||||||
|
errLine: 1,
|
||||||
|
errContains: `invalid environment variable name "1MINIO_ROOT_USER"`,
|
||||||
|
errExcludes: "digit-leading-secret",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "hyphenated name",
|
||||||
|
content: "MINIO-ROOT-USER=hyphen-secret",
|
||||||
|
errLine: 1,
|
||||||
|
errContains: `invalid environment variable name "MINIO-ROOT-USER"`,
|
||||||
|
errExcludes: "hyphen-secret",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "whitespace in name",
|
||||||
|
content: "MINIO ROOT USER=whitespace-secret",
|
||||||
|
errLine: 1,
|
||||||
|
errContains: `invalid environment variable name "MINIO ROOT USER"`,
|
||||||
|
errExcludes: "whitespace-secret",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "NUL in name",
|
||||||
|
content: "MINIO\x00ROOT=nul-name-secret",
|
||||||
|
errLine: 1,
|
||||||
|
errContains: "invalid environment variable name",
|
||||||
|
errExcludes: "nul-name-secret",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "NUL in value",
|
||||||
|
content: "MINIO_ROOT_USER=before\x00nul-value-secret",
|
||||||
|
errLine: 1,
|
||||||
|
errContains: "environment variable value contains NUL",
|
||||||
|
errExcludes: "nul-value-secret",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "diagnostic has file and line but no value",
|
||||||
|
content: "MINIO_ROOT_USER=valid\nBAD KEY=super-secret-value",
|
||||||
|
errLine: 2,
|
||||||
|
errContains: `invalid environment variable name "BAD KEY"`,
|
||||||
|
errExcludes: "super-secret-value",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = tmpfile.WriteString(testCase.content); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = tmpfile.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := minioEnvironFromFile(tmpfile.Name())
|
||||||
|
if testCase.errContains == "" {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, testCase.want) {
|
||||||
|
t.Errorf("expected %v, got %v", testCase.want, got)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected an error")
|
||||||
|
}
|
||||||
|
errText := err.Error()
|
||||||
|
location := fmt.Sprintf("%s:%d:", tmpfile.Name(), testCase.errLine)
|
||||||
|
if !strings.Contains(errText, location) {
|
||||||
|
t.Errorf("expected error to contain %q, got %q", location, errText)
|
||||||
|
}
|
||||||
|
if !strings.Contains(errText, testCase.errContains) {
|
||||||
|
t.Errorf("expected error to contain %q, got %q", testCase.errContains, errText)
|
||||||
|
}
|
||||||
|
if testCase.errExcludes != "" && strings.Contains(errText, testCase.errExcludes) {
|
||||||
|
t.Errorf("expected error to redact %q, got %q", testCase.errExcludes, errText)
|
||||||
|
}
|
||||||
|
if got != nil {
|
||||||
|
t.Errorf("expected no entries on parse error, got %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -167,7 +167,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
|
|||||||
notify.SetNotifyMQTT(newCfg, k, args)
|
notify.SetNotifyMQTT(newCfg, k, args)
|
||||||
}
|
}
|
||||||
for k, args := range cfg.Notify.MySQL {
|
for k, args := range cfg.Notify.MySQL {
|
||||||
notify.SetNotifyMySQL(newCfg, k, args)
|
if err := notify.SetNotifyMySQL(newCfg, k, args); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for k, args := range cfg.Notify.NATS {
|
for k, args := range cfg.Notify.NATS {
|
||||||
notify.SetNotifyNATS(newCfg, k, args)
|
notify.SetNotifyNATS(newCfg, k, args)
|
||||||
@@ -176,7 +178,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
|
|||||||
notify.SetNotifyNSQ(newCfg, k, args)
|
notify.SetNotifyNSQ(newCfg, k, args)
|
||||||
}
|
}
|
||||||
for k, args := range cfg.Notify.PostgreSQL {
|
for k, args := range cfg.Notify.PostgreSQL {
|
||||||
notify.SetNotifyPostgres(newCfg, k, args)
|
if err := notify.SetNotifyPostgres(newCfg, k, args); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for k, args := range cfg.Notify.Redis {
|
for k, args := range cfg.Notify.Redis {
|
||||||
notify.SetNotifyRedis(newCfg, k, args)
|
notify.SetNotifyRedis(newCfg, k, args)
|
||||||
|
|||||||
@@ -0,0 +1,264 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/config"
|
||||||
|
"github.com/minio/minio/internal/config/notify"
|
||||||
|
"github.com/minio/minio/internal/event/target"
|
||||||
|
)
|
||||||
|
|
||||||
|
func installLegacyConfigFile(t *testing.T, configure func(*serverConfigV33)) (string, []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
cfg := &serverConfigV33{
|
||||||
|
Version: "33",
|
||||||
|
Notify: notify.NewConfig(),
|
||||||
|
}
|
||||||
|
configure(cfg)
|
||||||
|
|
||||||
|
data, err := json.Marshal(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
oldConfigDir := globalConfigDir
|
||||||
|
globalConfigDir = &ConfigDir{path: t.TempDir()}
|
||||||
|
t.Cleanup(func() { globalConfigDir = oldConfigDir })
|
||||||
|
|
||||||
|
configFile := getConfigFile()
|
||||||
|
if err = os.WriteFile(configFile, data, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return configFile, data
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertLegacyMigrationError(t *testing.T, err error, subsystem, name, key, secret string) {
|
||||||
|
t.Helper()
|
||||||
|
var targetErr *notify.LegacyDatabaseTargetError
|
||||||
|
if !errors.As(err, &targetErr) {
|
||||||
|
t.Fatalf("error = %v, want *notify.LegacyDatabaseTargetError", err)
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} {
|
||||||
|
if !strings.Contains(msg, want) {
|
||||||
|
t.Errorf("error %q does not contain %q", msg, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(msg, secret) {
|
||||||
|
t.Errorf("migration error leaks database password %q: %s", secret, msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadConfigWithoutMigrateRejectsLegacyDatabaseTargets(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
subsystem string
|
||||||
|
key string
|
||||||
|
secret string
|
||||||
|
configure func(*serverConfigV33)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "postgres",
|
||||||
|
subsystem: config.NotifyPostgresSubSys,
|
||||||
|
key: target.PostgresConnectionString,
|
||||||
|
secret: "postgres-migration-secret",
|
||||||
|
configure: func(cfg *serverConfigV33) {
|
||||||
|
cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Port: "5432",
|
||||||
|
Username: "legacy-user",
|
||||||
|
Password: "postgres-migration-secret",
|
||||||
|
Database: "events",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mysql",
|
||||||
|
subsystem: config.NotifyMySQLSubSys,
|
||||||
|
key: target.MySQLDSNString,
|
||||||
|
secret: "mysql-migration-secret",
|
||||||
|
configure: func(cfg *serverConfigV33) {
|
||||||
|
cfg.Notify.MySQL["archive"] = target.MySQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Port: "3306",
|
||||||
|
User: "legacy-user",
|
||||||
|
Password: "mysql-migration-secret",
|
||||||
|
Database: "events",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
configFile, original := installLegacyConfigFile(t, test.configure)
|
||||||
|
got, err := readConfigWithoutMigrate(t.Context(), nil)
|
||||||
|
if got != nil {
|
||||||
|
t.Fatalf("config = %v, want nil on failed migration", got)
|
||||||
|
}
|
||||||
|
assertLegacyMigrationError(t, err, test.subsystem, "archive", test.key, test.secret)
|
||||||
|
|
||||||
|
after, readErr := os.ReadFile(configFile)
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatal(readErr)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(after, original) {
|
||||||
|
t.Fatal("failed migration rewrote the legacy source config")
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(configFile + ".old"); !errors.Is(statErr, os.ErrNotExist) {
|
||||||
|
t.Fatalf("failed migration created a backup/persistence artifact: %v", statErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadConfigWithoutMigrateMigratesCanonicalDatabaseTargets(t *testing.T) {
|
||||||
|
const (
|
||||||
|
postgresConnection = "host=postgres.example port=5432 dbname=events user=app password=secret sslmode=disable"
|
||||||
|
mysqlDSN = "app:secret@tcp(mysql.example:3306)/events?parseTime=true"
|
||||||
|
discardedLegacyValue = "discarded-legacy-value"
|
||||||
|
)
|
||||||
|
installLegacyConfigFile(t, func(cfg *serverConfigV33) {
|
||||||
|
cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Format: "namespace",
|
||||||
|
ConnectionString: postgresConnection,
|
||||||
|
Table: "events",
|
||||||
|
Port: discardedLegacyValue,
|
||||||
|
Username: discardedLegacyValue,
|
||||||
|
Password: discardedLegacyValue,
|
||||||
|
Database: discardedLegacyValue,
|
||||||
|
}
|
||||||
|
cfg.Notify.MySQL["archive"] = target.MySQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Format: "namespace",
|
||||||
|
DSN: mysqlDSN,
|
||||||
|
Table: "events",
|
||||||
|
Port: discardedLegacyValue,
|
||||||
|
User: discardedLegacyValue,
|
||||||
|
Password: discardedLegacyValue,
|
||||||
|
Database: discardedLegacyValue,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
got, err := readConfigWithoutMigrate(t.Context(), nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("readConfigWithoutMigrate: %v", err)
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
subsystem string
|
||||||
|
key string
|
||||||
|
want string
|
||||||
|
discarded string
|
||||||
|
}{
|
||||||
|
{config.NotifyPostgresSubSys, target.PostgresConnectionString, postgresConnection, discardedLegacyValue},
|
||||||
|
{config.NotifyMySQLSubSys, target.MySQLDSNString, mysqlDSN, discardedLegacyValue},
|
||||||
|
}
|
||||||
|
for _, test := range tests {
|
||||||
|
kvs := got[test.subsystem]["archive"]
|
||||||
|
if value := kvs.Get(test.key); value != test.want {
|
||||||
|
t.Errorf("%s %s = %q, want %q", test.subsystem, test.key, value, test.want)
|
||||||
|
}
|
||||||
|
if err := config.CheckValidKeys(test.subsystem+config.SubSystemSeparator+"archive", kvs, notify.DefaultNotificationKVS[test.subsystem]); err != nil {
|
||||||
|
t.Errorf("migrated %s target failed key validation: %v", test.subsystem, err)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"host", "port", "username", "password", "database"} {
|
||||||
|
if _, ok := kvs.Lookup(key); ok {
|
||||||
|
t.Errorf("migrated %s target contains legacy key %q", test.subsystem, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, kv := range kvs {
|
||||||
|
if strings.Contains(kv.Value, test.discarded) {
|
||||||
|
t.Errorf("migrated %s target contains discarded legacy value in %q", test.subsystem, kv.Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
postgresTargets, err := notify.GetNotifyPostgres(got[config.NotifyPostgresSubSys])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetNotifyPostgres: %v", err)
|
||||||
|
}
|
||||||
|
if value := postgresTargets["archive"].ConnectionString; value != postgresConnection {
|
||||||
|
t.Errorf("Postgres connection string = %q, want %q", value, postgresConnection)
|
||||||
|
}
|
||||||
|
mysqlTargets, err := notify.GetNotifyMySQL(got[config.NotifyMySQLSubSys])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetNotifyMySQL: %v", err)
|
||||||
|
}
|
||||||
|
if value := mysqlTargets["archive"].DSN; value != mysqlDSN {
|
||||||
|
t.Errorf("MySQL DSN = %q, want %q", value, mysqlDSN)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInitConfigSubsystemReturnsLegacyDatabaseTargetError(t *testing.T) {
|
||||||
|
obj, fsDir, err := prepareFS(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = obj.Shutdown(context.Background())
|
||||||
|
_ = os.RemoveAll(fsDir)
|
||||||
|
})
|
||||||
|
|
||||||
|
const secret = "startup-migration-secret"
|
||||||
|
installLegacyConfigFile(t, func(cfg *serverConfigV33) {
|
||||||
|
cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Port: "5432",
|
||||||
|
Username: "legacy-user",
|
||||||
|
Password: secret,
|
||||||
|
Database: "events",
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
globalServerConfigMu.RLock()
|
||||||
|
var before config.Config
|
||||||
|
if globalServerConfig != nil {
|
||||||
|
before = globalServerConfig.Clone()
|
||||||
|
}
|
||||||
|
globalServerConfigMu.RUnlock()
|
||||||
|
|
||||||
|
err = initConfigSubsystem(t.Context(), obj)
|
||||||
|
assertLegacyMigrationError(t, err, config.NotifyPostgresSubSys, "archive", target.PostgresConnectionString, secret)
|
||||||
|
if configRetriableErrors(err) {
|
||||||
|
t.Fatal("legacy database migration error must be startup-fatal, not retriable")
|
||||||
|
}
|
||||||
|
if !fatalServerConfigError(err) {
|
||||||
|
t.Fatal("legacy database migration error must abort server startup")
|
||||||
|
}
|
||||||
|
|
||||||
|
globalServerConfigMu.RLock()
|
||||||
|
var after config.Config
|
||||||
|
if globalServerConfig != nil {
|
||||||
|
after = globalServerConfig.Clone()
|
||||||
|
}
|
||||||
|
globalServerConfigMu.RUnlock()
|
||||||
|
if !reflect.DeepEqual(after, before) {
|
||||||
|
t.Fatal("failed migration activated a partial server configuration")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -326,7 +326,7 @@ func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool {
|
|||||||
if cycles <= 1 {
|
if cycles <= 1 {
|
||||||
return cycles == 1
|
return cycles == 1
|
||||||
}
|
}
|
||||||
return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles
|
return uint32(xxhash.Sum64String(string(h))>>32)%cycles == cycle%cycles
|
||||||
}
|
}
|
||||||
|
|
||||||
// addChild will add a child based on its hash.
|
// addChild will add a child based on its hash.
|
||||||
|
|||||||
@@ -0,0 +1,596 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/hash"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
func uploadPartHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, object, uploadID string, partNumber int, data []byte, headers map[string]string,
|
||||||
|
) (string, *httptest.ResponseRecorder) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(partNumber)),
|
||||||
|
int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("UploadPart failed: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
return canonicalizeETag(rec.Header()[xhttp.ETag][0]), rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func listPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, object, uploadID string, headers map[string]string,
|
||||||
|
) ListPartsResponse {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet,
|
||||||
|
getListMultipartURLWithParams("", bucket, object, uploadID, "1000", "", ""),
|
||||||
|
0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build ListParts request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("ListParts failed: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var response ListPartsResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||||
|
t.Fatalf("failed to decode ListParts response: %v", err)
|
||||||
|
}
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
func partChecksum(typ hash.ChecksumType, part Part) string {
|
||||||
|
switch typ.Base() {
|
||||||
|
case hash.ChecksumCRC32:
|
||||||
|
return part.ChecksumCRC32
|
||||||
|
case hash.ChecksumCRC32C:
|
||||||
|
return part.ChecksumCRC32C
|
||||||
|
case hash.ChecksumSHA1:
|
||||||
|
return part.ChecksumSHA1
|
||||||
|
case hash.ChecksumSHA256:
|
||||||
|
return part.ChecksumSHA256
|
||||||
|
case hash.ChecksumCRC64NVME:
|
||||||
|
return part.ChecksumCRC64NVME
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPartChecksum(typ hash.ChecksumType, response CopyObjectPartResponse) string {
|
||||||
|
switch typ.Base() {
|
||||||
|
case hash.ChecksumCRC32:
|
||||||
|
return response.ChecksumCRC32
|
||||||
|
case hash.ChecksumCRC32C:
|
||||||
|
return response.ChecksumCRC32C
|
||||||
|
case hash.ChecksumSHA1:
|
||||||
|
return response.ChecksumSHA1
|
||||||
|
case hash.ChecksumSHA256:
|
||||||
|
return response.ChecksumSHA256
|
||||||
|
case hash.ChecksumCRC64NVME:
|
||||||
|
return response.ChecksumCRC64NVME
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func completePartWithChecksum(typ hash.ChecksumType, partNumber int, etag, checksum string) CompletePart {
|
||||||
|
part := CompletePart{PartNumber: partNumber, ETag: etag}
|
||||||
|
switch typ.Base() {
|
||||||
|
case hash.ChecksumCRC32:
|
||||||
|
part.ChecksumCRC32 = checksum
|
||||||
|
case hash.ChecksumCRC32C:
|
||||||
|
part.ChecksumCRC32C = checksum
|
||||||
|
case hash.ChecksumSHA1:
|
||||||
|
part.ChecksumSHA1 = checksum
|
||||||
|
case hash.ChecksumSHA256:
|
||||||
|
part.ChecksumSHA256 = checksum
|
||||||
|
case hash.ChecksumCRC64NVME:
|
||||||
|
part.ChecksumCRC64NVME = checksum
|
||||||
|
}
|
||||||
|
return part
|
||||||
|
}
|
||||||
|
|
||||||
|
func completePartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, object, uploadID string, parts []CompletePart, headers map[string]string,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to encode CompleteMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPost,
|
||||||
|
getCompleteMultipartUploadURL("", bucket, object, uploadID),
|
||||||
|
int64(len(body)), bytes.NewReader(body), creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPartWithoutChecksumHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, source, object, uploadID, sourceRange string, headers map[string]string,
|
||||||
|
) CopyObjectPartResponse {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getCopyObjectPartURL("", bucket, object, uploadID, "1"),
|
||||||
|
0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPartCopy request: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source))
|
||||||
|
if sourceRange != "" {
|
||||||
|
req.Header.Set(xhttp.AmzCopySourceRange, sourceRange)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("UploadPartCopy failed: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var response CopyObjectPartResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||||
|
t.Fatalf("failed to decode UploadPartCopy response: %v", err)
|
||||||
|
}
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPIUploadPartServerSideChecksum exercises the data transformations that
|
||||||
|
// made installing a checksum hasher in the object layer unsafe. The checksum
|
||||||
|
// must always cover logical plaintext, regardless of compression or encryption.
|
||||||
|
func TestAPIUploadPartServerSideChecksum(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecExtendedObjectLayerAPITest(t, testAPIUploadPartServerSideChecksum,
|
||||||
|
[]string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
data := bytes.Repeat([]byte("multipart-checksum-plaintext-"), 48*1024)
|
||||||
|
want := mustChecksum(t, typ, data)
|
||||||
|
|
||||||
|
t.Run("upload", func(t *testing.T) {
|
||||||
|
object := "checksums/upload"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, data, nil)
|
||||||
|
if got := rec.Header().Get(typ.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
oi, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
checksums, _ := oi.decryptChecksums(0, nil)
|
||||||
|
if got := checksums[typ.String()]; got != want {
|
||||||
|
t.Fatalf("%s: stored checksum %q, want plaintext checksum %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("copy", func(t *testing.T) {
|
||||||
|
source := "checksums/source"
|
||||||
|
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||||
|
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
object := "checksums/copy"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, source, object, uploadID, "", nil)
|
||||||
|
if got := copyPartChecksum(typ, response); got != want {
|
||||||
|
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: copied ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: canonicalizeETag(response.ETag)}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumAlgorithms(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumAlgorithms,
|
||||||
|
endpoints: []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumAlgorithms(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
tests := []struct {
|
||||||
|
typ hash.ChecksumType
|
||||||
|
objType string
|
||||||
|
composite bool
|
||||||
|
}{
|
||||||
|
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeFullObject, false},
|
||||||
|
{hash.ChecksumCRC32C, xhttp.AmzChecksumTypeFullObject, false},
|
||||||
|
{hash.ChecksumCRC64NVME, xhttp.AmzChecksumTypeFullObject, false},
|
||||||
|
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeComposite, true},
|
||||||
|
{hash.ChecksumSHA1, xhttp.AmzChecksumTypeComposite, true},
|
||||||
|
{hash.ChecksumSHA256, xhttp.AmzChecksumTypeComposite, true},
|
||||||
|
}
|
||||||
|
data := bytes.Repeat([]byte("server-side-part-checksum"), 1024)
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.typ.String()+"/"+test.objType, func(t *testing.T) {
|
||||||
|
object := "algorithms/" + test.typ.String() + "/" + test.objType
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
test.typ.String(), test.objType)
|
||||||
|
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, data, nil)
|
||||||
|
if got := rec.Header().Get(test.typ.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := mustChecksum(t, test.typ, data)
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(test.typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
part := CompletePart{PartNumber: 1, ETag: etag}
|
||||||
|
if test.composite {
|
||||||
|
part = completePartWithChecksum(test.typ, 1, etag, want)
|
||||||
|
}
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{part}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("multi-part/FULL_OBJECT", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
parts, full := multipartChecksumTestData()
|
||||||
|
object := "algorithms/multi-part-full-object"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
etags := make([]string, len(parts))
|
||||||
|
for i, data := range parts {
|
||||||
|
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, i+1, data, nil)
|
||||||
|
if got := rec.Header().Get(typ.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
etags[i] = etag
|
||||||
|
}
|
||||||
|
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != len(parts) {
|
||||||
|
t.Fatalf("%s: ListParts returned %d parts, want %d", instanceType, len(listed.Parts), len(parts))
|
||||||
|
}
|
||||||
|
for i, part := range listed.Parts {
|
||||||
|
if got, want := partChecksum(typ, part), mustChecksum(t, typ, parts[i]); got != want {
|
||||||
|
t.Fatalf("%s: part %d checksum %q, want %q", instanceType, i+1, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
complete := make([]CompletePart, len(etags))
|
||||||
|
for i, etag := range etags {
|
||||||
|
complete[i] = CompletePart{PartNumber: i + 1, ETag: etag}
|
||||||
|
}
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, complete,
|
||||||
|
map[string]string{
|
||||||
|
typ.Key(): mustChecksum(t, typ, full),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: multi-part CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("zero-length-part", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
object := "algorithms/zero-length"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
etag, _ := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, nil, nil)
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, nil) {
|
||||||
|
t.Fatalf("%s: zero-length ListParts checksum mismatch: %+v", instanceType, listed.Parts)
|
||||||
|
}
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: zero-length CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("overwrite-part-checksum", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
object := "algorithms/overwrite"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
first := []byte("first part contents")
|
||||||
|
second := []byte("replacement part contents")
|
||||||
|
uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, first, nil)
|
||||||
|
etag, _ := uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, second, nil)
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, second) {
|
||||||
|
t.Fatalf("%s: overwritten ListParts checksum mismatch: %+v", instanceType, listed.Parts)
|
||||||
|
}
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: overwritten CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("copy/SHA256/COMPOSITE", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumSHA256
|
||||||
|
source := "algorithms/copy-source"
|
||||||
|
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||||
|
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
object := "algorithms/copy-SHA256"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||||
|
start, end := 7, len(data)-9
|
||||||
|
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, source, object, uploadID, "bytes="+strconv.Itoa(start)+"-"+strconv.Itoa(end-1), nil)
|
||||||
|
want := mustChecksum(t, typ, data[start:end])
|
||||||
|
if got := copyPartChecksum(typ, response); got != want {
|
||||||
|
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
part := completePartWithChecksum(typ, 1, canonicalizeETag(response.ETag), want)
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{part}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(_ ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
data := []byte("client checksum must remain authoritative")
|
||||||
|
|
||||||
|
t.Run("correct-value", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
object := "errors/correct-value"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
want := mustChecksum(t, typ, data)
|
||||||
|
_, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, data, map[string]string{typ.Key(): want})
|
||||||
|
if got := rec.Header().Get(typ.Key()); got != want {
|
||||||
|
t.Fatalf("%s: client checksum response %q, want %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: client checksum ListParts mismatch: %+v", instanceType, listed.Parts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("wrong-algorithm", func(t *testing.T) {
|
||||||
|
object := "errors/wrong-algorithm"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{hash.ChecksumSHA256.Key(): mustChecksum(t, hash.ChecksumSHA256, data)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
|
||||||
|
t.Fatalf("%s: wrong algorithm returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("wrong-value", func(t *testing.T) {
|
||||||
|
object := "errors/wrong-value"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{hash.ChecksumCRC32.Key(): mustChecksum(t, hash.ChecksumCRC32, []byte("wrong"))})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "XAmzContentChecksumMismatch" {
|
||||||
|
t.Fatalf("%s: wrong value returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumSSEC(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumSSEC,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumSSEC(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = false }()
|
||||||
|
|
||||||
|
key := bytes.Repeat([]byte{0x2a}, 32)
|
||||||
|
keyMD5 := md5.Sum(key)
|
||||||
|
ssecHeaders := map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||||
|
}
|
||||||
|
initHeaders := map[string]string{
|
||||||
|
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||||
|
}
|
||||||
|
object := "checksums/ssec"
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var initiated InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
|
||||||
|
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("ssec-checksum-plaintext"), 4096)
|
||||||
|
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, initiated.UploadID, 1, data, ssecHeaders)
|
||||||
|
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
completeHeaders := map[string]string{
|
||||||
|
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||||
|
}
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, completeHeaders)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumSSES3(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumSSES3,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumSSES3(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
GlobalKMS = KMS
|
||||||
|
defer func() { GlobalKMS = nil }()
|
||||||
|
|
||||||
|
object := "checksums/sse-s3"
|
||||||
|
initHeaders := map[string]string{
|
||||||
|
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES,
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var initiated InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
|
||||||
|
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("sse-s3-checksum-plaintext"), 4096)
|
||||||
|
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, initiated.UploadID, 1, data, nil)
|
||||||
|
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, map[string]string{
|
||||||
|
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -597,12 +597,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
|||||||
onlineDisks := er.getDisks()
|
onlineDisks := er.getDisks()
|
||||||
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
||||||
|
|
||||||
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
|
expectedChecksumType, checksumEnabled := multipartChecksumType(fi.Metadata)
|
||||||
if r.ContentCRCType().String() != cs {
|
if checksumEnabled {
|
||||||
|
got := r.contentChecksumType()
|
||||||
|
if !expectedChecksumType.IsSet() || !got.IsSet() || got.Base() != expectedChecksumType {
|
||||||
return pi, InvalidArgument{
|
return pi, InvalidArgument{
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
Object: fi.Name,
|
Object: fi.Name,
|
||||||
Err: fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()),
|
Err: fmt.Errorf("checksum missing, want %q, got %q",
|
||||||
|
fi.Metadata[hash.MinIOMultipartChecksum], got.String()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -725,6 +728,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
partChecksums := r.contentChecksum()
|
||||||
|
if checksumEnabled && partChecksums[expectedChecksumType.String()] == "" {
|
||||||
|
err := fmt.Errorf("internal error: checksum missing after reading part, want %q", expectedChecksumType.String())
|
||||||
|
bugLogIf(ctx, err)
|
||||||
|
return pi, toObjectErr(err, bucket, object, uploadID)
|
||||||
|
}
|
||||||
|
|
||||||
partInfo := ObjectPartInfo{
|
partInfo := ObjectPartInfo{
|
||||||
Number: partID,
|
Number: partID,
|
||||||
ETag: md5hex,
|
ETag: md5hex,
|
||||||
@@ -732,7 +742,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
|||||||
ActualSize: actualSize,
|
ActualSize: actualSize,
|
||||||
ModTime: UTCNow(),
|
ModTime: UTCNow(),
|
||||||
Index: index,
|
Index: index,
|
||||||
Checksums: r.ContentCRC(),
|
Checksums: partChecksums,
|
||||||
}
|
}
|
||||||
|
|
||||||
partFI, err := partInfo.MarshalMsg(nil)
|
partFI, err := partInfo.MarshalMsg(nil)
|
||||||
|
|||||||
@@ -1485,11 +1485,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
|||||||
// over opts.WantChecksum.
|
// over opts.WantChecksum.
|
||||||
if opts.WantServerSideChecksumType.IsSet() {
|
if opts.WantServerSideChecksumType.IsSet() {
|
||||||
serverSideChecksum := r.RawServerSideChecksumResult()
|
serverSideChecksum := r.RawServerSideChecksumResult()
|
||||||
if serverSideChecksum != nil {
|
if serverSideChecksum == nil || !serverSideChecksum.Valid() ||
|
||||||
fi.Checksum = serverSideChecksum.AppendTo(nil, nil)
|
serverSideChecksum.Type.Base() != opts.WantServerSideChecksumType.Base() {
|
||||||
if opts.EncryptFn != nil {
|
err := fmt.Errorf("internal error: server-side checksum missing, invalid, or mismatched after reading object, want %q", opts.WantServerSideChecksumType.String())
|
||||||
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
|
bugLogIf(ctx, err)
|
||||||
}
|
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||||
|
}
|
||||||
|
fi.Checksum = serverSideChecksum.AppendTo(nil, nil)
|
||||||
|
if opts.EncryptFn != nil {
|
||||||
|
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
|
||||||
}
|
}
|
||||||
} else if fi.Checksum == nil && opts.WantChecksum != nil {
|
} else if fi.Checksum == nil && opts.WantChecksum != nil {
|
||||||
// Trailing headers checksums should now be filled.
|
// Trailing headers checksums should now be filled.
|
||||||
|
|||||||
@@ -191,7 +191,9 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//nolint:staticcheck // Linux implementations can fail; BSD stubs return a constant nil error.
|
||||||
st, err := disk.GetDriveStats(d.Major, d.Minor)
|
st, err := disk.GetDriveStats(d.Major, d.Minor)
|
||||||
|
//nolint:staticcheck // Keep the shared cross-platform error handling.
|
||||||
if err == nil {
|
if err == nil {
|
||||||
dm.IOStats = madmin.DiskIOStats{
|
dm.IOStats = madmin.DiskIOStats{
|
||||||
ReadIOs: st.ReadIOs,
|
ReadIOs: st.ReadIOs,
|
||||||
|
|||||||
+45
-8
@@ -1038,9 +1038,10 @@ type SealMD5CurrFn func([]byte) []byte
|
|||||||
// PutObjReader is a type that wraps sio.EncryptReader and
|
// PutObjReader is a type that wraps sio.EncryptReader and
|
||||||
// underlying hash.Reader in a struct
|
// underlying hash.Reader in a struct
|
||||||
type PutObjReader struct {
|
type PutObjReader struct {
|
||||||
*hash.Reader // actual data stream
|
*hash.Reader // actual data stream
|
||||||
rawReader *hash.Reader // original data stream
|
rawReader *hash.Reader // original data stream used for ETag calculation
|
||||||
sealMD5Fn SealMD5CurrFn
|
checksumReader *hash.Reader // logical plaintext stream used for S3 checksum calculation
|
||||||
|
sealMD5Fn SealMD5CurrFn
|
||||||
}
|
}
|
||||||
|
|
||||||
// Size returns the absolute number of bytes the Reader
|
// Size returns the absolute number of bytes the Reader
|
||||||
@@ -1093,15 +1094,51 @@ func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto.
|
|||||||
// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's
|
// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's
|
||||||
// MD5Current method to construct md5sum when requested downstream.
|
// MD5Current method to construct md5sum when requested downstream.
|
||||||
func NewPutObjReader(rawReader *hash.Reader) *PutObjReader {
|
func NewPutObjReader(rawReader *hash.Reader) *PutObjReader {
|
||||||
return &PutObjReader{Reader: rawReader, rawReader: rawReader}
|
return &PutObjReader{Reader: rawReader, rawReader: rawReader, checksumReader: rawReader}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setChecksumReader sets the logical plaintext reader used for S3 checksums.
|
||||||
|
// It can differ from rawReader when the storage stream is compressed.
|
||||||
|
func (p *PutObjReader) setChecksumReader(r *hash.Reader) {
|
||||||
|
if r != nil {
|
||||||
|
p.checksumReader = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// contentChecksumType returns the effective client-provided or server-computed
|
||||||
|
// checksum type for the logical plaintext stream.
|
||||||
|
func (p *PutObjReader) contentChecksumType() hash.ChecksumType {
|
||||||
|
if p.checksumReader == nil {
|
||||||
|
return hash.ChecksumNone
|
||||||
|
}
|
||||||
|
if t := p.checksumReader.ContentCRCType(); t.IsSet() {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
return p.checksumReader.ServerSideChecksumType
|
||||||
|
}
|
||||||
|
|
||||||
|
// contentChecksum returns the effective checksum for part metadata. A
|
||||||
|
// client-provided checksum takes precedence; server computation is only a
|
||||||
|
// fallback when the client omitted one.
|
||||||
|
func (p *PutObjReader) contentChecksum() map[string]string {
|
||||||
|
if p.checksumReader == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if checksum := p.checksumReader.ContentCRC(); checksum != nil {
|
||||||
|
return checksum
|
||||||
|
}
|
||||||
|
if checksum := p.checksumReader.ServerSideChecksumResult; checksum != nil && checksum.Valid() {
|
||||||
|
return map[string]string{checksum.Type.String(): checksum.Encoded}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RawServerSideChecksumResult returns the ServerSideChecksumResult from the
|
// RawServerSideChecksumResult returns the ServerSideChecksumResult from the
|
||||||
// underlying rawReader, since the PutObjReader might be encrypted data and
|
// logical plaintext checksum reader, since the PutObjReader might contain
|
||||||
// thus any checksum from that would be incorrect.
|
// compressed or encrypted data and thus any checksum from that would be incorrect.
|
||||||
func (p *PutObjReader) RawServerSideChecksumResult() *hash.Checksum {
|
func (p *PutObjReader) RawServerSideChecksumResult() *hash.Checksum {
|
||||||
if p.rawReader != nil {
|
if p.checksumReader != nil {
|
||||||
return p.rawReader.ServerSideChecksumResult
|
return p.checksumReader.ServerSideChecksumResult
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,556 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/xml"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/hash"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
func setCopyChecksumCompression(allowEncrypted bool) func() {
|
||||||
|
globalCompressConfigMu.Lock()
|
||||||
|
previous := globalCompressConfig
|
||||||
|
globalCompressConfig.Enabled = true
|
||||||
|
globalCompressConfig.Extensions = []string{".txt"}
|
||||||
|
globalCompressConfig.MimeTypes = nil
|
||||||
|
globalCompressConfig.AllowEncrypted = allowEncrypted
|
||||||
|
globalCompressConfigMu.Unlock()
|
||||||
|
|
||||||
|
return func() {
|
||||||
|
globalCompressConfigMu.Lock()
|
||||||
|
globalCompressConfig = previous
|
||||||
|
globalCompressConfigMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyChecksumRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||||
|
bucket, source, destination string, headers map[string]string,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucket, destination),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build CopyObject request: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source))
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func putCopyChecksumSource(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||||
|
bucket, object string, data []byte, headers map[string]string,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucket, object),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build PutObject request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("PutObject(%s) failed: %d %s", object, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readCopyChecksumObject(t *testing.T, obj ObjectLayer, bucket, object string, opts ObjectOptions) []byte {
|
||||||
|
t.Helper()
|
||||||
|
gr, err := obj.GetObjectNInfo(t.Context(), bucket, object, nil, nil, opts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetObjectNInfo(%s) failed: %v", object, err)
|
||||||
|
}
|
||||||
|
defer gr.Close()
|
||||||
|
data, err := io.ReadAll(gr)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reading %s failed: %v", object, err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertCopyChecksum(t *testing.T, obj ObjectLayer, bucket, object string, typ hash.ChecksumType,
|
||||||
|
data []byte, compressed bool, decryptHeaders http.Header,
|
||||||
|
) ObjectInfo {
|
||||||
|
t.Helper()
|
||||||
|
oi, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetObjectInfo(%s) failed: %v", object, err)
|
||||||
|
}
|
||||||
|
if oi.IsCompressed() != compressed {
|
||||||
|
t.Fatalf("%s compressed=%v, want %v", object, oi.IsCompressed(), compressed)
|
||||||
|
}
|
||||||
|
checksums, _ := oi.decryptChecksums(0, decryptHeaders)
|
||||||
|
if got, want := checksums[typ.String()], mustChecksum(t, typ, data); got != want {
|
||||||
|
t.Fatalf("%s stored %s checksum %q, want logical object checksum %q (all: %v)",
|
||||||
|
object, typ.String(), got, want, checksums)
|
||||||
|
}
|
||||||
|
if got := checksums[xhttp.AmzChecksumType]; got != xhttp.AmzChecksumTypeFullObject {
|
||||||
|
t.Fatalf("%s checksum type %q, want %q", object, got, xhttp.AmzChecksumTypeFullObject)
|
||||||
|
}
|
||||||
|
return oi
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertCopyChecksumResponse(t *testing.T, rec *httptest.ResponseRecorder, typ hash.ChecksumType, data []byte) {
|
||||||
|
t.Helper()
|
||||||
|
var response CopyObjectResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||||
|
t.Fatalf("unable to decode CopyObjectResult: %v", err)
|
||||||
|
}
|
||||||
|
var got string
|
||||||
|
switch typ.Base() {
|
||||||
|
case hash.ChecksumCRC32:
|
||||||
|
got = response.ChecksumCRC32
|
||||||
|
case hash.ChecksumCRC32C:
|
||||||
|
got = response.ChecksumCRC32C
|
||||||
|
case hash.ChecksumSHA1:
|
||||||
|
got = response.ChecksumSHA1
|
||||||
|
case hash.ChecksumSHA256:
|
||||||
|
got = response.ChecksumSHA256
|
||||||
|
case hash.ChecksumCRC64NVME:
|
||||||
|
got = response.ChecksumCRC64NVME
|
||||||
|
}
|
||||||
|
if want := mustChecksum(t, typ, data); got != want {
|
||||||
|
t.Fatalf("CopyObjectResult %s checksum %q, want %q: %s", typ.String(), got, want, rec.Body.String())
|
||||||
|
}
|
||||||
|
if response.ChecksumType != xhttp.AmzChecksumTypeFullObject {
|
||||||
|
t.Fatalf("CopyObjectResult checksum type %q, want %q", response.ChecksumType, xhttp.AmzChecksumTypeFullObject)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPICopyObjectServerSideChecksum verifies that server-computed checksums
|
||||||
|
// cover the logical object, never the compressed storage stream.
|
||||||
|
func TestAPICopyObjectServerSideChecksum(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
for _, versioned := range []bool{false, true} {
|
||||||
|
name := "unversioned"
|
||||||
|
if versioned {
|
||||||
|
name = "versioned"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPICopyObjectServerSideChecksum,
|
||||||
|
endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"},
|
||||||
|
makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPICopyObjectServerSideChecksum(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
restoreCompression := setCopyChecksumCompression(true)
|
||||||
|
defer restoreCompression()
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("copy-object-checksum-plaintext-"), 64*1024)
|
||||||
|
source := "copy-checksum/source.bin"
|
||||||
|
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||||
|
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
compressedReader, _ := newS2CompressReader(bytes.NewReader(data), int64(len(data)), false)
|
||||||
|
compressed, err := io.ReadAll(compressedReader)
|
||||||
|
if closeErr := compressedReader.Close(); err == nil {
|
||||||
|
err = closeErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: independently compressing test data failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
typ hash.ChecksumType
|
||||||
|
explicit bool
|
||||||
|
extension string
|
||||||
|
compressed bool
|
||||||
|
}{
|
||||||
|
{name: "compressed/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".txt", compressed: true},
|
||||||
|
{name: "compressed/CRC32C", typ: hash.ChecksumCRC32C, explicit: true, extension: ".txt", compressed: true},
|
||||||
|
{name: "compressed/SHA1", typ: hash.ChecksumSHA1, explicit: true, extension: ".txt", compressed: true},
|
||||||
|
{name: "compressed/SHA256", typ: hash.ChecksumSHA256, explicit: true, extension: ".txt", compressed: true},
|
||||||
|
{name: "compressed/CRC64NVME", typ: hash.ChecksumCRC64NVME, explicit: true, extension: ".txt", compressed: true},
|
||||||
|
{name: "compressed/default", typ: hash.ChecksumCRC64NVME, extension: ".txt", compressed: true},
|
||||||
|
{name: "plain/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".bin"},
|
||||||
|
{name: "plain/default", typ: hash.ChecksumCRC64NVME, extension: ".bin"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
headers := map[string]string(nil)
|
||||||
|
if tc.explicit {
|
||||||
|
headers = map[string]string{xhttp.AmzChecksumAlgo: tc.typ.String()}
|
||||||
|
}
|
||||||
|
destination := "copy-checksum/destination-" + tc.typ.String() + "-" + string(rune('a'+i)) + tc.extension
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, tc.typ, data)
|
||||||
|
|
||||||
|
info := assertCopyChecksum(t, obj, bucketName, destination, tc.typ, data, tc.compressed, nil)
|
||||||
|
md5sum := md5.Sum(data)
|
||||||
|
if got, want := info.ETag, hex.EncodeToString(md5sum[:]); got != want {
|
||||||
|
t.Fatalf("%s: ETag %q, want logical object MD5 %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
if tc.compressed {
|
||||||
|
logical := mustChecksum(t, tc.typ, data)
|
||||||
|
if transformed := mustChecksum(t, tc.typ, compressed); logical == transformed {
|
||||||
|
t.Fatalf("%s: test payload does not distinguish logical and compressed checksum domains", instanceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) {
|
||||||
|
t.Fatalf("%s: round-trip body differs for %s", instanceType, tc.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tc.name == "compressed/CRC32" {
|
||||||
|
for _, method := range []string{http.MethodHead, http.MethodGet} {
|
||||||
|
url := getHeadObjectURL("", bucketName, destination)
|
||||||
|
if method == http.MethodGet {
|
||||||
|
url = getGetObjectURL("", bucketName, destination)
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(method, url, 0, nil,
|
||||||
|
credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{xhttp.AmzChecksumMode: "ENABLED"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build %s request: %v", method, err)
|
||||||
|
}
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(response, req)
|
||||||
|
if response.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s returned %d: %s", method, response.Code, response.Body.String())
|
||||||
|
}
|
||||||
|
if got, want := response.Header().Get(tc.typ.Key()), mustChecksum(t, tc.typ, data); got != want {
|
||||||
|
t.Fatalf("%s returned checksum %q, want %q", method, got, want)
|
||||||
|
}
|
||||||
|
if method == http.MethodGet && !bytes.Equal(response.Body.Bytes(), data) {
|
||||||
|
t.Fatalf("GET response body differs")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPICopyObjectServerSideChecksumEncryption(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPICopyObjectServerSideChecksumEncryption,
|
||||||
|
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
restoreCompression := setCopyChecksumCompression(true)
|
||||||
|
defer restoreCompression()
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("encrypted-copy-checksum-plaintext-"), 48*1024)
|
||||||
|
source := "copy-checksum/encrypted-source.bin"
|
||||||
|
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||||
|
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("SSE-S3", func(t *testing.T) {
|
||||||
|
secretKey, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
previousKMS := GlobalKMS
|
||||||
|
GlobalKMS = secretKey
|
||||||
|
defer func() { GlobalKMS = previousKMS }()
|
||||||
|
|
||||||
|
for _, variant := range []struct {
|
||||||
|
name string
|
||||||
|
extension string
|
||||||
|
compressed bool
|
||||||
|
}{
|
||||||
|
{name: "encrypted-only", extension: ".bin"},
|
||||||
|
{name: "compressed-encrypted", extension: ".txt", compressed: true},
|
||||||
|
} {
|
||||||
|
t.Run(variant.name, func(t *testing.T) {
|
||||||
|
destination := "copy-checksum/sse-s3-" + variant.name + variant.extension
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, map[string]string{
|
||||||
|
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||||
|
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES,
|
||||||
|
})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: SSE-S3 CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, nil)
|
||||||
|
if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) {
|
||||||
|
t.Fatalf("%s: SSE-S3 round-trip body differs", instanceType)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
encryptedSource := "copy-checksum/sse-s3-source.bin"
|
||||||
|
putCopyChecksumSource(t, apiRouter, credentials, bucketName, encryptedSource, data,
|
||||||
|
map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES})
|
||||||
|
destination := "copy-checksum/sse-s3-source-copy.txt"
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, encryptedSource, destination,
|
||||||
|
map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: SSE-S3 source CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil)
|
||||||
|
if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) {
|
||||||
|
t.Fatalf("%s: SSE-S3 source round-trip body differs", instanceType)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("SSE-C", func(t *testing.T) {
|
||||||
|
previousTLS := globalIsTLS
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = previousTLS }()
|
||||||
|
|
||||||
|
key := bytes.Repeat([]byte{0x2a}, 32)
|
||||||
|
keyMD5 := md5.Sum(key)
|
||||||
|
headers := map[string]string{
|
||||||
|
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||||
|
}
|
||||||
|
decryptHeaders := http.Header{}
|
||||||
|
for key, value := range headers {
|
||||||
|
decryptHeaders.Set(key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
getHeaders := make(map[string]string, len(headers))
|
||||||
|
for key, value := range headers {
|
||||||
|
if key != xhttp.AmzChecksumAlgo {
|
||||||
|
getHeaders[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, variant := range []struct {
|
||||||
|
name string
|
||||||
|
extension string
|
||||||
|
compressed bool
|
||||||
|
}{
|
||||||
|
{name: "encrypted-only", extension: ".bin"},
|
||||||
|
{name: "compressed-encrypted", extension: ".txt", compressed: true},
|
||||||
|
} {
|
||||||
|
t.Run(variant.name, func(t *testing.T) {
|
||||||
|
destination := "copy-checksum/sse-c-" + variant.name + variant.extension
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: SSE-C CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, decryptHeaders)
|
||||||
|
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, destination),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build SSE-C GetObject request: %v", err)
|
||||||
|
}
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(response, req)
|
||||||
|
if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) {
|
||||||
|
t.Fatalf("%s: SSE-C GetObject returned %d with %d bytes, want 200 with %d bytes",
|
||||||
|
instanceType, response.Code, response.Body.Len(), len(data))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPICopyObjectServerSideChecksumSourceVariants(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPICopyObjectServerSideChecksumSourceVariants,
|
||||||
|
endpoints: []string{
|
||||||
|
"NewMultipart", "PutObjectPart", "CompleteMultipart", "ListObjectParts",
|
||||||
|
"CopyObject", "PutObject", "HeadObject", "GetObject",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
restoreCompression := setCopyChecksumCompression(true)
|
||||||
|
defer restoreCompression()
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("source-variant-plaintext-"), 64*1024)
|
||||||
|
|
||||||
|
t.Run("compressed-source", func(t *testing.T) {
|
||||||
|
source := "copy-checksum/compressed-source.txt"
|
||||||
|
putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, nil)
|
||||||
|
if info, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{}); err != nil || !info.IsCompressed() {
|
||||||
|
t.Fatalf("%s: compressed source precondition failed: compressed=%v err=%v", instanceType, info.IsCompressed(), err)
|
||||||
|
}
|
||||||
|
destination := "copy-checksum/compressed-source-copy.txt"
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination,
|
||||||
|
map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil)
|
||||||
|
|
||||||
|
rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, source, map[string]string{
|
||||||
|
xhttp.AmzChecksumAlgo: hash.ChecksumSHA256.String(),
|
||||||
|
xhttp.AmzMetadataDirective: "REPLACE",
|
||||||
|
})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: in-place CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumSHA256, data)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, source, hash.ChecksumSHA256, data, true, nil)
|
||||||
|
if got := readCopyChecksumObject(t, obj, bucketName, source, ObjectOptions{}); !bytes.Equal(got, data) {
|
||||||
|
t.Fatalf("%s: in-place CopyObject body differs", instanceType)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("full-checksum-source", func(t *testing.T) {
|
||||||
|
source := "copy-checksum/full-checksum-source.bin"
|
||||||
|
want := mustChecksum(t, hash.ChecksumCRC32, data)
|
||||||
|
putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data,
|
||||||
|
map[string]string{xhttp.AmzChecksumCRC32: want})
|
||||||
|
destination := "copy-checksum/full-checksum-copy.txt"
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("multipart-composite-source", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
parts, full := multipartChecksumTestData()
|
||||||
|
source := "copy-checksum/multipart-source.bin"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||||
|
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, source, uploadID, typ, parts)
|
||||||
|
partChecksums := make([]string, len(parts))
|
||||||
|
for i, part := range parts {
|
||||||
|
partChecksums[i] = mustChecksum(t, typ, part)
|
||||||
|
}
|
||||||
|
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source, uploadID,
|
||||||
|
etags, partChecksums, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
sourceInfo, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: source GetObjectInfo failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
if _, multipart := sourceInfo.decryptChecksums(0, nil); !multipart {
|
||||||
|
t.Fatalf("%s: source checksum is not multipart composite", instanceType)
|
||||||
|
}
|
||||||
|
|
||||||
|
destination := "copy-checksum/multipart-copy.txt"
|
||||||
|
rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, typ, full)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, typ, full, true, nil)
|
||||||
|
if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, full) {
|
||||||
|
t.Fatalf("%s: multipart source round-trip body differs", instanceType)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, boundary := range []struct {
|
||||||
|
name string
|
||||||
|
data []byte
|
||||||
|
compressed bool
|
||||||
|
}{
|
||||||
|
{name: "at-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize)},
|
||||||
|
{name: "over-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize+1), compressed: true},
|
||||||
|
{name: "indexed", data: bytes.Repeat([]byte{'a'}, compMinIndexSize+1), compressed: true},
|
||||||
|
{name: "empty", data: nil},
|
||||||
|
} {
|
||||||
|
t.Run(boundary.name, func(t *testing.T) {
|
||||||
|
source := "copy-checksum/" + boundary.name + "-source.bin"
|
||||||
|
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||||
|
mustGetPutObjReader(t, bytes.NewReader(boundary.data), int64(len(boundary.data)), "", ""), ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
destination := "copy-checksum/" + boundary.name + "-copy.txt"
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination,
|
||||||
|
map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, boundary.data)
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, boundary.data, boundary.compressed, nil)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutObjectRejectsMissingServerSideChecksum(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testPutObjectRejectsMissingServerSideChecksum,
|
||||||
|
endpoints: []string{"PutObject"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testPutObjectRejectsMissingServerSideChecksum(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
data := []byte("the object layer must not silently omit a requested checksum")
|
||||||
|
for _, test := range []struct {
|
||||||
|
name string
|
||||||
|
hasherType hash.ChecksumType
|
||||||
|
}{
|
||||||
|
{name: "missing"},
|
||||||
|
{name: "mismatched", hasherType: hash.ChecksumCRC32C},
|
||||||
|
} {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
object := "copy-checksum/" + test.name + "-server-side-checksum"
|
||||||
|
reader := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "")
|
||||||
|
if test.hasherType.IsSet() {
|
||||||
|
reader.AddServerSideChecksumHasher(test.hasherType)
|
||||||
|
}
|
||||||
|
_, err := obj.PutObject(t.Context(), bucketName, object, reader,
|
||||||
|
ObjectOptions{WantServerSideChecksumType: hash.ChecksumCRC32})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "server-side checksum") {
|
||||||
|
t.Fatalf("%s: PutObject error %v, want server-side checksum invariant error", instanceType, err)
|
||||||
|
}
|
||||||
|
if _, err = obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||||
|
t.Fatalf("%s: failed PutObject left an object behind: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/hash"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAPICopyObjectMetadataOnlyCompression(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
for _, versioned := range []bool{false, true} {
|
||||||
|
name := "unversioned"
|
||||||
|
if versioned {
|
||||||
|
name = "versioned"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPICopyObjectMetadataOnlyCompression,
|
||||||
|
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
|
||||||
|
makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPICopyObjectMetadataOnlyCompression(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
data := bytes.Repeat([]byte("metadata-only-copy-plaintext-"), 64*1024)
|
||||||
|
want := mustChecksum(t, hash.ChecksumCRC32, data)
|
||||||
|
object := "copy-metadata/existing-checksum.txt"
|
||||||
|
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data,
|
||||||
|
map[string]string{xhttp.AmzChecksumCRC32: want})
|
||||||
|
before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil || before.IsCompressed() {
|
||||||
|
t.Fatalf("%s: invalid metadata-copy precondition: compressed=%v size=%d err=%v",
|
||||||
|
instanceType, before.IsCompressed(), before.Size, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
restoreCompression := setCopyChecksumCompression(true)
|
||||||
|
compressionRestored := false
|
||||||
|
defer func() {
|
||||||
|
if !compressionRestored {
|
||||||
|
restoreCompression()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object,
|
||||||
|
map[string]string{xhttp.AmzMetadataDirective: "REPLACE"})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, false, nil)
|
||||||
|
if got := readCopyChecksumObject(t, obj, bucketName, object, ObjectOptions{}); !bytes.Equal(got, data) {
|
||||||
|
prefix := got
|
||||||
|
if len(prefix) > 100 {
|
||||||
|
prefix = prefix[:100]
|
||||||
|
}
|
||||||
|
t.Fatalf("%s: metadata-only CopyObject body differs: got %d bytes, want %d, prefix %q",
|
||||||
|
instanceType, len(got), len(data), prefix)
|
||||||
|
}
|
||||||
|
afterMetadataCopy, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if before.VersionID != "" && afterMetadataCopy.VersionID == before.VersionID {
|
||||||
|
t.Fatalf("%s: versioned metadata-only copy did not create a new version", instanceType)
|
||||||
|
}
|
||||||
|
|
||||||
|
destination := "copy-metadata/rewritten.txt"
|
||||||
|
rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, object, destination, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: data-rewriting CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil)
|
||||||
|
|
||||||
|
compressedObject := "copy-metadata/preserve-compressed.txt"
|
||||||
|
putCopyChecksumSource(t, apiRouter, credentials, bucketName, compressedObject, data,
|
||||||
|
map[string]string{xhttp.AmzChecksumCRC32: want})
|
||||||
|
assertCopyChecksum(t, obj, bucketName, compressedObject, hash.ChecksumCRC32, data, true, nil)
|
||||||
|
|
||||||
|
restoreCompression()
|
||||||
|
compressionRestored = true
|
||||||
|
rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, compressedObject, compressedObject,
|
||||||
|
map[string]string{xhttp.AmzMetadataDirective: "REPLACE"})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: compressed metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
assertCopyChecksum(t, obj, bucketName, compressedObject, hash.ChecksumCRC32, data, true, nil)
|
||||||
|
if got := readCopyChecksumObject(t, obj, bucketName, compressedObject, ObjectOptions{}); !bytes.Equal(got, data) {
|
||||||
|
t.Fatalf("%s: compressed metadata-only CopyObject body differs", instanceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPICopyObjectSSECKeyRotationKeepsCompressionState(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
for _, versioned := range []bool{false, true} {
|
||||||
|
name := "unversioned"
|
||||||
|
if versioned {
|
||||||
|
name = "versioned"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPICopyObjectSSECKeyRotationKeepsCompressionState,
|
||||||
|
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
|
||||||
|
makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPICopyObjectSSECKeyRotationKeepsCompressionState(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousTLS := globalIsTLS
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = previousTLS }()
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("key-rotation-plaintext-"), 64*1024)
|
||||||
|
object := "copy-metadata/key-rotation.txt"
|
||||||
|
oldKey := bytes.Repeat([]byte{0x11}, 32)
|
||||||
|
oldMD5 := md5.Sum(oldKey)
|
||||||
|
newKey := bytes.Repeat([]byte{0x22}, 32)
|
||||||
|
newMD5 := md5.Sum(newKey)
|
||||||
|
|
||||||
|
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data, map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
|
||||||
|
})
|
||||||
|
before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil || before.IsCompressed() {
|
||||||
|
t.Fatalf("%s: invalid key-rotation precondition: compressed=%v err=%v", instanceType, before.IsCompressed(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
restoreCompression := setCopyChecksumCompression(true)
|
||||||
|
defer restoreCompression()
|
||||||
|
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]),
|
||||||
|
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
|
||||||
|
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
|
||||||
|
})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: key rotation failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
after, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if after.IsCompressed() {
|
||||||
|
t.Fatalf("%s: metadata-only key rotation stamped compression metadata", instanceType)
|
||||||
|
}
|
||||||
|
if before.VersionID != "" && after.VersionID == before.VersionID {
|
||||||
|
t.Fatalf("%s: versioned key rotation did not create a new version", instanceType)
|
||||||
|
}
|
||||||
|
|
||||||
|
getHeaders := map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]),
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build GetObject request: %v", err)
|
||||||
|
}
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(response, req)
|
||||||
|
if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) {
|
||||||
|
t.Fatalf("%s: post-rotation GetObject returned %d with %d bytes, want 200 with %d bytes: %s",
|
||||||
|
instanceType, response.Code, response.Body.Len(), len(data), response.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
+51
-13
@@ -1122,6 +1122,12 @@ func getRemoteInstanceTransport() http.RoundTripper {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// federatedInternalAppName is the minio-go application token that
|
||||||
|
// getRemoteInstanceClient attaches to every legacy federation proxy request. It
|
||||||
|
// is declared next to its only producer so that the literal keeps its historical
|
||||||
|
// file attribution in the rebrand compatibility baseline.
|
||||||
|
const federatedInternalAppName = "minio-federated"
|
||||||
|
|
||||||
// Returns a minio-go Client configured to access remote host described by destDNSRecord
|
// Returns a minio-go Client configured to access remote host described by destDNSRecord
|
||||||
// Applicable only in a federated deployment
|
// Applicable only in a federated deployment
|
||||||
var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) {
|
var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) {
|
||||||
@@ -1136,7 +1142,7 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
core.SetAppInfo("minio-federated", ReleaseTag)
|
core.SetAppInfo(federatedInternalAppName, ReleaseTag)
|
||||||
return core, nil
|
return core, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1357,6 +1363,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
} // no changes in storage-class expected so its a metadataonly operation.
|
} // no changes in storage-class expected so its a metadataonly operation.
|
||||||
|
|
||||||
var reader io.Reader = gr
|
var reader io.Reader = gr
|
||||||
|
sourceCompressMetadata := make(map[string]string, 2)
|
||||||
|
for _, key := range []string{
|
||||||
|
ReservedMetadataPrefix + "compression",
|
||||||
|
ReservedMetadataPrefix + "actual-size",
|
||||||
|
} {
|
||||||
|
if value, ok := srcInfo.UserDefined[key]; ok {
|
||||||
|
sourceCompressMetadata[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Set the actual size to the compressed/decrypted size if encrypted.
|
// Set the actual size to the compressed/decrypted size if encrypted.
|
||||||
actualSize, err := srcInfo.GetActualSize()
|
actualSize, err := srcInfo.GetActualSize()
|
||||||
@@ -1386,15 +1401,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
|
compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
|
||||||
|
|
||||||
reader = etag.NewReader(ctx, reader, nil, nil)
|
reader = etag.NewReader(ctx, reader, nil, nil)
|
||||||
wantEncryption := crypto.Requested(r.Header)
|
|
||||||
s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption)
|
|
||||||
dstOpts.IndexCB = cb
|
|
||||||
defer s2c.Close()
|
|
||||||
reader = etag.Wrap(s2c, reader)
|
|
||||||
length = -1
|
|
||||||
} else {
|
} else {
|
||||||
delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression")
|
|
||||||
delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size")
|
|
||||||
reader = gr
|
reader = gr
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1541,6 +1548,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if isDstCompressed {
|
||||||
|
checksumReader := srcInfo.Reader
|
||||||
|
wantEncryption := crypto.Requested(r.Header)
|
||||||
|
s2c, cb := newS2CompressReader(checksumReader, actualSize, wantEncryption)
|
||||||
|
dstOpts.IndexCB = cb
|
||||||
|
defer s2c.Close()
|
||||||
|
reader = etag.Wrap(s2c, checksumReader)
|
||||||
|
srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// The storage reader consumes compressed data; checksums remain bound to plaintext.
|
||||||
|
pReader = NewPutObjReader(srcInfo.Reader)
|
||||||
|
pReader.setChecksumReader(checksumReader)
|
||||||
|
}
|
||||||
|
|
||||||
if isTargetEncrypted {
|
if isTargetEncrypted {
|
||||||
var encReader io.Reader
|
var encReader io.Reader
|
||||||
kind, _ := crypto.IsRequested(r.Header)
|
kind, _ := crypto.IsRequested(r.Header)
|
||||||
@@ -1676,8 +1700,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
|
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
|
||||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||||
}
|
}
|
||||||
// Store the preserved compression metadata.
|
// Compression metadata must describe data that is actually rewritten.
|
||||||
maps.Copy(srcInfo.UserDefined, compressMetadata)
|
if !srcInfo.metadataOnly || srcInfo.Legacy || dstOpts.WantServerSideChecksumType.IsSet() {
|
||||||
|
if isDstCompressed {
|
||||||
|
maps.Copy(srcInfo.UserDefined, compressMetadata)
|
||||||
|
} else {
|
||||||
|
delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression")
|
||||||
|
delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
maps.Copy(srcInfo.UserDefined, sourceCompressMetadata)
|
||||||
|
}
|
||||||
|
|
||||||
// We need to preserve the encryption headers set in EncryptRequest,
|
// We need to preserve the encryption headers set in EncryptRequest,
|
||||||
// so we do not want to override them, copy them instead.
|
// so we do not want to override them, copy them instead.
|
||||||
@@ -1767,9 +1800,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
|
|
||||||
copyObjectFn := objectAPI.CopyObject
|
copyObjectFn := objectAPI.CopyObject
|
||||||
|
|
||||||
|
copySrcOpts := srcOpts
|
||||||
|
if srcInfo.metadataOnly && dstOpts.Versioned && copySrcOpts.VersionID == "" {
|
||||||
|
copySrcOpts.VersionID = srcInfo.VersionID
|
||||||
|
}
|
||||||
|
|
||||||
// Copy source object to destination, if source and destination
|
// Copy source object to destination, if source and destination
|
||||||
// object is same then only metadata is updated.
|
// object is same then only metadata is updated.
|
||||||
objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
@@ -1778,7 +1816,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
|||||||
|
|
||||||
origETag := objInfo.ETag
|
origETag := objInfo.ETag
|
||||||
objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
|
objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
|
||||||
response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
|
response := generateCopyObjectResponse(objInfo, r.Header)
|
||||||
encodedSuccessResponse := encodeResponse(response)
|
encodedSuccessResponse := encodeResponse(response)
|
||||||
|
|
||||||
if dsc := mustReplicate(ctx, dstBucket, dstObject, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, dstOpts)); dsc.ReplicateAny() {
|
if dsc := mustReplicate(ctx, dstBucket, dstObject, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, dstOpts)); dsc.ReplicateAny() {
|
||||||
|
|||||||
@@ -0,0 +1,367 @@
|
|||||||
|
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||||
|
// Copyright (c) 2025-2026 PGSTY
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"encoding/xml"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
miniogo "github.com/minio/minio-go/v7"
|
||||||
|
miniocredentials "github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
|
"github.com/minio/minio-go/v7/pkg/set"
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/config/dns"
|
||||||
|
"github.com/minio/minio/internal/hash"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
const federatedTestUserAgent = "MinIO (linux; amd64) minio-go/v7.0.99 minio-federated/RELEASE.TEST"
|
||||||
|
|
||||||
|
func TestAPIFederatedUploadPartChecksumResponse(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIFederatedUploadPartChecksumResponse,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIFederatedUploadPartChecksumResponse(_ ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
algorithms := []struct {
|
||||||
|
name string
|
||||||
|
typ hash.ChecksumType
|
||||||
|
checksumType string
|
||||||
|
}{
|
||||||
|
{name: "crc32-full-object", typ: hash.ChecksumCRC32, checksumType: xhttp.AmzChecksumTypeFullObject},
|
||||||
|
{name: "sha256-composite", typ: hash.ChecksumSHA256, checksumType: xhttp.AmzChecksumTypeComposite},
|
||||||
|
}
|
||||||
|
userAgents := []struct {
|
||||||
|
name string
|
||||||
|
ua string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "absent"},
|
||||||
|
{name: "ordinary-sdk", ua: "aws-sdk-go/1.55.5"},
|
||||||
|
{name: "federation", ua: federatedTestUserAgent, want: true},
|
||||||
|
{name: "lookalike-prefix", ua: "evil-minio-federated/RELEASE.TEST"},
|
||||||
|
{name: "lookalike-suffix", ua: "minio-federated-extra/RELEASE.TEST"},
|
||||||
|
{name: "missing-version", ua: "minio-federated"},
|
||||||
|
{name: "empty-version", ua: "minio-federated/"},
|
||||||
|
}
|
||||||
|
data := []byte("federated upload part checksum response")
|
||||||
|
|
||||||
|
for _, algorithm := range algorithms {
|
||||||
|
for _, userAgent := range userAgents {
|
||||||
|
t.Run(algorithm.name+"/"+userAgent.name, func(t *testing.T) {
|
||||||
|
object := "federation/response/" + algorithm.name + "/" + userAgent.name
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
algorithm.typ.String(), algorithm.checksumType)
|
||||||
|
headers := map[string]string{}
|
||||||
|
if userAgent.ua != "" {
|
||||||
|
headers["User-Agent"] = userAgent.ua
|
||||||
|
}
|
||||||
|
_, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, data, headers)
|
||||||
|
|
||||||
|
got := rec.Header().Get(algorithm.typ.Key())
|
||||||
|
if userAgent.want {
|
||||||
|
if want := mustChecksum(t, algorithm.typ, data); got != want {
|
||||||
|
t.Fatalf("%s: checksum %q, want %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
} else if got != "" {
|
||||||
|
t.Fatalf("%s: ordinary UploadPart exposed server checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
if got := rec.Header().Get(xhttp.AmzChecksumType); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned checksum type %q", instanceType, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIFederatedUploadPartChecksumMinIOGoWire(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIFederatedUploadPartChecksumMinIOGoWire,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIFederatedUploadPartChecksumMinIOGoWire(_ ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
server := httptest.NewServer(apiRouter)
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
core, err := miniogo.NewCore(server.Listener.Addr().String(), &miniogo.Options{
|
||||||
|
Creds: miniocredentials.NewStaticV4(credentials.AccessKey, credentials.SecretKey, ""),
|
||||||
|
Secure: false,
|
||||||
|
Region: globalMinioDefaultRegion,
|
||||||
|
BucketLookup: miniogo.BucketLookupPath,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: create minio-go Core: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
core.SetAppInfo("minio-federated", ReleaseTag)
|
||||||
|
|
||||||
|
object := "federation/minio-go-wire"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
data := []byte("minio-go must parse the remote computed checksum")
|
||||||
|
part, err := core.PutObjectPart(t.Context(), bucketName, object, uploadID, 1,
|
||||||
|
bytes.NewReader(data), int64(len(data)), miniogo.PutObjectPartOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: minio-go PutObjectPart: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
if want := mustChecksum(t, hash.ChecksumCRC32, data); part.ChecksumCRC32 != want {
|
||||||
|
t.Fatalf("%s: minio-go checksum %q, want %q", instanceType, part.ChecksumCRC32, want)
|
||||||
|
}
|
||||||
|
if part.ETag == "" {
|
||||||
|
t.Fatalf("%s: minio-go returned an empty ETag", instanceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIFederatedUploadPartChecksumConcurrentOverwrite(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIFederatedUploadPartChecksumConcurrentOverwrite,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIFederatedUploadPartChecksumConcurrentOverwrite(_ ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
object := "federation/concurrent-overwrite"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
hash.ChecksumSHA256.String(), xhttp.AmzChecksumTypeComposite)
|
||||||
|
data := [][]byte{
|
||||||
|
bytes.Repeat([]byte("first-writer-"), 4096),
|
||||||
|
bytes.Repeat([]byte("second-writer-"), 4096),
|
||||||
|
}
|
||||||
|
reqs := make([]*http.Request, len(data))
|
||||||
|
recorders := make([]*httptest.ResponseRecorder, len(data))
|
||||||
|
for i := range data {
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
|
||||||
|
int64(len(data[i])), bytes.NewReader(data[i]), credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{"User-Agent": federatedTestUserAgent})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: build concurrent request %d: %v", instanceType, i, err)
|
||||||
|
}
|
||||||
|
reqs[i] = req
|
||||||
|
recorders[i] = httptest.NewRecorder()
|
||||||
|
}
|
||||||
|
|
||||||
|
start := make(chan struct{})
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := range reqs {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
<-start
|
||||||
|
apiRouter.ServeHTTP(recorders[i], reqs[i])
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
close(start)
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
for i, rec := range recorders {
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: concurrent request %d failed: %d %s", instanceType, i, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
got := rec.Header().Get(hash.ChecksumSHA256.Key())
|
||||||
|
if want := mustChecksum(t, hash.ChecksumSHA256, data[i]); got != want {
|
||||||
|
t.Fatalf("%s: concurrent request %d checksum %q, want %q", instanceType, i, got, want)
|
||||||
|
}
|
||||||
|
// The ETag and the checksum must describe the same write, so a losing
|
||||||
|
// writer can never publish the winner's checksum next to its own ETag.
|
||||||
|
etags := rec.Header()[xhttp.ETag]
|
||||||
|
if len(etags) != 1 {
|
||||||
|
t.Fatalf("%s: concurrent request %d returned %d ETags", instanceType, i, len(etags))
|
||||||
|
}
|
||||||
|
md5sum := md5.Sum(data[i])
|
||||||
|
if want := hex.EncodeToString(md5sum[:]); canonicalizeETag(etags[0]) != want {
|
||||||
|
t.Fatalf("%s: concurrent request %d ETag %q, want %q", instanceType, i, etags[0], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// federationTestDNS is a minimal dns.Store so a single test process can play
|
||||||
|
// both federation roles.
|
||||||
|
type federationTestDNS struct {
|
||||||
|
records map[string][]dns.SrvRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f federationTestDNS) Put(string) error { return nil }
|
||||||
|
|
||||||
|
func (f federationTestDNS) Get(bucket string) ([]dns.SrvRecord, error) {
|
||||||
|
records, ok := f.records[bucket]
|
||||||
|
if !ok {
|
||||||
|
return nil, dns.ErrNoEntriesFound
|
||||||
|
}
|
||||||
|
return records, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f federationTestDNS) Delete(string) error { return nil }
|
||||||
|
func (f federationTestDNS) List() (map[string][]dns.SrvRecord, error) { return f.records, nil }
|
||||||
|
func (f federationTestDNS) DeleteRecord(dns.SrvRecord) error { return nil }
|
||||||
|
func (f federationTestDNS) Close() error { return nil }
|
||||||
|
func (f federationTestDNS) String() string { return "federation-test-dns" }
|
||||||
|
|
||||||
|
// remoteBucketObjectLayer reports one existing bucket as missing so that
|
||||||
|
// isRemoteCopyRequired takes the legacy federation branch while the same
|
||||||
|
// process can still serve that bucket as the remote deployment.
|
||||||
|
type remoteBucketObjectLayer struct {
|
||||||
|
ObjectLayer
|
||||||
|
remoteBucket string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l remoteBucketObjectLayer) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
|
||||||
|
if bucket == l.remoteBucket {
|
||||||
|
return BucketInfo{}, toObjectErr(errVolumeNotFound, bucket)
|
||||||
|
}
|
||||||
|
return l.ObjectLayer.GetBucketInfo(ctx, bucket, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPIFederatedCopyObjectPartChecksum drives the legacy etcd federation
|
||||||
|
// branch of CopyObjectPartHandler end to end: the proxy forwards the copied
|
||||||
|
// bytes through the real getRemoteInstanceClient and minio-go, a second HTTP
|
||||||
|
// endpoint serves the real PutObjectPartHandler, and CopyPartResult must carry
|
||||||
|
// the checksum computed by that exact remote write.
|
||||||
|
func TestAPIFederatedCopyObjectPartChecksum(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIFederatedCopyObjectPartChecksum,
|
||||||
|
endpoints: []string{
|
||||||
|
"CopyObjectPart", "NewMultipart", "PutObjectPart",
|
||||||
|
"ListObjectParts", "CompleteMultipart", "PutObject",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIFederatedCopyObjectPartChecksum(objectAPI ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
algorithms := []struct {
|
||||||
|
name string
|
||||||
|
typ hash.ChecksumType
|
||||||
|
checksumType string
|
||||||
|
}{
|
||||||
|
{name: "crc32-full-object", typ: hash.ChecksumCRC32, checksumType: xhttp.AmzChecksumTypeFullObject},
|
||||||
|
{name: "sha256-composite", typ: hash.ChecksumSHA256, checksumType: xhttp.AmzChecksumTypeComposite},
|
||||||
|
}
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("federated-upload-part-copy-"), 1024)
|
||||||
|
srcObject := "federation/copy-source.bin"
|
||||||
|
putCopyChecksumSource(t, apiRouter, credentials, bucketName, srcObject, data, nil)
|
||||||
|
|
||||||
|
// The destination bucket really exists so the remote endpoint can serve it;
|
||||||
|
// only the proxy's own bucket lookup is told that it lives elsewhere.
|
||||||
|
remoteBucket := getRandomBucketName()
|
||||||
|
if err := objectAPI.MakeBucket(t.Context(), remoteBucket, MakeBucketOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: unable to create the remote bucket: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
remote := httptest.NewServer(apiRouter)
|
||||||
|
defer remote.Close()
|
||||||
|
host, port, _ := strings.Cut(remote.Listener.Addr().String(), ":")
|
||||||
|
|
||||||
|
globalObjLayerMutex.Lock()
|
||||||
|
previousLayer := globalObjectAPI
|
||||||
|
globalObjectAPI = remoteBucketObjectLayer{ObjectLayer: previousLayer, remoteBucket: remoteBucket}
|
||||||
|
globalObjLayerMutex.Unlock()
|
||||||
|
previousDNS, previousFederation, previousIPs := globalDNSConfig, globalBucketFederation, globalDomainIPs
|
||||||
|
globalDNSConfig = federationTestDNS{records: map[string][]dns.SrvRecord{
|
||||||
|
bucketName: {{Host: host, Port: json.Number(port)}},
|
||||||
|
remoteBucket: {{Host: host, Port: json.Number(port)}},
|
||||||
|
}}
|
||||||
|
// Every DNS record resolves to this process, so the bucket forwarding
|
||||||
|
// middleware always serves locally and only the handler proxies.
|
||||||
|
globalDomainIPs = set.CreateStringSet(remote.Listener.Addr().String())
|
||||||
|
globalBucketFederation = true
|
||||||
|
defer func() {
|
||||||
|
globalObjLayerMutex.Lock()
|
||||||
|
globalObjectAPI = previousLayer
|
||||||
|
globalObjLayerMutex.Unlock()
|
||||||
|
globalDNSConfig, globalBucketFederation, globalDomainIPs = previousDNS, previousFederation, previousIPs
|
||||||
|
}()
|
||||||
|
|
||||||
|
for _, algorithm := range algorithms {
|
||||||
|
t.Run(algorithm.name, func(t *testing.T) {
|
||||||
|
object := "federation/copy-destination-" + algorithm.name + ".bin"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, remoteBucket, object,
|
||||||
|
algorithm.typ.String(), algorithm.checksumType)
|
||||||
|
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getCopyObjectPartURL("", remoteBucket, object, uploadID, "1"),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{xhttp.AmzCopySource: SlashSeparator + pathJoin(bucketName, srcObject)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: unable to build UploadPartCopy request: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: federated UploadPartCopy failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var response CopyObjectPartResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||||
|
t.Fatalf("%s: unable to decode CopyPartResult: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
want := mustChecksum(t, algorithm.typ, data)
|
||||||
|
if got := copyPartChecksum(algorithm.typ, response); got != want {
|
||||||
|
t.Fatalf("%s: CopyPartResult %s is %q, want %q: %s",
|
||||||
|
instanceType, algorithm.typ.String(), got, want, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// The persisted part must carry the same value, and the client must be
|
||||||
|
// able to complete the upload with what CopyPartResult returned.
|
||||||
|
parts := listPartsHTTP(t, apiRouter, credentials, remoteBucket, object, uploadID, nil)
|
||||||
|
if len(parts.Parts) != 1 {
|
||||||
|
t.Fatalf("%s: ListParts returned %d parts, want 1", instanceType, len(parts.Parts))
|
||||||
|
}
|
||||||
|
if got := partChecksum(algorithm.typ, parts.Parts[0]); got != want {
|
||||||
|
t.Fatalf("%s: persisted part %s is %q, want %q", instanceType, algorithm.typ.String(), got, want)
|
||||||
|
}
|
||||||
|
etag := canonicalizeETag(response.ETag)
|
||||||
|
completed := completePartsHTTP(t, apiRouter, credentials, remoteBucket, object, uploadID,
|
||||||
|
[]CompletePart{completePartWithChecksum(algorithm.typ, 1, etag, want)}, nil)
|
||||||
|
if completed.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload rejected the federated part: %d %s",
|
||||||
|
instanceType, completed.Code, completed.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -55,6 +55,88 @@ import (
|
|||||||
|
|
||||||
// Multipart objectAPIHandlers
|
// Multipart objectAPIHandlers
|
||||||
|
|
||||||
|
// isFederatedInternalRequest reports whether User-Agent carries the minio-go
|
||||||
|
// application token attached by getRemoteInstanceClient.
|
||||||
|
//
|
||||||
|
// This is only a response-shape hint. User-Agent is not authenticated and must
|
||||||
|
// never gate authorization, object visibility, or request validation. It is
|
||||||
|
// safe here because the only effect is returning the checksum of the body the
|
||||||
|
// caller was already authorized to upload.
|
||||||
|
func isFederatedInternalRequest(userAgent string) bool {
|
||||||
|
for _, product := range strings.Fields(userAgent) {
|
||||||
|
name, version, ok := strings.Cut(product, "/")
|
||||||
|
if ok && name == federatedInternalAppName && version != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// partChecksumMap returns the non-empty part checksums in the form expected by
|
||||||
|
// hash.AddChecksumHeader. x-amz-checksum-type is deliberately excluded because
|
||||||
|
// UploadPart does not return it and minio-go cannot carry it in ObjectPart.
|
||||||
|
func partChecksumMap(partInfo PartInfo) map[string]string {
|
||||||
|
checksums := make(map[string]string, 1)
|
||||||
|
if partInfo.ChecksumCRC32 != "" {
|
||||||
|
checksums[hash.ChecksumCRC32.String()] = partInfo.ChecksumCRC32
|
||||||
|
}
|
||||||
|
if partInfo.ChecksumCRC32C != "" {
|
||||||
|
checksums[hash.ChecksumCRC32C.String()] = partInfo.ChecksumCRC32C
|
||||||
|
}
|
||||||
|
if partInfo.ChecksumSHA1 != "" {
|
||||||
|
checksums[hash.ChecksumSHA1.String()] = partInfo.ChecksumSHA1
|
||||||
|
}
|
||||||
|
if partInfo.ChecksumSHA256 != "" {
|
||||||
|
checksums[hash.ChecksumSHA256.String()] = partInfo.ChecksumSHA256
|
||||||
|
}
|
||||||
|
if partInfo.ChecksumCRC64NVME != "" {
|
||||||
|
checksums[hash.ChecksumCRC64NVME.String()] = partInfo.ChecksumCRC64NVME
|
||||||
|
}
|
||||||
|
return checksums
|
||||||
|
}
|
||||||
|
|
||||||
|
// multipartChecksumType returns the base checksum type recorded when a
|
||||||
|
// multipart upload was created. The boolean reports whether an algorithm was
|
||||||
|
// recorded at all.
|
||||||
|
func multipartChecksumType(metadata map[string]string) (hash.ChecksumType, bool) {
|
||||||
|
algorithm := metadata[hash.MinIOMultipartChecksum]
|
||||||
|
if algorithm == "" {
|
||||||
|
return hash.ChecksumNone, false
|
||||||
|
}
|
||||||
|
t := hash.NewChecksumType(algorithm, metadata[hash.MinIOMultipartChecksumType])
|
||||||
|
if !t.IsSet() {
|
||||||
|
return t, true
|
||||||
|
}
|
||||||
|
return t.Base(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// prepareMultipartChecksumReader validates a supplied part checksum algorithm,
|
||||||
|
// or installs a server-side hasher when the client omitted the optional
|
||||||
|
// checksum. It must run before compression or encryption can consume reader.
|
||||||
|
func prepareMultipartChecksumReader(reader *hash.Reader, metadata map[string]string, bucket, object string) error {
|
||||||
|
want, ok := multipartChecksumType(metadata)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
got := reader.ContentCRCType()
|
||||||
|
if !got.IsSet() && reader.ServerSideChecksumType.IsSet() {
|
||||||
|
got = reader.ServerSideChecksumType
|
||||||
|
}
|
||||||
|
if !want.IsSet() || (got.IsSet() && got.Base() != want) {
|
||||||
|
return InvalidArgument{
|
||||||
|
Bucket: bucket,
|
||||||
|
Object: object,
|
||||||
|
Err: fmt.Errorf("checksum missing, want %q, got %q",
|
||||||
|
metadata[hash.MinIOMultipartChecksum], got.String()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !got.IsSet() {
|
||||||
|
reader.AddServerSideChecksumHasher(want)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// NewMultipartUploadHandler - New multipart upload.
|
// NewMultipartUploadHandler - New multipart upload.
|
||||||
// Notice: The S3 client can send secret keys in headers for encryption related jobs,
|
// Notice: The S3 client can send secret keys in headers for encryption related jobs,
|
||||||
// the handler should ensure to remove these keys before sending them to the object layer.
|
// the handler should ensure to remove these keys before sending them to the object layer.
|
||||||
@@ -465,7 +547,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
|
response := generateCopyObjectPartResponse(PartInfo{
|
||||||
|
ETag: partInfo.ETag,
|
||||||
|
LastModified: partInfo.LastModified,
|
||||||
|
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||||
|
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||||
|
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||||
|
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||||
|
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||||
|
})
|
||||||
encodedSuccessResponse := encodeResponse(response)
|
encodedSuccessResponse := encodeResponse(response)
|
||||||
|
|
||||||
// Write success response.
|
// Write success response.
|
||||||
@@ -475,12 +565,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
|
|
||||||
actualPartSize = length
|
actualPartSize = length
|
||||||
var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
|
var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
|
||||||
|
var checksumReader *hash.Reader
|
||||||
|
|
||||||
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
|
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if _, ok := multipartChecksumType(mi.UserDefined); ok {
|
||||||
|
checksumReader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err = prepareMultipartChecksumReader(checksumReader, mi.UserDefined, dstBucket, dstObject); err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
reader = checksumReader
|
||||||
|
}
|
||||||
|
|
||||||
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
||||||
|
|
||||||
@@ -512,6 +615,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
|
|
||||||
rawReader := srcInfo.Reader
|
rawReader := srcInfo.Reader
|
||||||
pReader := NewPutObjReader(rawReader)
|
pReader := NewPutObjReader(rawReader)
|
||||||
|
pReader.setChecksumReader(checksumReader)
|
||||||
|
|
||||||
var objectEncryptionKey crypto.ObjectKey
|
var objectEncryptionKey crypto.ObjectKey
|
||||||
if isEncrypted {
|
if isEncrypted {
|
||||||
@@ -591,7 +695,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
|
partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
|
||||||
}
|
}
|
||||||
|
|
||||||
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
|
response := generateCopyObjectPartResponse(partInfo)
|
||||||
encodedSuccessResponse := encodeResponse(response)
|
encodedSuccessResponse := encodeResponse(response)
|
||||||
|
|
||||||
// Write success response.
|
// Write success response.
|
||||||
@@ -745,6 +849,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
// Read compression metadata preserved in the init multipart for the decision.
|
// Read compression metadata preserved in the init multipart for the decision.
|
||||||
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||||
var idxCb func() []byte
|
var idxCb func() []byte
|
||||||
|
var checksumReader *hash.Reader
|
||||||
if isCompressed {
|
if isCompressed {
|
||||||
actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
|
actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -755,6 +860,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if err = prepareMultipartChecksumReader(actualReader, mi.UserDefined, bucket, object); err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checksumReader = actualReader
|
||||||
|
|
||||||
// Set compression metrics.
|
// Set compression metrics.
|
||||||
wantEncryption := crypto.Requested(r.Header)
|
wantEncryption := crypto.Requested(r.Header)
|
||||||
@@ -791,8 +901,16 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if checksumReader == nil {
|
||||||
|
if err = prepareMultipartChecksumReader(hashReader, mi.UserDefined, bucket, object); err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checksumReader = hashReader
|
||||||
|
}
|
||||||
|
|
||||||
pReader := NewPutObjReader(hashReader)
|
pReader := NewPutObjReader(hashReader)
|
||||||
|
pReader.setChecksumReader(checksumReader)
|
||||||
|
|
||||||
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
||||||
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
|
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
|
||||||
@@ -918,6 +1036,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
// Therefore, we have to set the ETag directly as map entry.
|
// Therefore, we have to set the ETag directly as map entry.
|
||||||
w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""}
|
w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""}
|
||||||
hash.TransferChecksumHeader(w, r)
|
hash.TransferChecksumHeader(w, r)
|
||||||
|
if isFederatedInternalRequest(r.UserAgent()) {
|
||||||
|
// Legacy federation proxies UploadPartCopy through minio-go
|
||||||
|
// Core.PutObjectPart, which can only recover checksums from response
|
||||||
|
// headers. Use the PartInfo returned by this exact write so the ETag and
|
||||||
|
// checksum cannot be mixed with a concurrent overwrite.
|
||||||
|
hash.AddChecksumHeader(w, partChecksumMap(partInfo))
|
||||||
|
}
|
||||||
|
|
||||||
writeSuccessResponseHeadersOnly(w)
|
writeSuccessResponseHeadersOnly(w)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -80,8 +80,6 @@ func setupTestReadDirFiles(t *testing.T) (testResults []result) {
|
|||||||
for i := range 10 {
|
for i := range 10 {
|
||||||
name := fmt.Sprintf("file-%d", i)
|
name := fmt.Sprintf("file-%d", i)
|
||||||
if err := os.WriteFile(filepath.Join(dir, name), []byte{}, os.ModePerm); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, name), []byte{}, os.ModePerm); err != nil {
|
||||||
// For cleanup, its required to add these entries into test results.
|
|
||||||
testResults = append(testResults, result{dir, entries})
|
|
||||||
t.Fatalf("Unable to create file, %s", err)
|
t.Fatalf("Unable to create file, %s", err)
|
||||||
}
|
}
|
||||||
entries = append(entries, name)
|
entries = append(entries, name)
|
||||||
@@ -105,8 +103,6 @@ func setupTestReadDirGeneric(t *testing.T) (testResults []result) {
|
|||||||
for i := range 10 {
|
for i := range 10 {
|
||||||
name := fmt.Sprintf("file-%d", i)
|
name := fmt.Sprintf("file-%d", i)
|
||||||
if err := os.WriteFile(filepath.Join(dir, "mydir", name), []byte{}, os.ModePerm); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, "mydir", name), []byte{}, os.ModePerm); err != nil {
|
||||||
// For cleanup, its required to add these entries into test results.
|
|
||||||
testResults = append(testResults, result{dir, entries})
|
|
||||||
t.Fatalf("Unable to write file, %s", err)
|
t.Fatalf("Unable to write file, %s", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -130,8 +126,6 @@ func setupTestReadDirSymlink(t *testing.T) (testResults []result) {
|
|||||||
name1 := fmt.Sprintf("file-%d", i)
|
name1 := fmt.Sprintf("file-%d", i)
|
||||||
name2 := fmt.Sprintf("file-%d", i+10)
|
name2 := fmt.Sprintf("file-%d", i+10)
|
||||||
if err := os.WriteFile(filepath.Join(dir, name1), []byte{}, os.ModePerm); err != nil {
|
if err := os.WriteFile(filepath.Join(dir, name1), []byte{}, os.ModePerm); err != nil {
|
||||||
// For cleanup, its required to add these entries into test results.
|
|
||||||
testResults = append(testResults, result{dir, entries})
|
|
||||||
t.Fatalf("Unable to create a file, %s", err)
|
t.Fatalf("Unable to create a file, %s", err)
|
||||||
}
|
}
|
||||||
// Symlink will not be added to entries.
|
// Symlink will not be added to entries.
|
||||||
|
|||||||
+15
-3
@@ -48,6 +48,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/color"
|
"github.com/minio/minio/internal/color"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
"github.com/minio/minio/internal/config/api"
|
"github.com/minio/minio/internal/config/api"
|
||||||
|
"github.com/minio/minio/internal/config/notify"
|
||||||
"github.com/minio/minio/internal/handlers"
|
"github.com/minio/minio/internal/handlers"
|
||||||
"github.com/minio/minio/internal/hash/sha256"
|
"github.com/minio/minio/internal/hash/sha256"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
@@ -535,6 +536,15 @@ func configRetriableErrors(err error) bool {
|
|||||||
notInitialized
|
notInitialized
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func fatalServerConfigError(err error) bool {
|
||||||
|
var configErr config.Err
|
||||||
|
if errors.As(err, &configErr) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
var migrationErr *notify.LegacyDatabaseTargetError
|
||||||
|
return errors.As(err, &migrationErr)
|
||||||
|
}
|
||||||
|
|
||||||
func bootstrapTraceMsg(msg string) {
|
func bootstrapTraceMsg(msg string) {
|
||||||
info := madmin.TraceInfo{
|
info := madmin.TraceInfo{
|
||||||
TraceType: madmin.TraceBootstrap,
|
TraceType: madmin.TraceBootstrap,
|
||||||
@@ -632,7 +642,10 @@ func initConfigSubsystem(ctx context.Context, newObject ObjectLayer) error {
|
|||||||
|
|
||||||
// Initialize config system.
|
// Initialize config system.
|
||||||
if err := globalConfigSys.Init(newObject); err != nil {
|
if err := globalConfigSys.Init(newObject); err != nil {
|
||||||
if configRetriableErrors(err) {
|
var migrationErr *notify.LegacyDatabaseTargetError
|
||||||
|
// Do not use fatalServerConfigError here: existing config.Err values
|
||||||
|
// retain the historical log-and-continue behavior at this boundary.
|
||||||
|
if configRetriableErrors(err) || errors.As(err, &migrationErr) {
|
||||||
return fmt.Errorf("Unable to initialize config system: %w", err)
|
return fmt.Errorf("Unable to initialize config system: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -965,10 +978,9 @@ func serverMain(ctx *cli.Context) {
|
|||||||
var err error
|
var err error
|
||||||
bootstrapTrace("initServerConfig", func() {
|
bootstrapTrace("initServerConfig", func() {
|
||||||
if err = initServerConfig(GlobalContext, newObject); err != nil {
|
if err = initServerConfig(GlobalContext, newObject); err != nil {
|
||||||
var cerr config.Err
|
|
||||||
// For any config error, we don't need to drop into safe-mode
|
// For any config error, we don't need to drop into safe-mode
|
||||||
// instead its a user error and should be fixed by user.
|
// instead its a user error and should be fixed by user.
|
||||||
if errors.As(err, &cerr) {
|
if fatalServerConfigError(err) {
|
||||||
logger.FatalIf(err, "Unable to initialize the server")
|
logger.FatalIf(err, "Unable to initialize the server")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -46,7 +46,9 @@ func oldLinux() bool {
|
|||||||
|
|
||||||
func setMaxResources(ctx serverCtxt) (err error) {
|
func setMaxResources(ctx serverCtxt) (err error) {
|
||||||
// Set the Go runtime max threads threshold to 90% of kernel setting.
|
// Set the Go runtime max threads threshold to 90% of kernel setting.
|
||||||
|
//nolint:staticcheck // Linux implementations can fail; BSD stubs return a constant nil error.
|
||||||
sysMaxThreads, err := sys.GetMaxThreads()
|
sysMaxThreads, err := sys.GetMaxThreads()
|
||||||
|
//nolint:staticcheck // Keep the shared cross-platform error handling.
|
||||||
if err == nil {
|
if err == nil {
|
||||||
minioMaxThreads := (sysMaxThreads * 90) / 100
|
minioMaxThreads := (sysMaxThreads * 90) / 100
|
||||||
// Only set max threads if it is greater than the default one
|
// Only set max threads if it is greater than the default one
|
||||||
|
|||||||
@@ -3734,7 +3734,7 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
|
|||||||
bms.ExpiryLCConfig = &expLclCfgStr
|
bms.ExpiryLCConfig = &expLclCfgStr
|
||||||
// if all non expiry rules only, ExpiryUpdatedAt would be nil
|
// if all non expiry rules only, ExpiryUpdatedAt would be nil
|
||||||
if meta.lifecycleConfig.ExpiryUpdatedAt != nil {
|
if meta.lifecycleConfig.ExpiryUpdatedAt != nil {
|
||||||
bms.ExpiryLCConfigUpdatedAt = *(meta.lifecycleConfig.ExpiryUpdatedAt)
|
bms.ExpiryLCConfigUpdatedAt = *meta.lifecycleConfig.ExpiryUpdatedAt
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) {
|
|||||||
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
|
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
|
||||||
return o, err
|
return o, err
|
||||||
}
|
}
|
||||||
bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:])
|
bts, err = msgp.ReadExactBytes(bts, x.VersionID[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "VersionID")
|
err = msgp.WrapError(err, "VersionID")
|
||||||
return o, err
|
return o, err
|
||||||
@@ -145,7 +145,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error)
|
|||||||
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
|
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
|
||||||
return o, err
|
return o, err
|
||||||
}
|
}
|
||||||
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
|
bts, err = msgp.ReadExactBytes(bts, z.VersionID[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "VersionID")
|
err = msgp.WrapError(err, "VersionID")
|
||||||
return o, err
|
return o, err
|
||||||
@@ -155,7 +155,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error)
|
|||||||
err = msgp.WrapError(err, "ModTime")
|
err = msgp.WrapError(err, "ModTime")
|
||||||
return o, err
|
return o, err
|
||||||
}
|
}
|
||||||
bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
|
bts, err = msgp.ReadExactBytes(bts, z.Signature[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Signature")
|
err = msgp.WrapError(err, "Signature")
|
||||||
return o, err
|
return o, err
|
||||||
@@ -195,7 +195,7 @@ func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
|
err = msgp.ArrayError{Wanted: 5, Got: zb0001}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
err = dc.ReadExactBytes((z.VersionID)[:])
|
err = dc.ReadExactBytes(z.VersionID[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "VersionID")
|
err = msgp.WrapError(err, "VersionID")
|
||||||
return err
|
return err
|
||||||
@@ -205,7 +205,7 @@ func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
err = msgp.WrapError(err, "ModTime")
|
err = msgp.WrapError(err, "ModTime")
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
err = dc.ReadExactBytes((z.Signature)[:])
|
err = dc.ReadExactBytes(z.Signature[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Signature")
|
err = msgp.WrapError(err, "Signature")
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -1101,7 +1101,7 @@ func (x *xlMetaV2) loadLegacy(buf []byte) error {
|
|||||||
return msgp.WrapError(err, "Versions")
|
return msgp.WrapError(err, "Versions")
|
||||||
}
|
}
|
||||||
if cap(x.versions) >= int(zb0002) {
|
if cap(x.versions) >= int(zb0002) {
|
||||||
x.versions = (x.versions)[:zb0002]
|
x.versions = x.versions[:zb0002]
|
||||||
} else {
|
} else {
|
||||||
x.versions = make([]xlMetaV2ShallowVersion, zb0002, zb0002+1)
|
x.versions = make([]xlMetaV2ShallowVersion, zb0002, zb0002+1)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -840,12 +840,11 @@ Received a message: {"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jp
|
|||||||
> database (string) database name (used only if `connection_string` is empty)
|
> database (string) database name (used only if `connection_string` is empty)
|
||||||
> ```
|
> ```
|
||||||
>
|
>
|
||||||
> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure
|
> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to
|
||||||
> to migrate to only using _connection_string_ option. To migrate, once you have upgraded all the servers use the
|
> _connection_string_ before starting SILO. On the old server, use the following command to update the target.
|
||||||
> following command to update the existing notification targets.
|
|
||||||
>
|
>
|
||||||
> ```
|
> ```
|
||||||
> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 username=psqluser password=psqlpass database=bucketevents"
|
> mc admin config set mysilo/ notify_postgres[:name] connection_string="host=hostname port=2832 user=psqluser password=psqlpass dbname=bucketevents"
|
||||||
> ```
|
> ```
|
||||||
>
|
>
|
||||||
> Please make sure this step is carried out, without this step PostgreSQL notification targets will not work,
|
> Please make sure this step is carried out, without this step PostgreSQL notification targets will not work,
|
||||||
@@ -973,9 +972,8 @@ key | value
|
|||||||
> database (string) database name (used only if `dsn_string` is empty)
|
> database (string) database name (used only if `dsn_string` is empty)
|
||||||
> ```
|
> ```
|
||||||
>
|
>
|
||||||
> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure
|
> These are now deprecated. SILO does not migrate an enabled target that only has these fields, so convert it to
|
||||||
> to migrate to only using _dsn_string_ option. To migrate, once you have upgraded all the servers use the
|
> _dsn_string_ before starting SILO. On the old server, use the following command to update the target.
|
||||||
> following command to update the existing notification targets.
|
|
||||||
>
|
>
|
||||||
> ```
|
> ```
|
||||||
> mc admin config set mysilo/ notify_mysql[:name] dsn_string="mysqluser:mysqlpass@tcp(localhost:2832)/bucketevents"
|
> mc admin config set mysilo/ notify_mysql[:name] dsn_string="mysqluser:mysqlpass@tcp(localhost:2832)/bucketevents"
|
||||||
@@ -1045,7 +1043,7 @@ Before updating the configuration, let's start with `mc admin config get` comman
|
|||||||
|
|
||||||
```sh
|
```sh
|
||||||
$ mc admin config get mysilo/ notify_mysql
|
$ mc admin config get mysilo/ notify_mysql
|
||||||
notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0
|
notify_mysql:myinstance enable=off format=namespace dsn_string= table= queue_dir= queue_limit=0
|
||||||
```
|
```
|
||||||
|
|
||||||
Use `mc admin config set` command to update MySQL notification configuration for the deployment with `dsn_string` parameter:
|
Use `mc admin config set` command to update MySQL notification configuration for the deployment with `dsn_string` parameter:
|
||||||
|
|||||||
@@ -358,7 +358,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) {
|
|||||||
e = msgp.ArrayError{Wanted: 5, Got: zb0001}
|
e = msgp.ArrayError{Wanted: 5, Got: zb0001}
|
||||||
return o, e
|
return o, e
|
||||||
}
|
}
|
||||||
bts, e = msgp.ReadExactBytes(bts, (z.VersionID)[:])
|
bts, e = msgp.ReadExactBytes(bts, z.VersionID[:])
|
||||||
if e != nil {
|
if e != nil {
|
||||||
e = msgp.WrapError(e, "VersionID")
|
e = msgp.WrapError(e, "VersionID")
|
||||||
return o, e
|
return o, e
|
||||||
@@ -368,7 +368,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, e error) {
|
|||||||
e = msgp.WrapError(e, "ModTime")
|
e = msgp.WrapError(e, "ModTime")
|
||||||
return o, e
|
return o, e
|
||||||
}
|
}
|
||||||
bts, e = msgp.ReadExactBytes(bts, (z.Signature)[:])
|
bts, e = msgp.ReadExactBytes(bts, z.Signature[:])
|
||||||
if e != nil {
|
if e != nil {
|
||||||
e = msgp.WrapError(e, "Signature")
|
e = msgp.WrapError(e, "Signature")
|
||||||
return o, e
|
return o, e
|
||||||
|
|||||||
@@ -745,7 +745,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt
|
|||||||
err = msgp.ArrayError{Wanted: want, Got: zb0001}
|
err = msgp.ArrayError{Wanted: want, Got: zb0001}
|
||||||
return o, err
|
return o, err
|
||||||
}
|
}
|
||||||
bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:])
|
bts, err = msgp.ReadExactBytes(bts, z.VersionID[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "VersionID")
|
err = msgp.WrapError(err, "VersionID")
|
||||||
return o, err
|
return o, err
|
||||||
@@ -755,7 +755,7 @@ func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte, hdrVer uint) (o []byt
|
|||||||
err = msgp.WrapError(err, "ModTime")
|
err = msgp.WrapError(err, "ModTime")
|
||||||
return o, err
|
return o, err
|
||||||
}
|
}
|
||||||
bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:])
|
bts, err = msgp.ReadExactBytes(bts, z.Signature[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = msgp.WrapError(err, "Signature")
|
err = msgp.WrapError(err, "Signature")
|
||||||
return o, err
|
return o, err
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ Entries carry a CVE identifier where one exists. Where none does, they carry a f
|
|||||||
| `SN-2026-003` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`2f55347f7`](https://github.com/pgsty/minio/commit/2f55347f78352aed8e08866d370c9426c73362cf) | S3/IAM bucket-policy condition values | Yes (policy-dependent) | Prevents raw request entries that spell condition-key names from shadowing or synthesizing internal condition values; confines `s3:signatureAge` to verified SigV4 presigned requests; separates query-only list fields from header-backed `x-amz-*` fields; and stops client request tags from impersonating stored existing-object tags. | The compatible query form remains for storage class and upload tagging on handlers that consume it; an explicitly present header wins, including an empty header. The historical `X-Amz-Tagging` Header mapping remains a client-supplied `RequestObjectTag` source, so use request-tag conditions only on operations that consume tags. Header-only `x-amz-*` policy keys no longer accept query substitutes. `aws:SourceIp` was left following the existing forwarding-header trust model; that model is addressed separately in the next row. See [Condition value sources and precedence](https://silo.pgsty.com/administration/identity-access-management/policy-based-access-control/#condition-value-sources). |
|
| `SN-2026-003` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`2f55347f7`](https://github.com/pgsty/minio/commit/2f55347f78352aed8e08866d370c9426c73362cf) | S3/IAM bucket-policy condition values | Yes (policy-dependent) | Prevents raw request entries that spell condition-key names from shadowing or synthesizing internal condition values; confines `s3:signatureAge` to verified SigV4 presigned requests; separates query-only list fields from header-backed `x-amz-*` fields; and stops client request tags from impersonating stored existing-object tags. | The compatible query form remains for storage class and upload tagging on handlers that consume it; an explicitly present header wins, including an empty header. The historical `X-Amz-Tagging` Header mapping remains a client-supplied `RequestObjectTag` source, so use request-tag conditions only on operations that consume tags. Header-only `x-amz-*` policy keys no longer accept query substitutes. `aws:SourceIp` was left following the existing forwarding-header trust model; that model is addressed separately in the next row. See [Condition value sources and precedence](https://silo.pgsty.com/administration/identity-access-management/policy-based-access-control/#condition-value-sources). |
|
||||||
| Not a vulnerability | `fe6dc4780` | Client source address (`aws:SourceIp`, audit `remotehost`, event notification `Host`) | N/A -- opt-in hardening | Adds an enforceable forwarded-header trust boundary, `MINIO_API_TRUSTED_PROXIES`. Set to a list of addresses or CIDR blocks, forwarded headers are believed only from those peers and forwarding chains are read right-to-left past listed hops -- which also stops the client-supplied left-most entry that an appending proxy (the stock nginx `$proxy_add_x_forwarded_for` recipe, or HAProxy's added second header line) leaves in place. Set to `none`, no forwarded header is believed at all. This is the guarantee `_MINIO_API_XFF_HEADER=off` never provided: it suppresses `X-Forwarded-For` alone, so `X-Real-IP` and RFC 7239 `Forwarded` remain one-line substitutions for anyone that setting was meant to stop. | **No behaviour change for any existing deployment**, so there is nothing to do on upgrade unless you want the new boundary. Not assigned a CVE: the default matches upstream, and upstream's own position (maintainer response in [discussion #17878](https://github.com/minio/minio/discussions/17878), Aug 2023) is that IP-based restrictions are impractical without reliable source-IP visibility. The gap being closed is that this was never written anywhere an operator would find it -- an `IpAddress` condition is accepted and behaves as though it works. **If you use `IpAddress` or `NotIpAddress` conditions, note that they were not enforceable before this change**, including behind a reverse proxy whose `X-Forwarded-For` recipe appends rather than overwrites. If you do not, the change affects only the accuracy of client addresses in logs. The new variable is opt-in and inert when unset; `_MINIO_API_XFF_HEADER` keeps its exact upstream semantics, and upstream's `TestXFFDisabled` is retained unmodified as the proof. An `IpAddress` condition remains unenforceable by default against a client with direct network access to the API port -- that is the condition the allowlist exists to fix, not a regression introduced here. When enabling the allowlist: it must name proxies, not the subnet they sit in, because entries are skipped while walking the chain, so a range that also covers clients lets those clients forge. Multi-node deployments must include their own node addresses, since MinIO forwards some requests between nodes and a client can force a hop through the `ListObjectsV2` continuation token; prefer the allowlist over `none` on a cluster for that reason. Loopback is always trusted as a peer so FTP and SFTP keep attributing their sessions. A malformed value stops startup, as does one that names no proxy at all (`","`) or one whose `env://` remote could not be read -- `env.Get` discards that error and yields an empty string, which would otherwise read as unset. Whitespace-only remains equivalent to unset. The policy is read after `MINIO_CONFIG_ENV_FILE` is loaded so environment-file deployments are covered; `_MINIO_API_XFF_HEADER` deliberately keeps upstream's earlier read timing, where a value written into an environment file is ignored. `MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES` now shares the same list parser, but is behaviourally untouched: the extraction is pure code motion, verified identical to the previous implementation across every combination of 37 allowlist values and 21 peer addresses. See [Client source address trust](source-address-trust.md). |
|
| Not a vulnerability | `fe6dc4780` | Client source address (`aws:SourceIp`, audit `remotehost`, event notification `Host`) | N/A -- opt-in hardening | Adds an enforceable forwarded-header trust boundary, `MINIO_API_TRUSTED_PROXIES`. Set to a list of addresses or CIDR blocks, forwarded headers are believed only from those peers and forwarding chains are read right-to-left past listed hops -- which also stops the client-supplied left-most entry that an appending proxy (the stock nginx `$proxy_add_x_forwarded_for` recipe, or HAProxy's added second header line) leaves in place. Set to `none`, no forwarded header is believed at all. This is the guarantee `_MINIO_API_XFF_HEADER=off` never provided: it suppresses `X-Forwarded-For` alone, so `X-Real-IP` and RFC 7239 `Forwarded` remain one-line substitutions for anyone that setting was meant to stop. | **No behaviour change for any existing deployment**, so there is nothing to do on upgrade unless you want the new boundary. Not assigned a CVE: the default matches upstream, and upstream's own position (maintainer response in [discussion #17878](https://github.com/minio/minio/discussions/17878), Aug 2023) is that IP-based restrictions are impractical without reliable source-IP visibility. The gap being closed is that this was never written anywhere an operator would find it -- an `IpAddress` condition is accepted and behaves as though it works. **If you use `IpAddress` or `NotIpAddress` conditions, note that they were not enforceable before this change**, including behind a reverse proxy whose `X-Forwarded-For` recipe appends rather than overwrites. If you do not, the change affects only the accuracy of client addresses in logs. The new variable is opt-in and inert when unset; `_MINIO_API_XFF_HEADER` keeps its exact upstream semantics, and upstream's `TestXFFDisabled` is retained unmodified as the proof. An `IpAddress` condition remains unenforceable by default against a client with direct network access to the API port -- that is the condition the allowlist exists to fix, not a regression introduced here. When enabling the allowlist: it must name proxies, not the subnet they sit in, because entries are skipped while walking the chain, so a range that also covers clients lets those clients forge. Multi-node deployments must include their own node addresses, since MinIO forwards some requests between nodes and a client can force a hop through the `ListObjectsV2` continuation token; prefer the allowlist over `none` on a cluster for that reason. Loopback is always trusted as a peer so FTP and SFTP keep attributing their sessions. A malformed value stops startup, as does one that names no proxy at all (`","`) or one whose `env://` remote could not be read -- `env.Get` discards that error and yields an empty string, which would otherwise read as unset. Whitespace-only remains equivalent to unset. The policy is read after `MINIO_CONFIG_ENV_FILE` is loaded so environment-file deployments are covered; `_MINIO_API_XFF_HEADER` deliberately keeps upstream's earlier read timing, where a value written into an environment file is ignored. `MINIO_IDENTITY_LDAP_STS_TRUSTED_PROXIES` now shares the same list parser, but is behaviourally untouched: the extraction is pure code motion, verified identical to the previous implementation across every combination of 37 allowlist values and 21 peer addresses. See [Client source address trust](source-address-trust.md). |
|
||||||
| `SN-2026-004` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`97b7d2804`](https://github.com/pgsty/minio/commit/97b7d28040d109061c0a46a4c01bfc7800a97cc1) | IAM policy evaluation of bucket-level actions | Yes (policy-dependent) | Withholds twelve sensitive bucket-level writes from an object-only resource pattern. The IAM matcher appended a trailing slash for bucket-level requests (empty object name), so a resource of `arn:aws:s3:::bucket/*` matched `"bucket/"` and authorized bucket-level actions it was never meant to reach -- upstream [minio/minio#20449](https://github.com/minio/minio/issues/20449). The bucket-policy evaluation path never had the slash and was already reference-correct. | **This is an authorization tightening; read this row before upgrading if you write your own bucket-scoped policies.** Withheld from `bucket/*` on `Allow` statements only: `PutBucketPolicy`, `DeleteBucketPolicy`, `PutBucketObjectLockConfiguration`, `PutBucketVersioning`, `PutReplicationConfiguration`, `PutBucketLifecycle`, `DeleteBucket`, `ForceDeleteBucket`, `PutBucketCors`, `DeleteBucketCors`, `PutBucketQOS`, `PutInventoryConfiguration`. Membership was decided by one question -- does reaching this action give the caller something its object-scoped grant does not already give it? -- because the bug only fires when the statement already grants the bucket action, which in practice means `s3:*`, so the affected principal already holds full object CRUD. Only actions that hand out access to others, defeat a protection aimed at write-holders, act under server credentials, outlive the grant, or destroy the bucket entity qualify. **Deliberately not withheld, and asserted by test so re-adding one is a deliberate act**: `ListBucket`, `GetBucketLocation` and the read/list family, `PutBucketTagging`, `PutBucketEncryption`, `PutBucketNotification`, and `CreateBucket` -- so `mc ls`, SDK session setup and ordinary tenant self-service keep working through `bucket/*`. Breaking those is what got upstream's own full fix reverted. **What to change**: add the bare bucket ARN (`arn:aws:s3:::bucket`) alongside `arn:aws:s3:::bucket/*` in any statement that legitimately grants one of the twelve. Built-in canned policies are unaffected (all use `Resource: "*"`). `Deny` statements are untouched, so no bucket lock is ever weakened, and `NotResource` exclusions keep their full reach. The hardening is monotone by construction rather than by argument: the protected path requires **both** the bare and the historical `"bucket/"` form to match, an intersection with the historical decision -- without that, a fixed-width wildcard such as `mybucke?` would match `"mybucket"` while never having matched `"mybucket/"`, and the hardening would have granted a write the buggy matcher refused. `MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH=on` restores the historical behaviour in full; it is read once at startup. Still deferred to a migration-gated release: the read/list family, a startup audit naming affected policies, and a self-explaining denial log. |
|
| `SN-2026-004` | [`silo-pkg v3.11.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.11.0) and [`97b7d2804`](https://github.com/pgsty/minio/commit/97b7d28040d109061c0a46a4c01bfc7800a97cc1) | IAM policy evaluation of bucket-level actions | Yes (policy-dependent) | Withholds twelve sensitive bucket-level writes from an object-only resource pattern. The IAM matcher appended a trailing slash for bucket-level requests (empty object name), so a resource of `arn:aws:s3:::bucket/*` matched `"bucket/"` and authorized bucket-level actions it was never meant to reach -- upstream [minio/minio#20449](https://github.com/minio/minio/issues/20449). The bucket-policy evaluation path never had the slash and was already reference-correct. | **This is an authorization tightening; read this row before upgrading if you write your own bucket-scoped policies.** Withheld from `bucket/*` on `Allow` statements only: `PutBucketPolicy`, `DeleteBucketPolicy`, `PutBucketObjectLockConfiguration`, `PutBucketVersioning`, `PutReplicationConfiguration`, `PutBucketLifecycle`, `DeleteBucket`, `ForceDeleteBucket`, `PutBucketCors`, `DeleteBucketCors`, `PutBucketQOS`, `PutInventoryConfiguration`. Membership was decided by one question -- does reaching this action give the caller something its object-scoped grant does not already give it? -- because the bug only fires when the statement already grants the bucket action, which in practice means `s3:*`, so the affected principal already holds full object CRUD. Only actions that hand out access to others, defeat a protection aimed at write-holders, act under server credentials, outlive the grant, or destroy the bucket entity qualify. **Deliberately not withheld, and asserted by test so re-adding one is a deliberate act**: `ListBucket`, `GetBucketLocation` and the read/list family, `PutBucketTagging`, `PutBucketEncryption`, `PutBucketNotification`, and `CreateBucket` -- so `mc ls`, SDK session setup and ordinary tenant self-service keep working through `bucket/*`. Breaking those is what got upstream's own full fix reverted. **What to change**: add the bare bucket ARN (`arn:aws:s3:::bucket`) alongside `arn:aws:s3:::bucket/*` in any statement that legitimately grants one of the twelve. Built-in canned policies are unaffected (all use `Resource: "*"`). `Deny` statements are untouched, so no bucket lock is ever weakened, and `NotResource` exclusions keep their full reach. The hardening is monotone by construction rather than by argument: the protected path requires **both** the bare and the historical `"bucket/"` form to match, an intersection with the historical decision -- without that, a fixed-width wildcard such as `mybucke?` would match `"mybucket"` while never having matched `"mybucket/"`, and the hardening would have granted a write the buggy matcher refused. `MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH=on` restores the historical behaviour in full; it is read once at startup. Still deferred to a migration-gated release: the read/list family, a startup audit naming affected policies, and a self-explaining denial log. |
|
||||||
|
| `SN-2026-005` | [`silo-pkg v3.12.0`](https://github.com/pgsty/silo-pkg/releases/tag/v3.12.0) and [`eee05a17c`](https://github.com/pgsty/silo/commit/eee05a17c34a07cebb27220d12697be74c8bd617) | IAM named-policy and service-account policy writes | No direct remote exploit; policy-dependent | Rejects S3, S3 Tables, and KMS ARN namespace prefixes that name no resource, including their historical `*arn:...` serialization, in both `Resource` and `NotResource`. A resource-matching `Deny` using such a prefix could silently fail to deny, while an `Allow` with the prefix in `NotResource` could match far more broadly than intended. The guard applies when creating named policies and when creating or updating service-account session policies. | **This is an authorization tightening for new and updated policies.** Existing policies keep loading, matching, importing, and replicating with unchanged runtime behavior, but a policy containing one of these prefixes cannot be submitted unchanged; replace it with the intended concrete resource, or use an explicit wildcard such as `arn:aws:s3:::*` only when all resources are intended. Enabling the strict path also rejects an admin statement that combines `Resource` with `NotResource`, and rejects non-S3 resources on bucket-scoped admin actions. Here “bare ARN prefix” means a namespace with no resource after it (`arn:aws:s3:::`); it is distinct from the valid “bare bucket ARN” in `SN-2026-004` (`arn:aws:s3:::bucket`). IAM import, site-replication receive paths, stored-policy loading, and STS inline policies remain on the permissive compatibility path in this release. |
|
||||||
|
|
||||||
## Dependency security updates
|
## Dependency security updates
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,19 @@
|
|||||||
module github.com/minio/minio
|
module github.com/minio/minio
|
||||||
|
|
||||||
go 1.26.5
|
go 1.27.0
|
||||||
|
|
||||||
// Use Pigsty's SILO Console v2.1.1 release while preserving upstream import paths.
|
// Use Pigsty's SILO Console v2.1.1 release while preserving upstream import paths.
|
||||||
// The pseudo-version pins v2.1.1's commit because the compatible module path has no /v2 suffix.
|
// The pseudo-version pins v2.1.1's commit because the compatible module path has no /v2 suffix.
|
||||||
replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a
|
replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a
|
||||||
|
|
||||||
// Use Pigsty's maintained mc fork for Console's embedded client code.
|
// Use Pigsty's maintained mc fork for Console's embedded client code.
|
||||||
replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76
|
replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb
|
||||||
|
|
||||||
// Use Pigsty's maintained SILO package fork while preserving upstream import paths.
|
// Use Pigsty's maintained SILO package fork while preserving upstream import paths.
|
||||||
// This retains the LDAP TLS fix tracked in https://github.com/pgsty/silo/issues/15.
|
// This retains the LDAP TLS fix tracked in https://github.com/pgsty/silo/issues/15.
|
||||||
// v3.11.0 follows upstream minio/pkg's 3.11 line and carries the
|
// v3.12.0 carries the minio/minio#20449 bucket-write boundary hardening and
|
||||||
// minio/minio#20449 bucket-write boundary hardening.
|
// rejects bare ARN prefixes on strict policy-write paths.
|
||||||
replace github.com/minio/pkg/v3 => github.com/pgsty/silo-pkg/v3 v3.11.0
|
replace github.com/minio/pkg/v3 => github.com/pgsty/silo-pkg/v3 v3.12.0
|
||||||
|
|
||||||
// v22.7.0 does not compile on NetBSD because its unix implementation uses
|
// v22.7.0 does not compile on NetBSD because its unix implementation uses
|
||||||
// CLOCK_MONOTONIC, which is unavailable there. Keep the last portable release
|
// CLOCK_MONOTONIC, which is unavailable there. Keep the last portable release
|
||||||
@@ -37,7 +37,7 @@ require (
|
|||||||
github.com/beevik/ntp v1.5.0
|
github.com/beevik/ntp v1.5.0
|
||||||
github.com/buger/jsonparser v1.1.2
|
github.com/buger/jsonparser v1.1.2
|
||||||
github.com/cespare/xxhash/v2 v2.3.0
|
github.com/cespare/xxhash/v2 v2.3.0
|
||||||
github.com/cheggaaa/pb v1.0.29
|
github.com/cheggaaa/pb v1.0.30
|
||||||
github.com/coreos/go-oidc/v3 v3.17.0
|
github.com/coreos/go-oidc/v3 v3.17.0
|
||||||
github.com/coreos/go-systemd/v22 v22.7.0
|
github.com/coreos/go-systemd/v22 v22.7.0
|
||||||
github.com/cosnicolaou/pbzip2 v1.0.6
|
github.com/cosnicolaou/pbzip2 v1.0.6
|
||||||
@@ -65,7 +65,7 @@ require (
|
|||||||
github.com/klauspost/reedsolomon v1.13.3
|
github.com/klauspost/reedsolomon v1.13.3
|
||||||
github.com/lib/pq v1.10.9
|
github.com/lib/pq v1.10.9
|
||||||
github.com/lithammer/shortuuid/v4 v4.2.0
|
github.com/lithammer/shortuuid/v4 v4.2.0
|
||||||
github.com/miekg/dns v1.1.72
|
github.com/miekg/dns v1.1.73
|
||||||
github.com/minio/cli v1.24.2
|
github.com/minio/cli v1.24.2
|
||||||
github.com/minio/console v1.7.6
|
github.com/minio/console v1.7.6
|
||||||
github.com/minio/csvparser v1.0.0
|
github.com/minio/csvparser v1.0.0
|
||||||
@@ -77,7 +77,7 @@ require (
|
|||||||
github.com/minio/madmin-go/v3 v3.0.110
|
github.com/minio/madmin-go/v3 v3.0.110
|
||||||
github.com/minio/minio-go/v7 v7.0.99
|
github.com/minio/minio-go/v7 v7.0.99
|
||||||
github.com/minio/mux v1.9.2
|
github.com/minio/mux v1.9.2
|
||||||
github.com/minio/pkg/v3 v3.6.1
|
github.com/minio/pkg/v3 v3.12.0
|
||||||
github.com/minio/selfupdate v0.6.0
|
github.com/minio/selfupdate v0.6.0
|
||||||
github.com/minio/simdjson-go v0.4.5
|
github.com/minio/simdjson-go v0.4.5
|
||||||
github.com/minio/sio v0.4.3
|
github.com/minio/sio v0.4.3
|
||||||
@@ -90,9 +90,9 @@ require (
|
|||||||
github.com/ncw/directio v1.0.5
|
github.com/ncw/directio v1.0.5
|
||||||
github.com/nsqio/go-nsq v1.1.0
|
github.com/nsqio/go-nsq v1.1.0
|
||||||
github.com/philhofer/fwd v1.2.0
|
github.com/philhofer/fwd v1.2.0
|
||||||
github.com/pierrec/lz4/v4 v4.1.26
|
github.com/pierrec/lz4/v4 v4.1.29
|
||||||
github.com/pkg/errors v0.9.1
|
github.com/pkg/errors v0.9.1
|
||||||
github.com/pkg/sftp v1.13.10
|
github.com/pkg/sftp v1.13.11
|
||||||
github.com/pkg/xattr v0.4.12
|
github.com/pkg/xattr v0.4.12
|
||||||
github.com/prometheus/client_golang v1.23.2
|
github.com/prometheus/client_golang v1.23.2
|
||||||
github.com/prometheus/client_model v0.6.2
|
github.com/prometheus/client_model v0.6.2
|
||||||
@@ -108,13 +108,13 @@ require (
|
|||||||
github.com/valyala/bytebufferpool v1.0.0
|
github.com/valyala/bytebufferpool v1.0.0
|
||||||
github.com/xdg/scram v1.0.5
|
github.com/xdg/scram v1.0.5
|
||||||
github.com/zeebo/xxh3 v1.1.0
|
github.com/zeebo/xxh3 v1.1.0
|
||||||
go.etcd.io/etcd/api/v3 v3.6.9
|
go.etcd.io/etcd/api/v3 v3.7.1
|
||||||
go.etcd.io/etcd/client/v3 v3.6.9
|
go.etcd.io/etcd/client/v3 v3.7.1
|
||||||
go.uber.org/atomic v1.11.0
|
go.uber.org/atomic v1.11.0
|
||||||
go.uber.org/zap v1.28.0
|
go.uber.org/zap v1.28.0
|
||||||
go.yaml.in/yaml/v3 v3.0.5
|
go.yaml.in/yaml/v3 v3.0.5
|
||||||
goftp.io/server/v2 v2.0.2
|
goftp.io/server/v2 v2.0.3
|
||||||
golang.org/x/crypto v0.54.0
|
golang.org/x/crypto v0.55.0
|
||||||
golang.org/x/oauth2 v0.36.0
|
golang.org/x/oauth2 v0.36.0
|
||||||
golang.org/x/sync v0.22.0
|
golang.org/x/sync v0.22.0
|
||||||
golang.org/x/sys v0.47.0
|
golang.org/x/sys v0.47.0
|
||||||
@@ -210,7 +210,6 @@ require (
|
|||||||
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
|
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
|
||||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||||
github.com/gorilla/mux v1.8.1 // indirect
|
|
||||||
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
|
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
|
||||||
github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect
|
github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect
|
||||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||||
@@ -233,9 +232,8 @@ require (
|
|||||||
github.com/lestrrat-go/dsig v1.0.0 // indirect
|
github.com/lestrrat-go/dsig v1.0.0 // indirect
|
||||||
github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect
|
github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect
|
||||||
github.com/lestrrat-go/httpcc v1.0.1 // indirect
|
github.com/lestrrat-go/httpcc v1.0.1 // indirect
|
||||||
github.com/lestrrat-go/httprc/v3 v3.0.1 // indirect
|
github.com/lestrrat-go/httprc/v3 v3.0.6 // indirect
|
||||||
github.com/lestrrat-go/jwx/v3 v3.0.12 // indirect
|
github.com/lestrrat-go/jwx/v3 v3.0.13 // indirect
|
||||||
github.com/lestrrat-go/option v1.0.1 // indirect
|
|
||||||
github.com/lestrrat-go/option/v2 v2.0.0 // indirect
|
github.com/lestrrat-go/option/v2 v2.0.0 // indirect
|
||||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||||
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect
|
||||||
@@ -283,12 +281,12 @@ require (
|
|||||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||||
github.com/unrolled/secure v1.17.0 // indirect
|
github.com/unrolled/secure v1.17.0 // indirect
|
||||||
github.com/valyala/fastjson v1.6.4 // indirect
|
github.com/valyala/fastjson v1.6.10 // indirect
|
||||||
github.com/vbauerster/mpb/v8 v8.12.0 // indirect
|
github.com/vbauerster/mpb/v8 v8.12.0 // indirect
|
||||||
github.com/xdg/stringprep v1.0.3 // indirect
|
github.com/xdg/stringprep v1.0.3 // indirect
|
||||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
go.etcd.io/etcd/client/pkg/v3 v3.6.9 // indirect
|
go.etcd.io/etcd/client/pkg/v3 v3.7.1 // indirect
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||||
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect
|
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect
|
||||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect
|
||||||
@@ -300,10 +298,11 @@ require (
|
|||||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||||
go.uber.org/multierr v1.11.0 // indirect
|
go.uber.org/multierr v1.11.0 // indirect
|
||||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||||
golang.org/x/mod v0.37.0 // indirect
|
golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect
|
||||||
golang.org/x/net v0.57.0 // indirect
|
golang.org/x/mod v0.40.0 // indirect
|
||||||
golang.org/x/text v0.40.0 // indirect
|
golang.org/x/net v0.58.0 // indirect
|
||||||
golang.org/x/tools v0.47.0 // indirect
|
golang.org/x/text v0.41.0 // indirect
|
||||||
|
golang.org/x/tools v0.49.0 // indirect
|
||||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
|
||||||
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect
|
||||||
|
|||||||
@@ -109,8 +109,8 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payR
|
|||||||
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
|
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
|
||||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||||
github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
|
github.com/cheggaaa/pb v1.0.30 h1:NylhgqJfXx3JVBGx6ywsXuhpz8caSMPmLArXyAv1bwU=
|
||||||
github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
|
github.com/cheggaaa/pb v1.0.30/go.mod h1:YgTBwa6PqwwDB/2UKdLuuFRNTwEkcCPsA5AmWivrBAg=
|
||||||
github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
|
github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
|
||||||
github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
|
github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
|
||||||
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
|
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
|
||||||
@@ -411,12 +411,10 @@ github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7
|
|||||||
github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU=
|
github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU=
|
||||||
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
|
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
|
||||||
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
|
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
|
||||||
github.com/lestrrat-go/httprc/v3 v3.0.1 h1:3n7Es68YYGZb2Jf+k//llA4FTZMl3yCwIjFIk4ubevI=
|
github.com/lestrrat-go/httprc/v3 v3.0.6 h1:4FpLQ18KK/ypPbVU3NLWJNRvH3kcYiqKqWfKGqNWxxI=
|
||||||
github.com/lestrrat-go/httprc/v3 v3.0.1/go.mod h1:2uAvmbXE4Xq8kAUjVrZOq1tZVYYYs5iP62Cmtru00xk=
|
github.com/lestrrat-go/httprc/v3 v3.0.6/go.mod h1:mSMtkZW92Z98M5YoNNztbRGxbXHql7tSitCvaxvo9l0=
|
||||||
github.com/lestrrat-go/jwx/v3 v3.0.12 h1:p25r68Y4KrbBdYjIsQweYxq794CtGCzcrc5dGzJIRjg=
|
github.com/lestrrat-go/jwx/v3 v3.0.13 h1:AdHKiPIYeCSnOJtvdpipPg/0SuFh9rdkN+HF3O0VdSk=
|
||||||
github.com/lestrrat-go/jwx/v3 v3.0.12/go.mod h1:HiUSaNmMLXgZ08OmGBaPVvoZQgJVOQphSrGr5zMamS8=
|
github.com/lestrrat-go/jwx/v3 v3.0.13/go.mod h1:2m0PV1A9tM4b/jVLMx8rh6rBl7F6WGb3EG2hufN9OQU=
|
||||||
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
|
|
||||||
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
|
||||||
github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss=
|
github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss=
|
||||||
github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg=
|
github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg=
|
||||||
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||||
@@ -451,8 +449,8 @@ github.com/mattn/go-runewidth v0.0.21/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
|
|||||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||||
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
|
github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
|
||||||
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
|
github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
|
||||||
github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
|
github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
|
||||||
github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
|
github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
|
||||||
github.com/minio/colorjson v1.0.8 h1:AS6gEQ1dTRYHmC4xuoodPDRILHP/9Wz5wYUGDQfPLpg=
|
github.com/minio/colorjson v1.0.8 h1:AS6gEQ1dTRYHmC4xuoodPDRILHP/9Wz5wYUGDQfPLpg=
|
||||||
@@ -547,23 +545,23 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzb
|
|||||||
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||||
github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76 h1:UIlUuz0LQKw4QlAljhv7nPDDFC1+n+e0iED7rWZrgZ8=
|
github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb h1:ixZKprZQpUIVrUlOy6H7oLb0a4B9eq6sLKrt8wNw1lU=
|
||||||
github.com/pgsty/mc v0.0.0-20260801042411-ad10a2a10b76/go.mod h1:cTbS+9jGR4Qs7xTf5DEhmCTbzcDWrKMs8ZmTUnCU49E=
|
github.com/pgsty/mc v0.0.0-20260806055018-b0021fd01ccb/go.mod h1:cTbS+9jGR4Qs7xTf5DEhmCTbzcDWrKMs8ZmTUnCU49E=
|
||||||
github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a h1:JfEQJkBdTwCXSrCv8P0R5tMpzxxhSdUTvg54mHwzxpA=
|
github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a h1:JfEQJkBdTwCXSrCv8P0R5tMpzxxhSdUTvg54mHwzxpA=
|
||||||
github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a/go.mod h1:7J8wCQsNT5S7GqCHnQqgj0T7Nagp1fklJhfBJI+v0XI=
|
github.com/pgsty/silo-console v0.0.0-20260806061103-72fc0a5ea52a/go.mod h1:7J8wCQsNT5S7GqCHnQqgj0T7Nagp1fklJhfBJI+v0XI=
|
||||||
github.com/pgsty/silo-pkg/v3 v3.11.0 h1:wjN5d+tWD8Twq+e7k/KBBVhnWXC8xTIlfTcnGIKkmjc=
|
github.com/pgsty/silo-pkg/v3 v3.12.0 h1:1Bjqjb3KCt0oYhBLpH7W/e/5khTUoIgXWA12An1fbUc=
|
||||||
github.com/pgsty/silo-pkg/v3 v3.11.0/go.mod h1:E2AB4oOgfDeb9In1KDBTrn9wzfvr0WzoPkbXW7wbwBQ=
|
github.com/pgsty/silo-pkg/v3 v3.12.0/go.mod h1:ohHtpAK7kBCffdMT4oqUbtkoZMHxA7znTCPNjrYw86I=
|
||||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||||
github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY=
|
github.com/pierrec/lz4/v4 v4.1.29 h1:CDQY6qZOLI4DW0Nx6R1vRrifrCeQHnNXkMb0hZWXFjg=
|
||||||
github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
github.com/pierrec/lz4/v4 v4.1.29/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
|
github.com/pkg/sftp v1.13.11 h1:0N92SLTB8JqASJB14ZLHHzFnBV8mG9zw4K7jghEFWuE=
|
||||||
github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
|
github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hsN0=
|
||||||
github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
|
github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
|
||||||
github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
|
||||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
|
||||||
@@ -639,7 +637,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
|
|||||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||||
@@ -666,8 +663,8 @@ github.com/unrolled/secure v1.17.0 h1:Io7ifFgo99Bnh0J7+Q+qcMzWM6kaDPCA5FroFZEdbW
|
|||||||
github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
|
github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
|
||||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||||
github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
|
github.com/valyala/fastjson v1.6.10 h1:/yjJg8jaVQdYR3arGxPE2X5z89xrlhS0eGXdv+ADTh4=
|
||||||
github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
|
github.com/valyala/fastjson v1.6.10/go.mod h1:e6FubmQouUNP73jtMLmcbxS6ydWIpOfhz34TSfO3JaE=
|
||||||
github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg=
|
github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg=
|
||||||
github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588=
|
github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588=
|
||||||
github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw=
|
github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw=
|
||||||
@@ -688,12 +685,12 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
|
|||||||
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
|
||||||
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
|
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
|
||||||
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
|
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
|
||||||
go.etcd.io/etcd/api/v3 v3.6.9 h1:UA7iKfEW1AzgihcBSGXci2kDGQiokSq41F9HMCI/RTI=
|
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
|
||||||
go.etcd.io/etcd/api/v3 v3.6.9/go.mod h1:csEk/qTfxKL36NqJdU15Tgtl65A8dyEY2BYo7PRsIwk=
|
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
|
||||||
go.etcd.io/etcd/client/pkg/v3 v3.6.9 h1:T8nuk8Lz64C+Hzb0coBFLMSlVSQZBpAtFk46swdM1DA=
|
go.etcd.io/etcd/client/pkg/v3 v3.7.1 h1:rKYsj3pRkR0eK3yjT3XOgrhqfmIfj9pzNgxjh7mfFv4=
|
||||||
go.etcd.io/etcd/client/pkg/v3 v3.6.9/go.mod h1:WEy3PpwbbEBVRdh1NVJYsuUe/8eyI21PNJRazeD8z/Y=
|
go.etcd.io/etcd/client/pkg/v3 v3.7.1/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0=
|
||||||
go.etcd.io/etcd/client/v3 v3.6.9 h1:3X555hQXmhRr27O37wls53g68CpUiPOiHXrZfz2Al+o=
|
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
|
||||||
go.etcd.io/etcd/client/v3 v3.6.9/go.mod h1:KO7H1HLYh1qaljuVZJQwBFk1lRce6pJzt+C81GEnrlM=
|
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||||
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU=
|
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU=
|
||||||
@@ -728,8 +725,8 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
|||||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
goftp.io/server/v2 v2.0.2 h1:tkZpqyXys+vC15W5yGMi8Kzmbv1QSgeKr8qJXBnJbm8=
|
goftp.io/server/v2 v2.0.3 h1:iz6Gxj7f2SFQVxrj0s1is+gueE6O9yTc+Ab0vtQ6Zn4=
|
||||||
goftp.io/server/v2 v2.0.2/go.mod h1:Fl1WdcV7fx1pjOWx7jEHb7tsJ8VwE7+xHu6bVJ6r2qg=
|
goftp.io/server/v2 v2.0.3/go.mod h1:Fl1WdcV7fx1pjOWx7jEHb7tsJ8VwE7+xHu6bVJ6r2qg=
|
||||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||||
@@ -742,15 +739,15 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0
|
|||||||
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||||
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
|
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
|
||||||
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
|
||||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
|
||||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
|
||||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
|
golang.org/x/exp v0.0.0-20260820142414-ca536658362e h1:01Ju2A/fZKkci4zqx0eZxw//DnRYOnBiGJG14hFBhO8=
|
||||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
|
golang.org/x/exp v0.0.0-20260820142414-ca536658362e/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
|
||||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
@@ -764,8 +761,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
|||||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
@@ -822,8 +819,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|||||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||||
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
@@ -833,8 +830,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
|
|||||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
|||||||
@@ -26,6 +26,25 @@ import (
|
|||||||
"github.com/minio/minio/internal/event/target"
|
"github.com/minio/minio/internal/event/target"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// LegacyDatabaseTargetError reports a pre-KV database notification target
|
||||||
|
// that cannot be migrated safely. It deliberately carries no configuration
|
||||||
|
// values so credentials cannot escape through startup logs.
|
||||||
|
type LegacyDatabaseTargetError struct {
|
||||||
|
subsystem string
|
||||||
|
target string
|
||||||
|
connectionKey string
|
||||||
|
invalid bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *LegacyDatabaseTargetError) Error() string {
|
||||||
|
if e.invalid {
|
||||||
|
return fmt.Sprintf("%s:%s has invalid %s or target settings; fix the target before migrating to SILO",
|
||||||
|
e.subsystem, e.target, e.connectionKey)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s:%s requires %s; discrete database connection fields are not migrated to SILO",
|
||||||
|
e.subsystem, e.target, e.connectionKey)
|
||||||
|
}
|
||||||
|
|
||||||
// SetNotifyKafka - helper for config migration from older config.
|
// SetNotifyKafka - helper for config migration from older config.
|
||||||
func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error {
|
func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error {
|
||||||
if !cfg.Enable {
|
if !cfg.Enable {
|
||||||
@@ -325,8 +344,21 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if cfg.ConnectionString == "" {
|
||||||
|
return &LegacyDatabaseTargetError{
|
||||||
|
subsystem: config.NotifyPostgresSubSys,
|
||||||
|
target: psqName,
|
||||||
|
connectionKey: target.PostgresConnectionString,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := cfg.Validate(); err != nil {
|
if err := cfg.Validate(); err != nil {
|
||||||
return err
|
return &LegacyDatabaseTargetError{
|
||||||
|
subsystem: config.NotifyPostgresSubSys,
|
||||||
|
target: psqName,
|
||||||
|
connectionKey: target.PostgresConnectionString,
|
||||||
|
invalid: true,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
s[config.NotifyPostgresSubSys][psqName] = config.KVS{
|
s[config.NotifyPostgresSubSys][psqName] = config.KVS{
|
||||||
@@ -346,26 +378,6 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg
|
|||||||
Key: target.PostgresTable,
|
Key: target.PostgresTable,
|
||||||
Value: cfg.Table,
|
Value: cfg.Table,
|
||||||
},
|
},
|
||||||
config.KV{
|
|
||||||
Key: target.PostgresHost,
|
|
||||||
Value: cfg.Host.String(),
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.PostgresPort,
|
|
||||||
Value: cfg.Port,
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.PostgresUsername,
|
|
||||||
Value: cfg.Username,
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.PostgresPassword,
|
|
||||||
Value: cfg.Password,
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.PostgresDatabase,
|
|
||||||
Value: cfg.Database,
|
|
||||||
},
|
|
||||||
config.KV{
|
config.KV{
|
||||||
Key: target.PostgresQueueDir,
|
Key: target.PostgresQueueDir,
|
||||||
Value: cfg.QueueDir,
|
Value: cfg.QueueDir,
|
||||||
@@ -538,8 +550,21 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if cfg.DSN == "" {
|
||||||
|
return &LegacyDatabaseTargetError{
|
||||||
|
subsystem: config.NotifyMySQLSubSys,
|
||||||
|
target: sqlName,
|
||||||
|
connectionKey: target.MySQLDSNString,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := cfg.Validate(); err != nil {
|
if err := cfg.Validate(); err != nil {
|
||||||
return err
|
return &LegacyDatabaseTargetError{
|
||||||
|
subsystem: config.NotifyMySQLSubSys,
|
||||||
|
target: sqlName,
|
||||||
|
connectionKey: target.MySQLDSNString,
|
||||||
|
invalid: true,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
s[config.NotifyMySQLSubSys][sqlName] = config.KVS{
|
s[config.NotifyMySQLSubSys][sqlName] = config.KVS{
|
||||||
@@ -559,26 +584,6 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error
|
|||||||
Key: target.MySQLTable,
|
Key: target.MySQLTable,
|
||||||
Value: cfg.Table,
|
Value: cfg.Table,
|
||||||
},
|
},
|
||||||
config.KV{
|
|
||||||
Key: target.MySQLHost,
|
|
||||||
Value: cfg.Host.String(),
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.MySQLPort,
|
|
||||||
Value: cfg.Port,
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.MySQLUsername,
|
|
||||||
Value: cfg.User,
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.MySQLPassword,
|
|
||||||
Value: cfg.Password,
|
|
||||||
},
|
|
||||||
config.KV{
|
|
||||||
Key: target.MySQLDatabase,
|
|
||||||
Value: cfg.Database,
|
|
||||||
},
|
|
||||||
config.KV{
|
config.KV{
|
||||||
Key: target.MySQLQueueDir,
|
Key: target.MySQLQueueDir,
|
||||||
Value: cfg.QueueDir,
|
Value: cfg.QueueDir,
|
||||||
|
|||||||
@@ -18,6 +18,8 @@
|
|||||||
package notify
|
package notify
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
@@ -26,6 +28,25 @@ import (
|
|||||||
"github.com/rabbitmq/amqp091-go"
|
"github.com/rabbitmq/amqp091-go"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func assertLegacyDatabaseTargetError(t *testing.T, err error, subsystem, name, key string, secrets ...string) {
|
||||||
|
t.Helper()
|
||||||
|
var targetErr *LegacyDatabaseTargetError
|
||||||
|
if !errors.As(err, &targetErr) {
|
||||||
|
t.Fatalf("error = %v, want *LegacyDatabaseTargetError", err)
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} {
|
||||||
|
if !strings.Contains(msg, want) {
|
||||||
|
t.Errorf("error %q does not contain %q", msg, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, secret := range secrets {
|
||||||
|
if secret != "" && strings.Contains(msg, secret) {
|
||||||
|
t.Errorf("error leaks configuration value %q: %s", secret, msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// T5 (NATS): a config produced by the legacy migration must survive validation
|
// T5 (NATS): a config produced by the legacy migration must survive validation
|
||||||
// and round-trip back through the parser unchanged. Before the fix the
|
// and round-trip back through the parser unchanged. Before the fix the
|
||||||
// migration wrote an env var name as a config key, so every migrated NATS
|
// migration wrote an env var name as a config key, so every migrated NATS
|
||||||
@@ -113,3 +134,172 @@ func TestSetNotifyAMQPRoundTrip(t *testing.T) {
|
|||||||
t.Errorf("Internal = true, want false (immediate must not be written to the internal key)")
|
t.Errorf("Internal = true, want false (immediate must not be written to the internal key)")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSetNotifyDatabaseTargetsRequireConnectionStrings(t *testing.T) {
|
||||||
|
postgresHost, err := xnet.ParseHost("legacy-postgres.example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mysqlHost, err := xnet.ParseURL("legacy-mysql.example")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
subsystem string
|
||||||
|
key string
|
||||||
|
set func(config.Config) error
|
||||||
|
secrets []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "postgres",
|
||||||
|
subsystem: config.NotifyPostgresSubSys,
|
||||||
|
key: target.PostgresConnectionString,
|
||||||
|
set: func(s config.Config) error {
|
||||||
|
return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Format: formatNamespace,
|
||||||
|
Table: "events",
|
||||||
|
Host: *postgresHost,
|
||||||
|
Port: "5432",
|
||||||
|
Username: "legacy-user",
|
||||||
|
Password: "legacy-postgres-password",
|
||||||
|
Database: "legacy-database",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
secrets: []string{postgresHost.String(), "5432", "legacy-user", "legacy-postgres-password", "legacy-database"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mysql",
|
||||||
|
subsystem: config.NotifyMySQLSubSys,
|
||||||
|
key: target.MySQLDSNString,
|
||||||
|
set: func(s config.Config) error {
|
||||||
|
return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Format: formatNamespace,
|
||||||
|
Table: "events",
|
||||||
|
Host: *mysqlHost,
|
||||||
|
Port: "3306",
|
||||||
|
User: "legacy-user",
|
||||||
|
Password: "legacy-mysql-password",
|
||||||
|
Database: "legacy-database",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
secrets: []string{mysqlHost.String(), "3306", "legacy-user", "legacy-mysql-password", "legacy-database"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
s := config.Config{test.subsystem: map[string]config.KVS{}}
|
||||||
|
err := test.set(s)
|
||||||
|
assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secrets...)
|
||||||
|
if _, ok := s[test.subsystem][testTargetName]; ok {
|
||||||
|
t.Fatal("unsupported target was emitted despite migration error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetNotifyDisabledDatabaseTargetsAreIgnored(t *testing.T) {
|
||||||
|
s := config.Config{
|
||||||
|
config.NotifyPostgresSubSys: map[string]config.KVS{},
|
||||||
|
config.NotifyMySQLSubSys: map[string]config.KVS{},
|
||||||
|
}
|
||||||
|
if err := SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{Password: "discarded-postgres-secret"}); err != nil {
|
||||||
|
t.Fatalf("SetNotifyPostgres: %v", err)
|
||||||
|
}
|
||||||
|
if err := SetNotifyMySQL(s, testTargetName, target.MySQLArgs{Password: "discarded-mysql-secret"}); err != nil {
|
||||||
|
t.Fatalf("SetNotifyMySQL: %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := s[config.NotifyPostgresSubSys][testTargetName]; ok {
|
||||||
|
t.Fatal("disabled Postgres target was emitted")
|
||||||
|
}
|
||||||
|
if _, ok := s[config.NotifyMySQLSubSys][testTargetName]; ok {
|
||||||
|
t.Fatal("disabled MySQL target was emitted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetNotifyInvalidDatabaseTargetsDoNotLeak(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
subsystem string
|
||||||
|
key string
|
||||||
|
secret string
|
||||||
|
set func(config.Config) error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "postgres",
|
||||||
|
subsystem: config.NotifyPostgresSubSys,
|
||||||
|
key: target.PostgresConnectionString,
|
||||||
|
secret: "postgres-dsn-secret",
|
||||||
|
set: func(s config.Config) error {
|
||||||
|
return SetNotifyPostgres(s, testTargetName, target.PostgreSQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Format: formatNamespace,
|
||||||
|
ConnectionString: "host=db password=postgres-dsn-secret",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mysql",
|
||||||
|
subsystem: config.NotifyMySQLSubSys,
|
||||||
|
key: target.MySQLDSNString,
|
||||||
|
secret: "mysql-dsn-secret",
|
||||||
|
set: func(s config.Config) error {
|
||||||
|
return SetNotifyMySQL(s, testTargetName, target.MySQLArgs{
|
||||||
|
Enable: true,
|
||||||
|
Format: formatNamespace,
|
||||||
|
DSN: "user:mysql-dsn-secret@tcp(db:3306/events",
|
||||||
|
Table: "events",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
s := config.Config{test.subsystem: map[string]config.KVS{}}
|
||||||
|
err := test.set(s)
|
||||||
|
assertLegacyDatabaseTargetError(t, err, test.subsystem, testTargetName, test.key, test.secret)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatabaseConnectionStringsSurviveKVTokenization(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
subsystem string
|
||||||
|
key string
|
||||||
|
input string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "postgres",
|
||||||
|
subsystem: config.NotifyPostgresSubSys,
|
||||||
|
key: target.PostgresConnectionString,
|
||||||
|
input: `notify_postgres:dsn connection_string="host=db port=5432 dbname=events user=app password=inside" table="events"`,
|
||||||
|
want: "host=db port=5432 dbname=events user=app password=inside",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mysql",
|
||||||
|
subsystem: config.NotifyMySQLSubSys,
|
||||||
|
key: target.MySQLDSNString,
|
||||||
|
input: `notify_mysql:dsn dsn_string="user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside" table="events"`,
|
||||||
|
want: "user:pass@tcp(db:3306)/events?host=db&port=3306&password=inside",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
s := config.Config{test.subsystem: map[string]config.KVS{}}
|
||||||
|
if _, err := s.SetKVS(test.input, DefaultNotificationKVS); err != nil {
|
||||||
|
t.Fatalf("SetKVS: %v", err)
|
||||||
|
}
|
||||||
|
if got := s[test.subsystem]["dsn"].Get(test.key); got != test.want {
|
||||||
|
t.Errorf("%s = %q, want %q", test.key, got, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -414,21 +414,9 @@ var configPkgConsts = map[string]string{
|
|||||||
"Comment": config.Comment,
|
"Comment": config.Comment,
|
||||||
}
|
}
|
||||||
|
|
||||||
// knownUnregisteredWrites records pre-existing instances of the exact defect
|
// knownUnregisteredWrites is a shrink-only ratchet for inherited migration
|
||||||
// this audit exists to catch: a legacy migration writing config keys that no
|
// gaps. Do not add entries to silence a new mismatch.
|
||||||
// default KVS registers, so the migrated config is rejected on the next load.
|
var knownUnregisteredWrites = map[string][]string{}
|
||||||
//
|
|
||||||
// These are inherited from upstream and are the same class as issue #39, but
|
|
||||||
// they are NOT part of the issue #39 fix and were left untouched deliberately.
|
|
||||||
// The Postgres/MySQL keys below are the pre-connection-string DSN fields; the
|
|
||||||
// migration still writes them and `password` carries a plaintext database
|
|
||||||
// password.
|
|
||||||
//
|
|
||||||
// This list must only ever shrink. Do not add entries to silence a new gap.
|
|
||||||
var knownUnregisteredWrites = map[string][]string{
|
|
||||||
"SetNotifyPostgres": {"host", "port", "username", "password", "database"},
|
|
||||||
"SetNotifyMySQL": {"host", "port", "username", "password", "database"},
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNotifyConfigKeysAreRegistered(t *testing.T) {
|
func TestNotifyConfigKeysAreRegistered(t *testing.T) {
|
||||||
targetConsts, err := parseTargetPkgStringConsts("../../event/target")
|
targetConsts, err := parseTargetPkgStringConsts("../../event/target")
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ func DisableDirectIO(f *os.File) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
flag &= ^(syscall.O_DIRECT)
|
flag &= ^syscall.O_DIRECT
|
||||||
_, err = unix.FcntlInt(fd, unix.F_SETFL, flag)
|
_, err = unix.FcntlInt(fd, unix.F_SETFL, flag)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func TestNatsConnPlain(t *testing.T) {
|
|||||||
Enable: true,
|
Enable: true,
|
||||||
Address: xnet.Host{
|
Address: xnet.Host{
|
||||||
Name: "localhost",
|
Name: "localhost",
|
||||||
Port: (xnet.Port(opts.Port)),
|
Port: xnet.Port(opts.Port),
|
||||||
IsPortSet: true,
|
IsPortSet: true,
|
||||||
},
|
},
|
||||||
Subject: "test",
|
Subject: "test",
|
||||||
@@ -59,7 +59,7 @@ func TestNatsConnUserPass(t *testing.T) {
|
|||||||
Enable: true,
|
Enable: true,
|
||||||
Address: xnet.Host{
|
Address: xnet.Host{
|
||||||
Name: "localhost",
|
Name: "localhost",
|
||||||
Port: (xnet.Port(opts.Port)),
|
Port: xnet.Port(opts.Port),
|
||||||
IsPortSet: true,
|
IsPortSet: true,
|
||||||
},
|
},
|
||||||
Subject: "test",
|
Subject: "test",
|
||||||
@@ -85,7 +85,7 @@ func TestNatsConnToken(t *testing.T) {
|
|||||||
Enable: true,
|
Enable: true,
|
||||||
Address: xnet.Host{
|
Address: xnet.Host{
|
||||||
Name: "localhost",
|
Name: "localhost",
|
||||||
Port: (xnet.Port(opts.Port)),
|
Port: xnet.Port(opts.Port),
|
||||||
IsPortSet: true,
|
IsPortSet: true,
|
||||||
},
|
},
|
||||||
Subject: "test",
|
Subject: "test",
|
||||||
@@ -116,7 +116,7 @@ func TestNatsConnNKeySeed(t *testing.T) {
|
|||||||
Enable: true,
|
Enable: true,
|
||||||
Address: xnet.Host{
|
Address: xnet.Host{
|
||||||
Name: "localhost",
|
Name: "localhost",
|
||||||
Port: (xnet.Port(opts.Port)),
|
Port: xnet.Port(opts.Port),
|
||||||
IsPortSet: true,
|
IsPortSet: true,
|
||||||
},
|
},
|
||||||
Subject: "test",
|
Subject: "test",
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ func TestNatsConnTLSCustomCA(t *testing.T) {
|
|||||||
Enable: true,
|
Enable: true,
|
||||||
Address: xnet.Host{
|
Address: xnet.Host{
|
||||||
Name: "localhost",
|
Name: "localhost",
|
||||||
Port: (xnet.Port(opts.Port)),
|
Port: xnet.Port(opts.Port),
|
||||||
IsPortSet: true,
|
IsPortSet: true,
|
||||||
},
|
},
|
||||||
Subject: "test",
|
Subject: "test",
|
||||||
@@ -56,7 +56,7 @@ func TestNatsConnTLSCustomCAHandshakeFirst(t *testing.T) {
|
|||||||
Enable: true,
|
Enable: true,
|
||||||
Address: xnet.Host{
|
Address: xnet.Host{
|
||||||
Name: "localhost",
|
Name: "localhost",
|
||||||
Port: (xnet.Port(opts.Port)),
|
Port: xnet.Port(opts.Port),
|
||||||
IsPortSet: true,
|
IsPortSet: true,
|
||||||
},
|
},
|
||||||
Subject: "test",
|
Subject: "test",
|
||||||
@@ -80,7 +80,7 @@ func TestNatsConnTLSClientAuthorization(t *testing.T) {
|
|||||||
Enable: true,
|
Enable: true,
|
||||||
Address: xnet.Host{
|
Address: xnet.Host{
|
||||||
Name: "localhost",
|
Name: "localhost",
|
||||||
Port: (xnet.Port(opts.Port)),
|
Port: xnet.Port(opts.Port),
|
||||||
IsPortSet: true,
|
IsPortSet: true,
|
||||||
},
|
},
|
||||||
Subject: "test",
|
Subject: "test",
|
||||||
|
|||||||
@@ -1806,8 +1806,8 @@ func (ww *wsWriter) writeFrame(w io.Writer, f ws.Frame) error {
|
|||||||
const (
|
const (
|
||||||
bit0 = 0x80
|
bit0 = 0x80
|
||||||
len7 = int64(125)
|
len7 = int64(125)
|
||||||
len16 = int64(^(uint16(0)))
|
len16 = int64(^uint16(0))
|
||||||
len64 = int64(^(uint64(0)) >> 1)
|
len64 = int64(^uint64(0) >> 1)
|
||||||
)
|
)
|
||||||
|
|
||||||
bts := ww.tmp[:]
|
bts := ww.tmp[:]
|
||||||
|
|||||||
@@ -247,7 +247,7 @@ func (c ChecksumType) StringFull() string {
|
|||||||
|
|
||||||
// FullObjectRequested will return if the checksum type indicates full object checksum was requested.
|
// FullObjectRequested will return if the checksum type indicates full object checksum was requested.
|
||||||
func (c ChecksumType) FullObjectRequested() bool {
|
func (c ChecksumType) FullObjectRequested() bool {
|
||||||
return c&(ChecksumFullObject) == ChecksumFullObject || c.Is(ChecksumCRC64NVME)
|
return c&ChecksumFullObject == ChecksumFullObject || c.Is(ChecksumCRC64NVME)
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsMultipartComposite returns true if the checksum is multipart and full object was not requested.
|
// IsMultipartComposite returns true if the checksum is multipart and full object was not requested.
|
||||||
|
|||||||
@@ -58,10 +58,10 @@ func WaitPipe() (*PipeReader, *PipeWriter) {
|
|||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
return &PipeReader{
|
return &PipeReader{
|
||||||
PipeReader: r,
|
PipeReader: r,
|
||||||
wait: wg.Wait,
|
wait: wg.Wait,
|
||||||
}, &PipeWriter{
|
}, &PipeWriter{
|
||||||
PipeWriter: w,
|
PipeWriter: w,
|
||||||
done: wg.Done,
|
done: wg.Done,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user