// Copyright (c) 2026 Feng Ruohang // SPDX-License-Identifier: AGPL-3.0-or-later // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "bytes" "context" "encoding/json" "encoding/pem" "errors" "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" "sync" "sync/atomic" "testing" "time" "github.com/minio/kms-go/kes" "github.com/minio/minio/internal/config" "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/kms" ) // Use the configured KES client over TLS, rather than a mocked KMS interface: // cancellation must survive every layer, including the HTTP request context. func connectCancellationKES(t *testing.T, handler http.Handler) *kms.KMS { t.Helper() for _, key := range []string{kms.EnvKMSEndpoint, kms.EnvKMSSecretKey, kms.EnvKMSSecretKeyFile} { t.Setenv(key, "") mustIAM(t, os.Unsetenv(key)) } server := httptest.NewTLSServer(handler) t.Cleanup(server.Close) ca := filepath.Join(t.TempDir(), "ca.pem") mustIAM(t, os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600)) key, err := kes.GenerateAPIKey(nil) mustIAM(t, err) t.Setenv(kms.EnvKESEndpoint, server.URL) t.Setenv(kms.EnvKESAPIKey, key.String()) t.Setenv(kms.EnvKESDefaultKey, "cancellation-key") t.Setenv(kms.EnvKESServerCA, ca) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) k, err := kms.Connect(ctx, nil) mustIAM(t, err) return k } func cancellationDecryptHandler(t *testing.T, k *kms.KMS, before func(*http.Request)) http.Handler { t.Helper() return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // KMS Connect periodically describes the default key to keep it cached. // This maintenance request is independent of the decrypt under test. if r.Method == http.MethodGet && r.URL.Path == "/v1/key/describe/cancellation-key" { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"name": "cancellation-key"}) return } if r.URL.Path != "/v1/key/decrypt/cancellation-key" { t.Errorf("unexpected KES path: %s", r.URL.Path) w.WriteHeader(http.StatusNotFound) return } var request struct { Ciphertext []byte `json:"ciphertext"` Context []byte `json:"context"` } if err := json.NewDecoder(r.Body).Decode(&request); err != nil { t.Error(err) return } // Drain the HTTP/1 body so net/http can observe a peer disconnect while // the handler is blocked in the simulated KES operation. _, _ = io.Copy(io.Discard, r.Body) _ = r.Body.Close() before(r) if r.Context().Err() != nil { return } var aad kms.Context if err := json.Unmarshal(request.Context, &aad); err != nil { t.Error(err) return } plain, err := k.Decrypt(r.Context(), &kms.DecryptRequest{Name: "cancellation-key", Ciphertext: request.Ciphertext, AssociatedData: aad}) w.Header().Set("Content-Type", "application/json") if err != nil { w.WriteHeader(http.StatusBadRequest) _ = json.NewEncoder(w).Encode(map[string]string{"message": kes.ErrDecrypt.Error()}) return } _ = json.NewEncoder(w).Encode(map[string][]byte{"plaintext": plain}) }) } func TestKMSRequestCancellation(t *testing.T) { for _, operation := range []string{"iam", "sse-list"} { for _, expiry := range []string{"cancel", "deadline"} { t.Run(operation+"/"+expiry, func(t *testing.T) { builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{42}, 32)) mustIAM(t, err) entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) var calls atomic.Int32 var enteredOnce, canceledOnce sync.Once unblock := sync.OnceFunc(func() { close(release) }) client := connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(r *http.Request) { calls.Add(1) enteredOnce.Do(func() { close(entered) }) select { case <-r.Context().Done(): canceledOnce.Do(func() { close(canceled) }) case <-release: } })) t.Cleanup(unblock) prior := GlobalKMS GlobalKMS = client t.Cleanup(func() { GlobalKMS = prior }) var run func(context.Context) error if operation == "iam" { const name = "config/iam/policies/cancel/policy.json" plaintext := []byte(`{"Version":1}`) ciphertext, err := config.EncryptBytes(builtin, plaintext, kms.Context{minioMetaBucket: minioMetaBucket + "/" + name}) mustIAM(t, err) run = func(ctx context.Context) error { got, err := decryptData(ctx, ciphertext, name) if err == nil && !bytes.Equal(got, plaintext) { return errors.New("config plaintext changed") } return err } } else { var metadata []map[string]string var buckets, names []string var expected []crypto.ObjectKey for i := range 3 { name := fmt.Sprintf("object-%d", i) dek, err := builtin.GenerateKey(context.Background(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{"bucket": "bucket/" + name}}) mustIAM(t, err) key := crypto.GenerateKey(dek.Plaintext, nil) sealed := key.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), "bucket", name) metadata = append(metadata, crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed)) buckets, names = append(buckets, "bucket"), append(names, name) expected = append(expected, key) } run = func(ctx context.Context) error { keys, err := crypto.S3.UnsealObjectKeys(ctx, client, metadata, buckets, names) if err != nil { return err } if len(keys) != len(expected) { return errors.New("partial key list returned as success") } for i := range keys { if keys[i] != expected[i] { return errors.New("object key or ordering changed") } } return nil } } ctx, cancel := context.WithCancel(context.Background()) if expiry == "deadline" { cancel() ctx, cancel = context.WithTimeout(context.Background(), 300*time.Millisecond) } defer cancel() done := make(chan error, 1) go func() { done <- run(ctx) }() select { case <-entered: case err := <-done: t.Fatalf("decrypt returned before reaching KES: %v (type=%v)", err, client.Type) case <-time.After(5 * time.Second): t.Fatal("decrypt did not reach KES") } want := context.DeadlineExceeded if expiry == "cancel" { cancel() want = context.Canceled } select { case err := <-done: if !errors.Is(err, want) { t.Fatalf("got %v, want %v", err, want) } // The current KES SDK may sleep twice for up to 999ms after a // deadline error. Include that existing backoff plus the 300ms // request deadline; cancellation must still stop new decrypts. case <-time.After(3 * time.Second): unblock() <-done t.Fatal("request exceeded the cancellation and KES retry budget") } awaitIAMPolicyReload(t, canceled) if n := calls.Load(); n != 1 { t.Fatalf("cancellation launched a fallback or more object decryptions: %d calls", n) } unblock() mustIAM(t, run(context.Background())) before := calls.Load() cctx, ccancel := context.WithCancel(context.Background()) ccancel() if err := run(cctx); !errors.Is(err, context.Canceled) || calls.Load() != before { t.Fatal("already-canceled request still issued decryption work") } }) } } } func TestIAMDecryptAssociatedDataCompatibility(t *testing.T) { builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{42}, 32)) mustIAM(t, err) var calls atomic.Int32 client := connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(*http.Request) { calls.Add(1) })) prior := GlobalKMS GlobalKMS = client t.Cleanup(func() { GlobalKMS = prior }) const name = "config/iam/policies/compat/policy.json" plaintext := []byte(`{"Version":1}`) for _, legacy := range []bool{false, true} { t.Run(fmt.Sprint(legacy), func(t *testing.T) { aad, wantCalls := minioMetaBucket+"/"+name, int32(1) if legacy { aad, wantCalls = name, 2 } ciphertext, err := config.EncryptBytes(builtin, plaintext, kms.Context{minioMetaBucket: aad}) mustIAM(t, err) before := calls.Load() got, err := decryptData(context.Background(), ciphertext, name) mustIAM(t, err) if !bytes.Equal(got, plaintext) || calls.Load()-before != wantCalls { t.Fatal("current/legacy encrypted IAM records no longer decrypt correctly") } }) } }