mirror of
https://github.com/pgsty/minio.git
synced 2026-10-28 11:37:19 +03:00
Compare commits
49 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dd905b29f4 | |||
| 2f728c7643 | |||
| c3ad836aed | |||
| dc6dc3d806 | |||
| dcc2f20ba8 | |||
| a13efd5760 | |||
| 39278e068e | |||
| 263ee0510d | |||
| 641eb51a40 | |||
| 45d6e5d065 | |||
| dabc894132 | |||
| fb08ca7b6a | |||
| 034f7bbf73 | |||
| aec7ddac46 | |||
| 6921f8c186 | |||
| 25d928c9df | |||
| ee3e7ea716 | |||
| 2a04d5f485 | |||
| 9449e6a4a8 | |||
| 9c4bb39571 | |||
| 86ce8d6012 | |||
| b15aa97c94 | |||
| f54c04846e | |||
| 6fdea3b8d9 | |||
| 380fcbfe47 | |||
| a02fa9644c | |||
| 652a1c09d3 | |||
| c86df14f4d | |||
| 03875d61e3 | |||
| 1865985aa1 | |||
| 076a13f9d2 | |||
| 9a4e089f56 | |||
| 9255f443a6 | |||
| ea7c794a94 | |||
| 16c21428d9 | |||
| 7e83f9950a | |||
| 1e3ab56b06 | |||
| 5ad1f344b7 | |||
| 8a4097309f | |||
| 0501a198e3 | |||
| d2eb0ade91 | |||
| 5c074efd1d | |||
| 03fe5e9c21 | |||
| 0b1c21cb4b | |||
| b55495aef0 | |||
| d8b77e5f3b | |||
| cd1f6cec3c | |||
| 4f17a23e33 | |||
| be44ff2669 |
@@ -148,7 +148,9 @@ jobs:
|
||||
for rpm_file in finalize/packages/*.rpm; do
|
||||
signature_output="$(sudo rpmkeys --checksig --verbose "${rpm_file}")"
|
||||
printf '%s\n' "${signature_output}"
|
||||
if ! printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]' | grep -q "key id ${key_id}: ok"; then
|
||||
normalized_signature="$(printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]')"
|
||||
full_fingerprint="$(printf '%s' "${expected_fingerprint}" | tr '[:upper:]' '[:lower:]')"
|
||||
if ! printf '%s\n' "${normalized_signature}" | grep -Eq "key id ${key_id}: ok|key fingerprint: ${full_fingerprint}: ok"; then
|
||||
echo "Signature verification failed for ${rpm_file}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -118,6 +118,14 @@ jobs:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run maintenance storage timeout and compatibility tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd ./internal/grid ./internal/ioutil -run '^Test(StorageTimeout|StorageRollback|StoragePrerelease|MaintenancePartialFailedOverwriteAvailability|SingleRequest|WithDeadline)' -count=1 -timeout=5m
|
||||
|
||||
- name: Compile deferred consistency diagnostics
|
||||
run: go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+41
-7
@@ -1,6 +1,12 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
## RELEASE.2026-10-26T00-00-00Z
|
||||
|
||||
Package version: `20261026000000.0.0`; published from source `c3ad836aed0d`.
|
||||
Artifacts are listed in the [GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z).
|
||||
The coordinated [English release notes](https://silo.pgsty.com/blog/release/silo-20261026/)
|
||||
and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261026/)
|
||||
cover all components, upgrade guidance, and validation scope.
|
||||
|
||||
- Fix LDAP login for long normalized DNs without hiding unreadable IAM
|
||||
revocation records. Only length errors for valid keys with an ASCII path
|
||||
@@ -8,12 +14,17 @@
|
||||
Thanks to [@910patin](https://github.com/910patin) for the original fix and
|
||||
reproduction in [#250](https://github.com/pgsty/silo/pull/250).
|
||||
- Build the server and standalone debugging tools with Go 1.27.3 and refresh
|
||||
Go dependencies, including silo-pkg v3.15.1, the published mcli 20261025
|
||||
source and upstream minio-go `ab93db7ba435` for multipart, read/seek and
|
||||
batch-delete fixes. Retain output-format and NetBSD compatibility selections.
|
||||
- Embed the unreleased Console 2.5.0 candidate `258717be077b`, including its
|
||||
GUI update, diagnostic WebSocket closing handshake and complete health-report
|
||||
decoding. This selects a source commit, not a new Console release.
|
||||
Go dependencies, including silo-pkg v3.15.2, mcli 20261026
|
||||
and upstream minio-go `320af050c445` for multipart, read/seek,
|
||||
batch-delete and bucket-replication removal fixes. Retain output-format and NetBSD compatibility selections.
|
||||
- Embed Console v2.5.1 source `fff1618d9abb`, including its GUI update,
|
||||
diagnostic WebSocket closing handshake, complete health-report decoding,
|
||||
group-membership preservation, OIDC group permissions, TLS-proxy session
|
||||
cookies, credential exports and accessibility fixes.
|
||||
- Bundle mcli 20261026 in container builds, the verified client installer and
|
||||
Helm client jobs. Include client identity isolation, resync status and rate
|
||||
fixes, and bounded batch-status fallback. Scripts must handle the additive
|
||||
batch `pending` state and nonzero exits for partial resync mutations.
|
||||
- Read current bucket inventory in account-info responses so Console refreshes
|
||||
immediately reflect successful creation and deletion. This management endpoint
|
||||
now performs a live list instead of returning a cached inventory.
|
||||
@@ -21,11 +32,34 @@
|
||||
Peer and etcd notifications reload committed state, so delayed delete events
|
||||
preserve subsequent edits. Keep deletion revisions, policy-in-use checks and
|
||||
custom-policy deletion behavior; storage failures retain the previous policy.
|
||||
- Reload policies without holding the authentication-cache lock, retain IAM
|
||||
write ordering and publish replacements only after a complete successful load.
|
||||
Propagate cancellation through IAM/KMS and SSE-S3 decryption without shortening
|
||||
the caller's policy-query deadline.
|
||||
- Preserve authoritative copies when a multi-pool delete or bulk cleanup fails,
|
||||
choose an available pool for a new delete marker, and keep tier data referenced
|
||||
by surviving copies. This does not introduce strict cross-pool read recovery.
|
||||
- Preserve remote tier data when lifecycle expiry only adds a delete marker,
|
||||
including non-null versions in suspended buckets. Forward `FULL_OBJECT`
|
||||
checksum type for multipart replicas. Thanks to
|
||||
[@botdeniska](https://github.com/botdeniska) for these lifecycle, replication
|
||||
and multi-pool deletion contributions.
|
||||
- Preserve Kafka's default broker protocol 2.1 and terminal-width behavior;
|
||||
fix legacy DARE v1 key unsealing and verify DARE v1/v2 and MinKMS TLS compatibility.
|
||||
- Forward storage RPC delete options, including the previous data directory
|
||||
needed by overwrite rollback. Preserve existing drive/RPC timeouts, request
|
||||
cancellation, object locking, read-quorum and healing behavior.
|
||||
- Update bundled curl to 8.23.0 and require patched static libraries. Apply
|
||||
[Red Hat's compromised-CA mitigation](https://access.redhat.com/security/vulnerabilities/RHSB-2026-012);
|
||||
the root blocklist also rejects legitimate chains under the affected roots.
|
||||
- Ship Windows platform archives as ZIP; Linux and macOS retain tar.gz.
|
||||
|
||||
Storage timeouts retain their existing meaning: a returned timeout does not
|
||||
prove that the underlying disk operation stopped or that a write did not
|
||||
commit. Late commits or delayed rollback can affect newer writes; stale reads
|
||||
remain possible after partial failures, lease loss or network partitions. This
|
||||
release does not provide distributed fencing or repair previously inconsistent
|
||||
objects.
|
||||
|
||||
These source changes are not included in the published 20261018 artifacts.
|
||||
|
||||
|
||||
@@ -53,6 +53,20 @@ Before opening a pull request:
|
||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||
`.minio.sys`, ARN, module/import-path, or serialized compatibility name.
|
||||
|
||||
The maintenance branch retains existing storage timeouts, cancellation,
|
||||
object locking, read-quorum and healing behavior. The normal suite checks local
|
||||
and RPC timeout/cancellation, remote rollback options, and readability after a
|
||||
failed partial overwrite. A timeout does not prove that a disk worker stopped
|
||||
or that a write did not commit.
|
||||
|
||||
Stronger mutation lifetime, rollback ownership, generation ordering, strict-read
|
||||
and cross-pool recovery diagnostics are preserved behind `consistency_strict`.
|
||||
Compile them with `go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'`
|
||||
and run individual scenarios explicitly. They describe guarantees outside the
|
||||
maintenance contract and may fail; report their results separately from the
|
||||
normal suite. They must not be used to claim distributed fencing or repair of
|
||||
previously inconsistent objects.
|
||||
|
||||
### Commit changes
|
||||
|
||||
After verification, commit your changes with a concise message and a DCO
|
||||
|
||||
+42
-36
File diff suppressed because one or more lines are too long
@@ -17,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-10-18T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=b5392a642dbc9685b517ee901f032d1d5e88f41704436f0f08dde6060b577db3
|
||||
ARG MC_ARM64_SHA256=17f3eb694ed34e155b331b1215832140749656d8d4b60e2e3a4d0573b8f320d7
|
||||
ARG MC_VERSION=RELEASE.2026-10-26T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=54bd131039dad6677f083a75207b28f82818c067c8e65fae1973f51510d268e1
|
||||
ARG MC_ARM64_SHA256=9bb294f1912f8c6b9ac08c9a0b8d2964b86306a7c440b987ee5f0570d18de3c5
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/"><img alt="Website" src="https://img.shields.io/badge/Website-silo.pgsty.com-1d588c"></a>
|
||||
<a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
|
||||
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
|
||||
</p>
|
||||
@@ -35,19 +35,13 @@
|
||||
> [!NOTE]
|
||||
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
|
||||
|
||||
## SILO 20261018000000
|
||||
## SILO 20261026000000
|
||||
|
||||
[SILO 20261018000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z) is available as platform archives, Linux packages, and standard or distroless Docker images for amd64 and arm64. The release tag is `RELEASE.2026-10-18T00-00-00Z`; the package version is `20261018000000.0.0`.
|
||||
[SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) is available as platform archives, Linux packages, and standard or distroless Docker images for amd64 and arm64. The release tag is `RELEASE.2026-10-26T00-00-00Z`; the package version is `20261026000000.0.0`, built from source `c3ad836aed0d`.
|
||||
|
||||
This release strengthens Object Lock and IAM authorization, makes scoped STS revocation durable across processes and sites, protects concurrent updates during pool migration, and improves LIST recovery, multipart cleanup, and S3 error handling. It includes Console v2.4.2 and silo-pkg v3.14.3, and is built with Go 1.27.2. Use mcli 20261018 for administration.
|
||||
This release fixes LDAP long-DN login, IAM policy refresh, multi-pool deletion safety, tiered-object expiry, multipart replication checksums, and remote rollback-option propagation. It embeds Console v2.5.1 (`fff1618d9abb`) and uses silo-pkg v3.15.2, Go 1.27.3, and upstream minio-go `v7.4.2-0.20261025165926-320af050c445`. Use mcli 20261026 for administration; the container images bundle it as `mcli`.
|
||||
|
||||
Read the [release notes](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z) before upgrading, especially the removal of strict multipart listing, coordinated server upgrades, and measured performance costs. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people behind the code, reports, and documentation.
|
||||
|
||||
Current source is unreleased: Go 1.27.3, silo-pkg v3.15.1, mcli 20261025,
|
||||
upstream minio-go `ab93db7ba435`, and the Console 2.5.0 candidate `258717be077b`.
|
||||
It includes the Server IAM cache repair needed by Console's **Reset to default**
|
||||
action. See [CHANGELOG.md](CHANGELOG.md#unreleased); the published artifacts above
|
||||
retain their original component versions.
|
||||
The Server IAM cache repair enables Console's **Reset to default** action. Existing storage timeouts and cancellation behavior remain: a timeout does not prove that a disk operation stopped or a write did not commit. Read the [coordinated release notes](https://silo.pgsty.com/blog/release/silo-20261026/) and [remaining storage limits](CHANGELOG.md#release2026-10-26t00-00-00z) before upgrading. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people behind the code, reports, and documentation.
|
||||
|
||||
## Overview
|
||||
|
||||
@@ -68,7 +62,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z server /data --console-address ":9001"
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
<p align="center">
|
||||
|
||||
+6
-11
@@ -24,7 +24,7 @@
|
||||
<p align="center">
|
||||
<a href="https://silo.pgsty.com/zh/"><img alt="官网" src="https://img.shields.io/badge/%E5%AE%98%E7%BD%91-silo.pgsty.com%2Fzh-1d588c"></a>
|
||||
<a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a>
|
||||
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
|
||||
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
|
||||
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
|
||||
</p>
|
||||
@@ -35,18 +35,13 @@
|
||||
> [!NOTE]
|
||||
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
|
||||
|
||||
## SILO 20261018000000
|
||||
## SILO 20261026000000
|
||||
|
||||
[SILO 20261018000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z) 已正式发布,提供各平台归档包、Linux 软件包,以及 amd64/arm64 标准版和 distroless Docker 镜像。版本标签为 `RELEASE.2026-10-18T00-00-00Z`,包版本为 `20261018000000.0.0`。
|
||||
[SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) 已正式发布,提供各平台归档包、Linux 软件包,以及 amd64/arm64 标准版和 distroless Docker 镜像。版本标签为 `RELEASE.2026-10-26T00-00-00Z`,包版本为 `20261026000000.0.0`,构建源码为 `c3ad836aed0d`。
|
||||
|
||||
本版加强 Object Lock 与 IAM 鉴权,实现跨进程和站点的持久 STS 撤销,保护多池迁移中的并发更新,并改进 LIST 恢复、分片清理与 S3 错误处理。内置 Console v2.4.2、silo-pkg v3.14.3,使用 Go 1.27.2 构建,配套管理客户端为 mcli 20261018。
|
||||
本版修复 LDAP 长 DN 登录、IAM 策略刷新、多池删除安全、分层对象过期、分片复制校验和及远端回滚参数透传。内置 Console v2.5.1(`fff1618d9abb`),使用 silo-pkg v3.15.2、Go 1.27.3 与上游 minio-go `v7.4.2-0.20261025165926-320af050c445`。配套管理客户端为 mcli 20261026,容器镜像内置该版本的 `mcli`。
|
||||
|
||||
升级前请阅读[发行说明](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z),特别是 strict 分片列表的移除、服务端协调升级要求与实测性能成本。代码、问题报告和文档贡献者见 [CONTRIBUTORS.md](CONTRIBUTORS.md)。
|
||||
|
||||
当前源码尚未发布:使用 Go 1.27.3、silo-pkg v3.15.1、mcli 20261025、上游
|
||||
minio-go `ab93db7ba435`,并内嵌 Console 2.5.0 候选提交 `258717be077b`。
|
||||
其中包含 Console **Reset to default(恢复默认)** 所需的 Server IAM 缓存修复。
|
||||
详见 [CHANGELOG.md](CHANGELOG.md#unreleased);上述已发布制品仍保留原有组件版本。
|
||||
Server IAM 缓存修复使 Console **Reset to default(恢复默认)** 操作即时生效。存储超时与取消行为保持既有语义:超时不代表底层磁盘操作已停止,也不代表写入未提交。升级前请阅读[联合发布说明](https://silo.pgsty.com/zh/blog/release/silo-20261026/)与[仍存的存储限制](CHANGELOG.md#release2026-10-26t00-00-00z)。代码、问题报告和文档贡献者见 [CONTRIBUTORS.md](CONTRIBUTORS.md)。
|
||||
|
||||
## 概述
|
||||
|
||||
@@ -67,7 +62,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
|
||||
-e MINIO_ROOT_USER=minioadmin \
|
||||
-e MINIO_ROOT_PASSWORD=change-me-long-password \
|
||||
-v "$PWD/data:/data" \
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-18T00-00-00Z server /data --console-address ":9001"
|
||||
docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
|
||||
```
|
||||
|
||||
<p align="center">
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-10-18T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-10-26T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type abandonedCleanupCommitDisk struct {
|
||||
StorageAPI
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d abandonedCleanupCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
var success int64
|
||||
gate := &scopeStagedContext{
|
||||
Context: ctx, meta: filepath.Join(d.Endpoint().Path, dv, dp, fi.DataDir, "part.1"),
|
||||
entered: d.entered, release: d.release,
|
||||
tracker: healthDiskCtxValue{lastSuccess: &success},
|
||||
}
|
||||
return d.StorageAPI.RenameData(gate, sv, sp, fi, dv, dp, o)
|
||||
}
|
||||
|
||||
// A lost namespace lease can admit cleanup while a disk transaction is between
|
||||
// moving its external shards and replacing xl.meta. Those shards are live work,
|
||||
// even though the old metadata does not refer to them yet. Cleanup must wait.
|
||||
func TestMutationAbandonedCleanupWaitsForCommit(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-during-commit"
|
||||
body := strings.Repeat("new payload", 200000)
|
||||
mutationPut(t, z, bucket, object, 0, strings.Repeat("old payload", 200000), time.Now().Add(-time.Minute))
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = abandonedCleanupCommitDisk{d, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
|
||||
committed <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before shard barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not move all external shards")
|
||||
}
|
||||
}
|
||||
t.Log("All eight new external shard directories exist; old xl.meta is still active")
|
||||
cleaned := make(chan error, len(disks))
|
||||
for _, d := range disks {
|
||||
go func(d StorageAPI) { cleaned <- d.CleanAbandonedData(context.Background(), bucket, object) }(d)
|
||||
}
|
||||
early := 0
|
||||
timer := time.NewTimer(150 * time.Millisecond)
|
||||
wait:
|
||||
for {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
early++
|
||||
t.Errorf("abandoned-data cleanup overtook unfinished commit: %v", err)
|
||||
if early == len(disks) {
|
||||
break wait
|
||||
}
|
||||
case <-timer.C:
|
||||
break wait
|
||||
}
|
||||
}
|
||||
timer.Stop()
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err != nil {
|
||||
t.Fatalf("PUT failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not finish")
|
||||
}
|
||||
for i := early; i < len(disks); i++ {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup did not finish")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("acknowledged PUT unreadable after abandoned-data cleanup: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("acknowledged external shards lost: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup can also arrive after a disk has committed but before its delayed
|
||||
// reply lets the coordinator decide to undo a failed quorum. The previous
|
||||
// generation's backup is still owned by that unfinished transaction.
|
||||
func TestMutationAbandonedCleanupPreservesUndoBackup(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-before-undo"
|
||||
body := strings.Repeat("last acknowledged", 150000)
|
||||
base := time.Now().Add(-time.Minute)
|
||||
mutationPut(t, z, bucket, object, 0, body, base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = delayedRenameResult{d, i >= 4, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
next := strings.Repeat("failed update", 180000)
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(next), int64(len(next)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Second)})
|
||||
committed <- err
|
||||
}()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before reply barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("disk commit did not reach reply barrier")
|
||||
}
|
||||
}
|
||||
for _, d := range disks[:4] {
|
||||
if err := d.CleanAbandonedData(t.Context(), bucket, object); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err == nil {
|
||||
t.Fatal("four-of-eight commit unexpectedly reached write quorum")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("failed PUT did not finish undo")
|
||||
}
|
||||
swap(disks)
|
||||
for i, d := range disks {
|
||||
fi, err := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !fi.ModTime.Equal(base) {
|
||||
t.Errorf("disk%d lost last acknowledged generation after failed PUT: mtime=%v err=%v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("last acknowledged value unreadable after cleanup and undo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("last acknowledged value lost after cleanup and undo: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
@@ -511,14 +511,13 @@ func testSiteReplicationStatusDetectsCorsTimestampMismatch(obj ObjectLayer, _ st
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-status-service", "cors-status-service-secret-key")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-status-service",
|
||||
secretKey: "cors-status-service-secret-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
@@ -585,14 +584,13 @@ func testSiteReplicationStatusCountsCorsPerSite(obj ObjectLayer, _ string, bucke
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-count-service", "cors-count-service-secret-key")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-count-service",
|
||||
secretKey: "cors-count-service-secret-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
@@ -1190,14 +1188,13 @@ func testCorsReplicationDispatchStatusHealReload(obj ObjectLayer, _ string, buck
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-integration-svc", "cors-integration-service-secret")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-integration-svc",
|
||||
secretKey: "cors-integration-service-secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
// Tests create endpoints with ellipses and without.
|
||||
func TestCreateServerEndpoints(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
args []string
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
@@ -25,9 +26,42 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/dnscache"
|
||||
)
|
||||
|
||||
type endpointTestResolver struct {
|
||||
dnscache.DNSResolver
|
||||
}
|
||||
|
||||
func (r endpointTestResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
|
||||
// These endpoint fixtures test locality and parsing, not public DNS.
|
||||
for i, name := range []string{"example.org", "example.com", "example.net", "example.edu"} {
|
||||
if host == name {
|
||||
return []string{fmt.Sprintf("203.0.113.%d", i+1)}, nil
|
||||
}
|
||||
}
|
||||
return r.DNSResolver.LookupHost(ctx, host)
|
||||
}
|
||||
|
||||
func useTestEndpointDNS(t *testing.T) {
|
||||
t.Helper()
|
||||
previous := globalDNSCache
|
||||
globalDNSCache = &dnscache.Resolver{Resolver: endpointTestResolver{net.DefaultResolver}}
|
||||
t.Cleanup(func() { globalDNSCache = previous })
|
||||
}
|
||||
|
||||
func useStaticTestEndpoints(t *testing.T) {
|
||||
t.Helper()
|
||||
useTestEndpointDNS(t)
|
||||
// These fixtures deliberately use localhost, including inside containers.
|
||||
previous := orchestrated
|
||||
orchestrated = false
|
||||
t.Cleanup(func() { orchestrated = previous })
|
||||
}
|
||||
|
||||
func TestNewEndpoint(t *testing.T) {
|
||||
useTestEndpointDNS(t)
|
||||
u2, _ := url.Parse("https://example.org/path")
|
||||
u4, _ := url.Parse("http://192.168.253.200/path")
|
||||
rootSlashFoo, _ := filepath.Abs("/foo")
|
||||
@@ -130,6 +164,7 @@ func TestNewEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCreateEndpoints(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
@@ -353,6 +388,7 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
|
||||
// is considered a remote service from localhost:9000 perspective.
|
||||
func TestGetLocalPeer(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
@@ -393,6 +429,7 @@ func TestGetLocalPeer(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetRemotePeers(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// All metadata is valid and readable. Only the completed generations differ.
|
||||
// No timing, missing files, synthetic read error or malformed metadata is needed.
|
||||
func TestReadGenerationConflictIsNotAbsence(t *testing.T) {
|
||||
raw := make([]RawFileInfo, 8)
|
||||
for i := range raw {
|
||||
fi := newFileInfo("object", 4, 4)
|
||||
fi.Erasure.Index = i + 1
|
||||
fi.ModTime = time.Unix(100+int64(i%4), 0)
|
||||
fi.DataDir = mustGetUUID()
|
||||
fi.Metadata = map[string]string{"etag": fmt.Sprint(i % 4)}
|
||||
var meta xlMetaV2
|
||||
if err := meta.AddVersion(fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf, err := meta.AppendTo(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw[i] = RawFileInfo{Buf: buf}
|
||||
}
|
||||
_, errs := pickLatestQuorumFilesInfo(t.Context(), raw, make([]error, len(raw)), "bucket", "object", true, false)
|
||||
err := reduceReadQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 4)
|
||||
if api := toAPIError(t.Context(), toObjectErr(err, "bucket", "object")); api.HTTPStatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("readable conflicting generations reported as absence: %v (%+v)", err, api)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsReadGenerationConflictDoesNotExposeOldValue(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"})
|
||||
for _, tc := range []struct {
|
||||
pool int
|
||||
mode string
|
||||
}{
|
||||
{0, "split"},
|
||||
{1, "split"},
|
||||
{0, "older-quorum"},
|
||||
{1, "older-quorum"},
|
||||
{0, "newer-quorum"},
|
||||
{1, "newer-quorum"},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s/pool-%d", tc.mode, tc.pool), func(t *testing.T) {
|
||||
broken := tc.pool
|
||||
object := fmt.Sprintf("conflict-%s-%d", tc.mode, broken)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{MTime: base})
|
||||
set := z.serverPools[broken].getHashedSet(object)
|
||||
disks := set.getDisks()
|
||||
saved := make([][]byte, len(disks))
|
||||
savedParts := make([]map[string][]byte, len(disks))
|
||||
for generation := range 4 {
|
||||
putConsistencyObject(t, z, bucket, object, broken, fmt.Sprintf("newer-%d", generation),
|
||||
ObjectOptions{MTime: base.Add(time.Duration(generation+1) * time.Minute)})
|
||||
for i, disk := range disks {
|
||||
g := i % 4
|
||||
switch tc.mode {
|
||||
case "older-quorum":
|
||||
g = 0
|
||||
if i >= len(disks)-3 {
|
||||
g = i - (len(disks) - 3) + 1
|
||||
}
|
||||
case "newer-quorum":
|
||||
g = 3
|
||||
if i < 3 {
|
||||
g = i
|
||||
}
|
||||
}
|
||||
if g != generation {
|
||||
continue
|
||||
}
|
||||
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved[i] = bytes.Clone(buf)
|
||||
savedParts[i] = readGenerationParts(t, disk, bucket, object, buf)
|
||||
}
|
||||
}
|
||||
for i, disk := range disks {
|
||||
for path, data := range savedParts[i] {
|
||||
if err := disk.WriteAll(ctx, bucket, path, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := disk.WriteAll(ctx, bucket, pathJoin(object, xlStorageFormatFile), saved[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
for _, method := range []string{http.MethodHead, http.MethodGet} {
|
||||
rec := consistencyRequest(t, router, method, bucket, object, version)
|
||||
want := http.StatusServiceUnavailable
|
||||
if tc.mode == "newer-quorum" {
|
||||
want = http.StatusOK
|
||||
}
|
||||
if rec.Code != want {
|
||||
t.Errorf("%s version=%q: want status=%d, got=%d body=%q", method, version, want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != "newer-3" {
|
||||
t.Errorf("latest quorum did not return its payload: %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
if tc.mode != "newer-quorum" {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
_, err := set.CopyObject(ctx, bucket, object, bucket, object,
|
||||
ObjectInfo{metadataOnly: true, UserDefined: map[string]string{"copy-test": "conflicting-source"}},
|
||||
ObjectOptions{VersionID: version}, ObjectOptions{})
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("COPY-to-self version=%q: want read-quorum error, got %v", version, err)
|
||||
}
|
||||
for _, scan := range []madmin.HealScanMode{madmin.HealNormalScan, madmin.HealDeepScan} {
|
||||
_, err := set.HealObject(ctx, bucket, object, version, madmin.HealOpts{ScanMode: scan, Remove: true})
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("heal version=%q scan=%v: want read-quorum error, got %v", version, scan, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, disk := range disks {
|
||||
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
|
||||
if err != nil || !bytes.Equal(buf, saved[i]) {
|
||||
t.Errorf("read or heal destroyed conflicting generation on disk %d: %v", i, err)
|
||||
}
|
||||
if tc.mode != "newer-quorum" {
|
||||
for path, data := range savedParts[i] {
|
||||
got, err := disk.ReadAll(ctx, bucket, path)
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Errorf("read or heal destroyed generation shard on disk %d: %s: %v", i, path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func readGenerationParts(t *testing.T, disk StorageAPI, bucket, object string, metadata []byte) map[string][]byte {
|
||||
t.Helper()
|
||||
fi, err := fileInfoFromRaw(RawFileInfo{Buf: metadata}, bucket, object, false, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts := make(map[string][]byte)
|
||||
if fi.InlineData() || fi.Size == 0 {
|
||||
return parts
|
||||
}
|
||||
// Later PUTs remove the previous data directory. Save its real bitrot
|
||||
// shards too, so restoring metadata never fabricates missing-part damage.
|
||||
for _, part := range fi.Parts {
|
||||
path := pathJoin(object, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
|
||||
data, err := disk.ReadAll(t.Context(), bucket, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts[path] = data
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func TestPoolsReadUnavailablePoolDoesNotExposeOldValue(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
for broken := range 2 {
|
||||
t.Run(fmt.Sprintf("broken-pool-%d", broken), func(t *testing.T) {
|
||||
object := fmt.Sprintf("unavailable-%d", broken)
|
||||
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{})
|
||||
set := z.serverPools[broken].getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
faulty := append([]StorageAPI(nil), disks...)
|
||||
for i := range faulty {
|
||||
faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
|
||||
}
|
||||
switchDisks(faulty)
|
||||
if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil || isErrObjectNotFound(err) {
|
||||
t.Errorf("unknown pool state allowed an old value: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type generationReadGateDisk struct {
|
||||
StorageAPI
|
||||
ready chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d generationReadGateDisk) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
|
||||
fi, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return fi, err
|
||||
}
|
||||
|
||||
func (d generationReadGateDisk) ReadVersion(ctx context.Context, orig, volume, path, version string, opts ReadOptions) (FileInfo, error) {
|
||||
fi, err := d.StorageAPI.ReadVersion(ctx, orig, volume, path, version, opts)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return fi, err
|
||||
}
|
||||
|
||||
func TestReadGenerationFastInfoWaitsForSlowDisk(t *testing.T) {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
t.Run("version="+version, func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "slow-newer-generation"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: base})
|
||||
saved := make([][]byte, len(disks)-1)
|
||||
for i := range saved {
|
||||
saved[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 0, "new", ObjectOptions{MTime: base.Add(time.Minute)})
|
||||
for i, meta := range saved {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ready, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
gate := generationReadGateDisk{StorageAPI: disk, ready: ready}
|
||||
if i == len(disks)-1 {
|
||||
gate.release = release
|
||||
}
|
||||
wrapped[i] = gate
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true, VersionID: version}, false)
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("metadata reads did not reach the barrier")
|
||||
}
|
||||
}
|
||||
early := false
|
||||
select {
|
||||
case err := <-done:
|
||||
early = true
|
||||
t.Errorf("fast read returned before the newer disk: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
if !early {
|
||||
select {
|
||||
case err := <-done:
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Fatalf("fast read exposed the old generation: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("fast read did not finish after the newer disk")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// An acknowledged recovery overwrite must resolve the unreadable pool, even
|
||||
// when another pool already has a readable copy of the same object.
|
||||
// Real inline metadata is combined in the same 15-old/1-new state used by
|
||||
// that regression. Capacity adapters only make destination choice deterministic.
|
||||
func TestPoolsMinorityOverwriteRecovery(t *testing.T) {
|
||||
for _, versioning := range []string{"unversioned", "versioned", "suspended"} {
|
||||
for _, mode := range []string{"single-pool", "dual-pool-only-inconsistent-copy", "dual-pool-readable-other-copy"} {
|
||||
t.Run(versioning+"/"+mode, func(t *testing.T) {
|
||||
testPoolsMinorityOverwriteRecovery(t, mode, versioning, "")
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, encryption := range []string{"SSE-C", "SSE-S3", "SSE-KMS"} {
|
||||
t.Run(encryption, func(t *testing.T) {
|
||||
testPoolsMinorityOverwriteRecovery(t, "dual-pool-readable-other-copy", "unversioned", encryption)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testPoolsMinorityOverwriteRecovery(t *testing.T, mode, versioning, encryption string) {
|
||||
t.Helper()
|
||||
var z *erasureServerPools
|
||||
if mode == "single-pool" {
|
||||
z, _ = mutationPools(t)
|
||||
} else {
|
||||
z, _ = consistencyPools(t)
|
||||
}
|
||||
const object = "review-retry"
|
||||
for _, pool := range z.serverPools {
|
||||
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
|
||||
defer restore()
|
||||
}
|
||||
bucket, router := conditionalPutBucket(t, z, versioning)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
oldBody := bytes.Repeat([]byte{'a'}, 8192)
|
||||
pendingBody := bytes.Repeat([]byte{'b'}, 8192)
|
||||
recoveryBody := bytes.Repeat([]byte{'c'}, 8192)
|
||||
seed := func(pool int, body []byte, when time.Time) {
|
||||
t.Helper()
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{MTime: when})
|
||||
}
|
||||
seed(0, oldBody, oldTime)
|
||||
disks := z.serverPools[0].getHashedSet(object).getDisks()
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
oldMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
seed(0, pendingBody, oldTime.Add(time.Minute))
|
||||
for i := 0; i < len(disks)-1; i++ {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "dual-pool-readable-other-copy" {
|
||||
seed(1, oldBody, oldTime.Add(-time.Minute))
|
||||
// Ordinary allocation can select this healthy other pool.
|
||||
// Force that permitted choice without altering metadata reads.
|
||||
restore := conditionalPutPool(t, z, object, 1)
|
||||
defer restore()
|
||||
}
|
||||
before := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
||||
if before.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("fixture must refuse ambiguous read; got %d", before.Code)
|
||||
}
|
||||
var headers, readHeaders map[string]string
|
||||
if encryption != "" {
|
||||
oldKMS, oldTLS := GlobalKMS, globalIsTLS
|
||||
GlobalKMS, globalIsTLS = kms.NewStub("recovery-key"), true
|
||||
defer func() { GlobalKMS, globalIsTLS = oldKMS, oldTLS }()
|
||||
headers = map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
|
||||
switch encryption {
|
||||
case "SSE-C":
|
||||
key := bytes.Repeat([]byte{0x42}, 32)
|
||||
digest := md5.Sum(key)
|
||||
headers = map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(digest[:]),
|
||||
}
|
||||
readHeaders = headers
|
||||
case "SSE-KMS":
|
||||
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||
headers[xhttp.AmzServerSideEncryptionKmsID] = "recovery-key"
|
||||
}
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
result := multipartConditionRequest(t, router, http.MethodPut, url, string(recoveryBody), headers)
|
||||
t.Logf("recovery PUT HTTP=%d response=%s", result.Code, result.Body.String())
|
||||
if result.Code != http.StatusOK {
|
||||
t.Fatalf("unconditional recovery PUT: %d %s", result.Code, result.Body.String())
|
||||
}
|
||||
after := multipartConditionRequest(t, router, http.MethodGet, url, "", readHeaders)
|
||||
t.Logf("after acknowledged recovery PUT: GET HTTP=%d matches_new=%t", after.Code, bytes.Equal(after.Body.Bytes(), recoveryBody))
|
||||
for i, pool := range z.serverPools {
|
||||
oi, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
t.Logf("pool=%d mtime=%s err=%v", i, oi.ModTime, err)
|
||||
}
|
||||
if after.Code != http.StatusOK || !bytes.Equal(after.Body.Bytes(), recoveryBody) {
|
||||
t.Fatalf("acknowledged recovery overwrite did not restore reads: HTTP=%d body=%s", after.Code, after.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Read failures activated by a completed disk mutation verify that a recovery
|
||||
// ACK is not issued from preflight information alone.
|
||||
type recoveryReadFaultDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
fail *atomic.Bool
|
||||
afterRename bool
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
|
||||
if bucket == d.bucket && object == d.object && d.fail.Load() {
|
||||
return RawFileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadXL(ctx, bucket, object, data)
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
|
||||
if bucket == d.bucket && object == d.object && d.fail.Load() {
|
||||
return FileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
res, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
if err == nil && d.afterRename && dv == d.bucket && dp == d.object {
|
||||
d.fail.Store(true)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
|
||||
for _, mode := range []string{"two-inconsistent", "suspended", "rebalancing", "offline", "read-fails-after-commit", "newer-readable-other-copy"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
const object = "refused-recovery"
|
||||
bucket, router := conditionalPutBucket(t, z, "unversioned")
|
||||
for _, pool := range z.serverPools {
|
||||
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
|
||||
defer restore()
|
||||
}
|
||||
base := time.Now().UTC().Add(-time.Hour)
|
||||
old := bytes.Repeat([]byte{'a'}, 8192)
|
||||
pending := bytes.Repeat([]byte{'b'}, 8192)
|
||||
replacement := bytes.Repeat([]byte{'c'}, 8192)
|
||||
inconsistent := func(pool int) {
|
||||
t.Helper()
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(old), ObjectOptions{MTime: base})
|
||||
disks := z.serverPools[pool].getHashedSet(object).getDisks()
|
||||
saved := make([][]byte, len(disks))
|
||||
for i, d := range disks {
|
||||
saved[i] = mustReadNullQuorumMeta(t, d, bucket, object)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(pending), ObjectOptions{MTime: base.Add(time.Minute)})
|
||||
for i := 0; i < len(disks)-1; i++ {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, saved[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
inconsistent(0)
|
||||
otherTime := base.Add(-time.Minute)
|
||||
if mode == "newer-readable-other-copy" {
|
||||
otherTime = base.Add(2 * time.Hour)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 1, string(old), ObjectOptions{MTime: otherTime})
|
||||
if mode == "two-inconsistent" {
|
||||
inconsistent(1)
|
||||
}
|
||||
// Select a healthy destination if the resolver incorrectly ignores pool0.
|
||||
postCommitFailure := mode == "read-fails-after-commit" || mode == "newer-readable-other-copy"
|
||||
if !postCommitFailure {
|
||||
restore := conditionalPutPool(t, z, object, 1)
|
||||
defer restore()
|
||||
}
|
||||
if mode == "suspended" {
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
}
|
||||
if mode == "rebalancing" {
|
||||
z.rebalMu.Lock()
|
||||
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}, {}}}
|
||||
z.rebalMu.Unlock()
|
||||
}
|
||||
var fail atomic.Bool
|
||||
if mode == "offline" || mode == "read-fails-after-commit" {
|
||||
fail.Store(mode == "offline")
|
||||
restore := conditionalPutSwapDisks(z.serverPools[0], object, func(d StorageAPI) StorageAPI {
|
||||
return recoveryReadFaultDisk{StorageAPI: d, bucket: bucket, object: object, fail: &fail, afterRename: mode == "read-fails-after-commit"}
|
||||
})
|
||||
defer restore()
|
||||
}
|
||||
snapshot := func() [][32]byte {
|
||||
var result [][32]byte
|
||||
for _, pool := range z.serverPools {
|
||||
for _, disk := range pool.getHashedSet(object).getDisks() {
|
||||
b, err := disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result = append(result, sha256.Sum256(b))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
before := snapshot()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getGetObjectURL("", bucket, object), int64(len(replacement)), bytes.NewReader(replacement), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := httptest.NewRecorder()
|
||||
router.ServeHTTP(result, req.WithContext(t.Context()))
|
||||
if result.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("unresolved recovery returned HTTP=%d: %s", result.Code, result.Body.String())
|
||||
}
|
||||
switch {
|
||||
case !postCommitFailure:
|
||||
after := snapshot()
|
||||
for i := range before {
|
||||
if before[i] != after[i] {
|
||||
t.Errorf("refused preflight modified shard %d", i)
|
||||
}
|
||||
}
|
||||
case mode == "read-fails-after-commit":
|
||||
fail.Store(false)
|
||||
got := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
||||
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), replacement) {
|
||||
t.Errorf("read validation failure must not undo committed recovery: HTTP=%d", got.Code)
|
||||
}
|
||||
default:
|
||||
// A readable but different latest copy cannot confirm recovery.
|
||||
// The committed replacement must still remain in its target pool.
|
||||
r, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || !bytes.Equal(body, replacement) {
|
||||
t.Fatalf("failed recovery validation must preserve committed data: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPutObjectGenerationFollowsCommitLock(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
for _, preserve := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "generated", true: "preserved"}[preserve], func(t *testing.T) {
|
||||
const object = "ordered-generation"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
lock := set.NewNSLock(bucket, object)
|
||||
lc, err := lock.GetLock(t.Context(), globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
lock.Unlock(lc)
|
||||
}
|
||||
}()
|
||||
|
||||
opts := ObjectOptions{}
|
||||
if preserve {
|
||||
opts.MTime = time.Unix(1234567890, 0)
|
||||
}
|
||||
reader := mustGetPutObjReader(t, strings.NewReader("payload"), 7, "", "")
|
||||
type result struct {
|
||||
info ObjectInfo
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
info, err := set.PutObject(t.Context(), bucket, object, reader, opts)
|
||||
done <- result{info, err}
|
||||
}()
|
||||
|
||||
// Wait for the real writer to queue behind the held namespace
|
||||
// lock. No elapsed delay is used to infer its progress.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
set.nsMutex.lockMapMutex.Lock()
|
||||
entry := set.nsMutex.lockMap[pathJoin(bucket, object)]
|
||||
waiting := entry != nil && entry.ref == 2
|
||||
set.nsMutex.lockMapMutex.Unlock()
|
||||
if waiting {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case got := <-done:
|
||||
t.Fatalf("PUT bypassed the held commit lock: %v", got.err)
|
||||
default:
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("PUT did not queue for the commit lock")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
beforeUnlock := UTCNow()
|
||||
lock.Unlock(lc)
|
||||
locked = false
|
||||
got := <-done
|
||||
if got.err != nil {
|
||||
t.Fatal(got.err)
|
||||
}
|
||||
if preserve {
|
||||
if !got.info.ModTime.Equal(opts.MTime) {
|
||||
t.Fatalf("explicit modification time changed: %s", got.info.ModTime)
|
||||
}
|
||||
} else if got.info.ModTime.Before(beforeUnlock) {
|
||||
t.Fatalf("generation %s predates commit lock release %s", got.info.ModTime, beforeUnlock)
|
||||
}
|
||||
for i, disk := range set.getDisks() {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, nullVersionID, ReadOptions{})
|
||||
if err != nil || !fi.ModTime.Equal(got.info.ModTime) {
|
||||
t.Errorf("disk %d stored a different generation: %s, %v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
)
|
||||
|
||||
type compatibilityFailedOverwrite struct {
|
||||
StorageAPI
|
||||
commit bool
|
||||
}
|
||||
|
||||
func (d compatibilityFailedOverwrite) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if d.commit {
|
||||
if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil {
|
||||
return RenameDataResp{}, err
|
||||
}
|
||||
}
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
|
||||
// Maintenance compatibility requirement: after a failed overwrite with one residue, keep the previously acknowledged quorum readable.
|
||||
// This tests a known history; it does not claim the resulting snapshot proves it.
|
||||
func TestMaintenancePartialFailedOverwriteAvailability(t *testing.T) {
|
||||
// Lock RPC tests configure a single-drive backend (EC:0). Pin this
|
||||
// fixture to 12+4 so its fifteen old shards really have read quorum.
|
||||
previousStorageClass := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previousStorageClass) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dirs, err := getRandomDisks(16)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
|
||||
if err != nil {
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
defer func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) }()
|
||||
bucket := "maintenance-compatibility"
|
||||
key := "partial-overwrite"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldBody := bytes.Repeat([]byte("old-A"), 65536)
|
||||
newBody := bytes.Repeat([]byte("new-B"), 65536)
|
||||
put := func(body []byte) error {
|
||||
_, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
|
||||
return err
|
||||
}
|
||||
if err := put(oldBody); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(key)
|
||||
getDisks := set.getDisks
|
||||
disks := getDisks()
|
||||
fi, err := disks[0].ReadVersion(ctx, "", bucket, key, "", ReadOptions{})
|
||||
if err != nil || fi.Erasure.DataBlocks != 12 || fi.Erasure.ParityBlocks != 4 {
|
||||
t.Fatalf("compatibility fixture must have 12+4 erasure coding: %+v, %v", fi.Erasure, err)
|
||||
}
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = compatibilityFailedOverwrite{d, i == 0}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return wrapped }
|
||||
err = put(newBody)
|
||||
set.getDisks = getDisks
|
||||
if err == nil {
|
||||
t.Fatal("partial overwrite must fail")
|
||||
}
|
||||
t.Logf("Overwrite completed with failure; one new residue, fifteen original shards: %v", err)
|
||||
r, err := z.GetObjectNInfo(ctx, bucket, key, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("MAINTENANCE COMPATIBILITY: previously readable acknowledged value now refused: %v", err)
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || !bytes.Equal(data, oldBody) {
|
||||
t.Fatalf("old acknowledged payload mismatch: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStrictListFallbackRejectsSplitGenerations(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
var generations [3][][]byte
|
||||
for generation := range generations {
|
||||
body := bytes.Repeat([]byte{byte('a' + generation)}, 8192)
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Duration(generation) * time.Minute)})
|
||||
generations[generation] = make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
generations[generation][i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
}
|
||||
// All sixteen drives are online, but the 7/5/4 split leaves no generation
|
||||
// with the eight matching snapshots required by the listing resolver.
|
||||
for i, disk := range disks {
|
||||
generation := 0
|
||||
if i >= 7 {
|
||||
generation = 1
|
||||
}
|
||||
if i >= 12 {
|
||||
generation = 2
|
||||
}
|
||||
if err := disk.WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, generations[generation][i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) {
|
||||
t.Fatalf("split metadata must remain a read-quorum failure: %v", err)
|
||||
}
|
||||
entries := make(metaCacheEntries, len(disks))
|
||||
for i, disk := range disks {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disk, bucket, object)}
|
||||
if _, err := entries[i].xlmeta(); err != nil {
|
||||
t.Fatalf("drive %d no longer has valid metadata: %v", i, err)
|
||||
}
|
||||
}
|
||||
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
|
||||
entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, nil, &resolver)})
|
||||
if entry != nil || !isErrReadQuorum(err) {
|
||||
t.Fatalf("split live metadata became absence: entry=%v err=%v", entry, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStrictSingleNullQuorumReadAndConsumers(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
_, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: modTime})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(oldBody, oldTime)
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
oldMeta[i], err = disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(newBody, oldTime.Add(time.Minute))
|
||||
newMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
newMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
// Model an incomplete overwrite using real inline shards: fifteen old
|
||||
// copies retain a read quorum; the final disk contains the newer minority.
|
||||
// The old quorum does not prove that the newer overwrite was uncommitted.
|
||||
for i := range 15 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checkRead := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
|
||||
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object))
|
||||
if get.Code != 200 || !bytes.Equal(get.Body.Bytes(), body) || head.Code != 200 || head.Header().Get("Content-Length") != "8192" {
|
||||
t.Fatalf("GET/HEAD lost readable quorum: GET=%d HEAD=%d body=%q", get.Code, head.Code, get.Body.String())
|
||||
}
|
||||
oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil || !oi.ModTime.Equal(modTime) {
|
||||
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
|
||||
}
|
||||
}
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||
rec := nullQuorumRequest(t, router, method, getGetObjectURL("", bucket, object))
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("%s exposed an unresolved old quorum: %d", method, rec.Code)
|
||||
}
|
||||
}
|
||||
if _, err := z.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{Remove: true}); !isErrReadQuorum(err) {
|
||||
t.Fatalf("heal accepted an unresolved old quorum: %v", err)
|
||||
}
|
||||
// Reading must not reinterpret the minority as absent or delete it.
|
||||
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
|
||||
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
|
||||
}
|
||||
// Now model the safe direction: the latest generation has the quorum.
|
||||
// Keep the migration, Walk and scanner assertions on that readable state.
|
||||
for i := range 15 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := disks[15].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[15]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newTime := oldTime.Add(time.Minute)
|
||||
checkRead(newBody, newTime)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
var names []string
|
||||
consume := func(entry metaCacheEntry) {
|
||||
versions, err := entry.fileInfoVersions(bucket)
|
||||
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(newTime) {
|
||||
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
|
||||
return
|
||||
}
|
||||
// Migration reopens the selected name/version on all drives;
|
||||
// it must not treat the selected listing metadata as disk agreement.
|
||||
reader, err := set.GetObjectNInfo(t.Context(), bucket, entry.name, nil, nil, ObjectOptions{VersionID: nullVersionID, NoLock: true, NoDecryption: true})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil || !bytes.Equal(body, newBody) {
|
||||
t.Errorf("migration could not read selected object data: %v", err)
|
||||
}
|
||||
names = append(names, entry.name)
|
||||
}
|
||||
var err error
|
||||
if kind == "rebalance" {
|
||||
err = set.listObjectsToRebalance(t.Context(), bucket, consume)
|
||||
} else {
|
||||
err = set.listObjectsToDecommission(t.Context(), decomBucketInfo{Name: bucket}, consume)
|
||||
}
|
||||
if err != nil || !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("migration listing dropped the quorum: %v, %v", names, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, latestOnly := range []bool{false, true} {
|
||||
results := make(chan itemOrErr[ObjectInfo], 16)
|
||||
if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{AskDisks: "strict", LatestOnly: latestOnly}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for result := range results {
|
||||
if result.Err != nil || !result.Item.ModTime.Equal(newTime) {
|
||||
t.Fatalf("Walk returned invalid metadata: %+v", result)
|
||||
}
|
||||
names = append(names, result.Item.Name)
|
||||
}
|
||||
if !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("Walk dropped the quorum: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the scanner's actual abandoned-child path. Make the scan drive
|
||||
// miss the object, while keeping the latest quorum and an older minority on
|
||||
// the remaining disks. Its queued heal must read all disks and repair both.
|
||||
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[14]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanNullQuorumAbandoned(t, z, bucket, object, newTime)
|
||||
for i, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
|
||||
if err != nil || !fi.ModTime.Equal(newTime) {
|
||||
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
checkRead(newBody, newTime)
|
||||
put(newBody, oldTime.Add(2*time.Minute))
|
||||
checkRead(newBody, oldTime.Add(2*time.Minute))
|
||||
}
|
||||
@@ -815,11 +815,22 @@ func scanNullQuorumAbandoned(t *testing.T, z *erasureServerPools, bucket, object
|
||||
disks := z.serverPools[0].sets[0].getDisks()
|
||||
seq := newBgHealSequence()
|
||||
defer seq.cancelCtx()
|
||||
previousState, previousRoutine := globalBackgroundHealState, globalBackgroundHealRoutine
|
||||
globalBackgroundHealState = &allHealState{healSeqMap: map[string]*healSequence{"test": seq}}
|
||||
// Disk monitors may still read the shared state between object-layer tests.
|
||||
// Install the test sequence under its lock without replacing the state.
|
||||
state := globalBackgroundHealState
|
||||
state.Lock()
|
||||
previousSequences := state.healSeqMap
|
||||
state.healSeqMap = map[string]*healSequence{"test": seq}
|
||||
state.Unlock()
|
||||
previousRoutine := globalBackgroundHealRoutine
|
||||
routine := &healRoutine{tasks: make(chan healTask)}
|
||||
globalBackgroundHealRoutine = routine
|
||||
defer func() { globalBackgroundHealState, globalBackgroundHealRoutine = previousState, previousRoutine }()
|
||||
defer func() {
|
||||
globalBackgroundHealRoutine = previousRoutine
|
||||
state.Lock()
|
||||
state.healSeqMap = previousSequences
|
||||
state.Unlock()
|
||||
}()
|
||||
workerDone := make(chan struct{})
|
||||
var healed []string
|
||||
go func() {
|
||||
|
||||
@@ -186,6 +186,7 @@ func TestGetAPIEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckLocalServerAddr(t *testing.T) {
|
||||
useTestEndpointDNS(t)
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
expectedErr error
|
||||
|
||||
@@ -399,12 +399,15 @@ func getCompleteMultipartMD5(parts []CompletePart) string {
|
||||
return s3MD5
|
||||
}
|
||||
|
||||
// Clean unwanted fields from metadata
|
||||
// Keep metadata written by pre-release builds out of public object metadata.
|
||||
const rollbackIDKey = ReservedMetadataPrefixLower + "rollback-id"
|
||||
|
||||
// Clean unwanted fields from metadata.
|
||||
func cleanMetadata(metadata map[string]string) map[string]string {
|
||||
// Remove STANDARD StorageClass
|
||||
metadata = removeStandardStorageClass(metadata)
|
||||
// Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging".
|
||||
return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified", VersionPurgeStatusKey)
|
||||
return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified", VersionPurgeStatusKey, rollbackIDKey)
|
||||
}
|
||||
|
||||
// Filter X-Amz-Storage-Class field only if it is set to STANDARD.
|
||||
|
||||
@@ -0,0 +1,652 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
"github.com/minio/minio/internal/config/drive"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
)
|
||||
|
||||
func mutationPools(t testing.TB) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
// These scenarios require a 4+4 set: five physical commits form a
|
||||
// readable generation, while four successful replies fail write quorum.
|
||||
// Other server tests can leave EC:0 configured before this fixture runs.
|
||||
previousStorageClass := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previousStorageClass) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
dirs, err := getRandomDisks(8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints := mustGetPoolEndpoints(0, dirs[:8]...)
|
||||
|
||||
obj, _, err := initObjectLayer(ctx, endpoints)
|
||||
if err != nil {
|
||||
cancel()
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
old := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(old) })
|
||||
bucket := "scope-probe"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func mutationPut(t testing.TB, z *erasureServerPools, bucket, obj string, pool int, body string, when time.Time) {
|
||||
t.Helper()
|
||||
_, err := z.serverPools[pool].PutObject(context.Background(), bucket, obj, mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""), ObjectOptions{MTime: when})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type mutationAmbiguousRename struct {
|
||||
StorageAPI
|
||||
index int
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d mutationAmbiguousRename) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
if d.index >= 5 {
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
r, e := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, o)
|
||||
d.entered <- struct{}{}
|
||||
<-d.release
|
||||
if d.index == 4 && e == nil {
|
||||
return r, context.DeadlineExceeded
|
||||
}
|
||||
return r, e
|
||||
}
|
||||
|
||||
// A physical fifth commit loses its reply. B successfully updates tags on all
|
||||
// five copies of A. A then undoes four known successes. B's tag update retains
|
||||
// the DataDir and ModTime, so the proposed ownership predicate cannot see it.
|
||||
func TestMutationRollbackPreservesAcknowledgedTags(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "tag-undo"
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, "initial", base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 5), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = mutationAmbiguousRename{d, i, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
aDone := make(chan error, 1)
|
||||
go func() {
|
||||
body := "pending-A"
|
||||
_, e := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
|
||||
aDone <- e
|
||||
}()
|
||||
for range 5 {
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish physical commits")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
const tags = "owner=acknowledged-B"
|
||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{NoLock: true}); err != nil {
|
||||
t.Fatalf("B tags failed: %v", err)
|
||||
}
|
||||
count := func() int {
|
||||
n := 0
|
||||
for _, d := range disks {
|
||||
fi, e := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if e == nil && fi.Metadata[xhttp.AmzObjectTagging] == tags {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
if n := count(); n != 5 {
|
||||
t.Fatalf("B did not update all five A copies: %d", n)
|
||||
}
|
||||
t.Log("B tag update acknowledged on five physical copies; fifth A reply will report timeout")
|
||||
unblock()
|
||||
select {
|
||||
case e := <-aDone:
|
||||
if e == nil {
|
||||
t.Fatal("A unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("A result=%v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
if n := count(); n != 5 {
|
||||
t.Errorf("acknowledged tags lost physical copies: got %d, want 5", n)
|
||||
}
|
||||
oi, e := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if oi.UserTags != tags {
|
||||
t.Fatalf("version-scoped rollback lost acknowledged tag update: tags=%q", oi.UserTags)
|
||||
}
|
||||
}
|
||||
|
||||
type delayedRenameResult struct {
|
||||
StorageAPI
|
||||
fail bool
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d delayedRenameResult) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if d.fail {
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
r, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
d.entered <- struct{}{}
|
||||
<-d.release // Delay the reply, after the physical mutation and local unlock.
|
||||
return r, err
|
||||
}
|
||||
|
||||
// One pool, eight real disk directories, no edited object metadata. A commits
|
||||
// on four disks (below write quorum five); its replies are delayed. Model a lost
|
||||
// namespace lease by letting B execute with NoLock, like the previous lease
|
||||
// fixture. B adds an explicit version and succeeds on seven disks. A then
|
||||
// fails quorum and rolls back. A must not erase the separately acknowledged B.
|
||||
func TestMutationRollbackPreservesConcurrentVersion(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "rollback-and-version"
|
||||
size, _ := strconv.Atoi(os.Getenv("SILO_TEST_MUTATION_BYTES"))
|
||||
payload := func(v string) string {
|
||||
if size > len(v) {
|
||||
return v + strings.Repeat(".", size-len(v))
|
||||
}
|
||||
return v
|
||||
}
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, payload("initial-null"), base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
aDisks := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
aDisks[i] = delayedRenameResult{d, i >= 4, entered, release}
|
||||
}
|
||||
swap(aDisks)
|
||||
aDone := make(chan error, 1)
|
||||
go func() {
|
||||
body := payload("pending-null-A")
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
||||
ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
|
||||
aDone <- err
|
||||
}()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-aDone:
|
||||
t.Fatalf("A returned early: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not reach the reply barrier")
|
||||
}
|
||||
}
|
||||
t.Log("A physically committed on four disks; replies held; other four RenameData calls failed")
|
||||
bDisks := append([]StorageAPI(nil), disks...)
|
||||
bDisks[7] = nil
|
||||
swap(bDisks)
|
||||
bVersion := mustGetUUID()
|
||||
body := payload("acknowledged-version-B")
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
||||
ObjectOptions{NoLock: true, Versioned: true, VersionID: bVersion, MTime: base.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("B failed: %v", err)
|
||||
}
|
||||
countB := func() int {
|
||||
n := 0
|
||||
for _, d := range disks {
|
||||
if _, e := d.ReadVersion(t.Context(), "", bucket, object, bVersion, ReadOptions{}); e == nil {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
if n := countB(); n != 7 {
|
||||
t.Fatalf("B was not committed on the expected seven drives: %d", n)
|
||||
}
|
||||
t.Log("B succeeded on seven drives and its version is present on all seven")
|
||||
unblock()
|
||||
select {
|
||||
case err := <-aDone:
|
||||
if err == nil {
|
||||
t.Fatal("A unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("A failed and rolled back: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
swap(disks)
|
||||
t.Logf("B metadata survives on %d of eight drives after A's rollback", countB())
|
||||
if n := countB(); n != 7 {
|
||||
t.Fatalf("unrelated rollback removed B's acknowledged copies: got %d, want 7", n)
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: bVersion})
|
||||
if err != nil {
|
||||
t.Fatalf("acknowledged B became unreadable after unrelated rollback: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil || !bytes.Equal(got, []byte(body)) {
|
||||
t.Fatalf("B payload changed: readErr=%v length=%d", err, len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationDeleteMarkerDoesNotReenter(t *testing.T) {
|
||||
disk, _, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := FileInfo{Volume: "bucket", Name: "absent", VersionID: mustGetUUID(), Deleted: true, ModTime: UTCNow()}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- disk.storage.DeleteVersion(t.Context(), "bucket", "absent", fi, true, DeleteOptions{}) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("DeleteVersion(forceDelMarker) blocked acquiring the same guard again through WriteMetadata")
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup can arrive after an RPC reply is lost while the real disk commit is
|
||||
// still running. Both single and bulk cleanup must retain its staged metadata.
|
||||
func TestMutationStagingCleanupWaitsForCommit(t *testing.T) {
|
||||
for _, bulk := range []bool{false, true} {
|
||||
t.Run(strconv.FormatBool(bulk), func(t *testing.T) {
|
||||
disk, root, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var lastSuccess int64
|
||||
gate := &stagedRenameContext{
|
||||
Context: context.Background(), meta: filepath.Join(root, minioMetaTmpBucket, "staged", xlStorageFormatFile),
|
||||
entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
}
|
||||
unblock := sync.OnceFunc(func() { close(gate.release) })
|
||||
defer unblock()
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
committed <- err
|
||||
}()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("commit did not reach staging barrier")
|
||||
}
|
||||
cleaned := make(chan error, 1)
|
||||
go func() {
|
||||
if bulk {
|
||||
cleaned <- disk.DeleteBulk(t.Context(), minioMetaTmpBucket, pathJoin("staged", xlStorageFormatFile))
|
||||
} else {
|
||||
cleaned <- disk.Delete(t.Context(), minioMetaTmpBucket, "staged", DeleteOptions{Recursive: true})
|
||||
}
|
||||
}()
|
||||
early := false
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
early = true
|
||||
t.Errorf("cleanup overtook admitted disk commit: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("commit did not finish")
|
||||
}
|
||||
if !early {
|
||||
select {
|
||||
case <-cleaned:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup did not finish")
|
||||
}
|
||||
}
|
||||
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
|
||||
if err != nil || got.DataDir != fi.DataDir || !got.ModTime.Equal(fi.ModTime) {
|
||||
t.Fatalf("cleanup lost committed metadata: got=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type reviewReadGate struct {
|
||||
StorageAPI
|
||||
ready chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d reviewReadGate) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
|
||||
r, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
func TestReviewFastReadAlreadyWaitsForSlowDisk(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "healthy-slow-read"
|
||||
mutationPut(t, z, bucket, object, 0, "healthy-inline", time.Now())
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
ready, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
g := reviewReadGate{StorageAPI: d, ready: ready}
|
||||
if i == len(disks)-1 {
|
||||
g.release = release
|
||||
}
|
||||
wrapped[i] = g
|
||||
}
|
||||
swap(wrapped)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true}, true)
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("reads did not reach barrier")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("read returned before the final disk: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Log("seven healthy replies are insufficient to return; final unbuffered mrfCheck send is still waiting")
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("read did not finish after slow disk release")
|
||||
}
|
||||
}
|
||||
|
||||
// Pause the real trash worker at its disk-space callback, after moving the
|
||||
// external shards but before DeleteVersion can replace xl.meta. An outer join
|
||||
// must also contain this nested worker, and later mutations must remain excluded.
|
||||
func TestMutationDeleteJoinsNestedWorker(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "nested-timeout"
|
||||
mutationPut(t, z, bucket, object, 0, strings.Repeat("x", 2<<20), time.Now())
|
||||
disk := z.serverPools[0].getHashedSet(object).getDisks()[0].(*xlStorageDiskIDCheck)
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
oldCache := disk.storage.diskInfoCache
|
||||
disk.storage.diskInfoCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) (DiskInfo, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
return DiskInfo{Used: 100, Free: 100, UsedInodes: 100, FreeInodes: 100}, nil
|
||||
})
|
||||
defer func() { disk.storage.diskInfoCache = oldCache }()
|
||||
oldTimeout := globalDriveConfig.GetMaxTimeout()
|
||||
globalDriveConfig.Update(drive.Config{MaxTimeout: 1200 * time.Millisecond})
|
||||
defer globalDriveConfig.Update(drive.Config{MaxTimeout: oldTimeout})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, e := xioutil.WithDeadline(context.Background(), globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, disk.storage.DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{UndoWrite: true})
|
||||
})
|
||||
done <- e
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("trash callback not reached")
|
||||
}
|
||||
early := false
|
||||
select {
|
||||
case e := <-done:
|
||||
early = true
|
||||
t.Errorf("outer join returned before nested disk worker completed: %v", e)
|
||||
case <-time.After(1500 * time.Millisecond):
|
||||
}
|
||||
updated := make(chan error, 1)
|
||||
go func() {
|
||||
updated <- disk.storage.UpdateMetadata(context.Background(), bucket, object, FileInfo{VersionID: fi.VersionID, Metadata: map[string]string{"after-timeout": "yes"}}, UpdateMetadataOpts{})
|
||||
}()
|
||||
writerEarly := false
|
||||
select {
|
||||
case e := <-updated:
|
||||
writerEarly = true
|
||||
t.Errorf("later metadata writer passed unfinished deletion: %v", e)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
if !early {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("delete did not finish")
|
||||
}
|
||||
}
|
||||
if !writerEarly {
|
||||
select {
|
||||
case <-updated:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("later writer did not finish")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Even an update that leaves all public fields unchanged is a later successful
|
||||
// operation. An old writer must not regain undo ownership through an ABA cycle.
|
||||
func TestMutationNoopUpdateRevokesUndo(t *testing.T) {
|
||||
for _, method := range []string{"update", "write", "aba"} {
|
||||
t.Run(method, func(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "no-op-update"
|
||||
mutationPut(t, z, bucket, object, 0, "acknowledged", time.Now())
|
||||
disks := z.serverPools[0].getHashedSet(object).getDisks()
|
||||
for _, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch method {
|
||||
case "update":
|
||||
err = disk.UpdateMetadata(t.Context(), bucket, object, fi, UpdateMetadataOpts{})
|
||||
case "write":
|
||||
err = disk.WriteMetadata(t.Context(), "", bucket, object, fi)
|
||||
case "aba":
|
||||
for _, etag := range []string{"temporary", fi.Metadata["etag"]} {
|
||||
err = disk.UpdateMetadata(t.Context(), bucket, object, FileInfo{Metadata: map[string]string{"etag": etag}}, UpdateMetadataOpts{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, object, fi, false, DeleteOptions{UndoWrite: true}); err != errFileVersionNotFound {
|
||||
t.Fatalf("old writer retained undo after %s: %v", method, err)
|
||||
}
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil || string(body) != "acknowledged" {
|
||||
t.Fatalf("later acknowledged state lost: %q, %v", body, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationCanceledSuccessfulCommitIsNotUndone(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cancel-after-commit"
|
||||
mutationPut(t, z, bucket, object, 0, "initial", time.Now().Add(-time.Hour))
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
wrapped[i] = delayedRenameResult{disk, false, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader("committed"), 9, "", ""), ObjectOptions{})
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("physical commit barrier not reached")
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
unblock()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("commit did not finish")
|
||||
}
|
||||
swap(disks)
|
||||
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil || string(body) != "committed" {
|
||||
t.Fatalf("cancellation rolled back an actual successful quorum: %q, %v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Partial no-op updates may revoke undo on only some disks. That internal
|
||||
// capability must not split the quorum of otherwise identical object contents.
|
||||
func TestMutationPartialNoopPreservesReadability(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "partial-noop"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
const body = "acknowledged-data"
|
||||
const tags = "owner=unchanged"
|
||||
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: tags}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
partial := append([]StorageAPI(nil), disks...)
|
||||
for i := 4; i < len(partial); i++ {
|
||||
partial[i] = nil
|
||||
}
|
||||
swap(partial)
|
||||
if _, err = set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err == nil {
|
||||
t.Fatal("four drives unexpectedly gave write quorum")
|
||||
}
|
||||
swap(disks)
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatalf("unchanged acknowledged object became unreadable after no-op partial update: %v", err)
|
||||
}
|
||||
got, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(got) != body || r.ObjInfo.UserTags != tags {
|
||||
t.Fatalf("body/tags changed: %q %v %+v", got, err, r.ObjInfo)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"maps"
|
||||
)
|
||||
|
||||
// Compare logical version state, excluding per-shard checksums/index, physical
|
||||
// DataDir placement and properties of other versions in the same xl.meta.
|
||||
// Different DataDir values remain readable for the existing rebalance recovery
|
||||
// contract; rollback separately requires an exact DataDir match.
|
||||
func sameFileInfoVersion(a, b FileInfo) bool {
|
||||
if a.VersionID != b.VersionID || a.Deleted != b.Deleted || a.XLV1 != b.XLV1 || a.Size != b.Size ||
|
||||
a.TransitionStatus != b.TransitionStatus || a.TransitionTier != b.TransitionTier ||
|
||||
a.TransitionedObjName != b.TransitionedObjName || a.TransitionVersionID != b.TransitionVersionID ||
|
||||
!sameVersionMetadata(a.Metadata, b.Metadata) || !bytes.Equal(a.Checksum, b.Checksum) || len(a.Parts) != len(b.Parts) {
|
||||
return false
|
||||
}
|
||||
if !a.Deleted && a.Size != 0 && !a.Erasure.Equal(b.Erasure) {
|
||||
return false
|
||||
}
|
||||
for i, p := range a.Parts {
|
||||
q := b.Parts[i]
|
||||
if p.Number != q.Number || p.Size != q.Size || p.ActualSize != q.ActualSize || p.ETag != q.ETag || !bytes.Equal(p.Index, q.Index) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// The undo capability controls rollback ownership, not the logical contents.
|
||||
// A partial no-op update can revoke it without changing a single public field.
|
||||
func sameVersionMetadata(a, b map[string]string) bool {
|
||||
if a[rollbackIDKey] == b[rollbackIDKey] {
|
||||
return maps.Equal(a, b)
|
||||
}
|
||||
for k, v := range a {
|
||||
if k != rollbackIDKey {
|
||||
if other, ok := b[k]; !ok || other != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
for k := range b {
|
||||
if k != rollbackIDKey {
|
||||
if _, ok := a[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
func lateCommitPools(t *testing.T) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
dirs, err := getRandomDisks(16)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
|
||||
if err != nil {
|
||||
cancel()
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
|
||||
bucket := "late-commit-regression"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func lateCommitSwitch(set *erasureObjects) ([]StorageAPI, func([]StorageAPI)) {
|
||||
original := set.getDisks()
|
||||
var active atomic.Value
|
||||
active.Store(original)
|
||||
set.getDisks = func() []StorageAPI { return active.Load().([]StorageAPI) }
|
||||
return original, func(disks []StorageAPI) { active.Store(disks) }
|
||||
}
|
||||
|
||||
type lateCommitDisk struct {
|
||||
StorageAPI
|
||||
index int
|
||||
earlyDone chan<- struct{}
|
||||
releaseRename <-chan struct{}
|
||||
renameReady chan<- struct{}
|
||||
releaseUndo <-chan struct{}
|
||||
}
|
||||
|
||||
func (d lateCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
defer func() { d.renameReady <- struct{}{} }()
|
||||
if d.index >= 1 {
|
||||
<-d.releaseRename
|
||||
}
|
||||
result, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if d.index < 1 {
|
||||
d.earlyDone <- struct{}{}
|
||||
<-d.releaseRename
|
||||
}
|
||||
// Model a lost successful response. Real disk commit has already happened.
|
||||
if d.index < 1 || d.index >= 12 {
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (d lateCommitDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
|
||||
if opts.UndoWrite {
|
||||
<-d.releaseUndo
|
||||
}
|
||||
return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, force, opts)
|
||||
}
|
||||
|
||||
// All metadata changes use production PutObject/RenameData/UndoWrite on real disks.
|
||||
// Fault injection only controls timing, namespace lease age, and transport results.
|
||||
func TestMutationReadDuringLateCommitAndUndo(t *testing.T) {
|
||||
z, bucket := lateCommitPools(t)
|
||||
object := "late-undo-16"
|
||||
bodyA, bodyB := strings.Repeat("writer-A", 262144), strings.Repeat("writer-B", 262144)
|
||||
label := func(s string) string {
|
||||
if s == bodyA {
|
||||
return "A (2 MiB, exact payload match)"
|
||||
}
|
||||
if s == bodyB {
|
||||
return "B (2 MiB, exact payload match)"
|
||||
}
|
||||
if s == "" {
|
||||
return "empty"
|
||||
}
|
||||
return "UNEXPECTED PAYLOAD"
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
put := func(ctx context.Context, body string) (ObjectInfo, error) {
|
||||
return set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
|
||||
}
|
||||
if _, err := put(t.Context(), "seed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lockers := make([]*localLocker, 4)
|
||||
clients := make([]dsync.NetLocker, 4)
|
||||
for i := range lockers {
|
||||
lockers[i] = newLocker()
|
||||
clients[i] = lockers[i]
|
||||
}
|
||||
newLock := func() *distLockInstance {
|
||||
return &distLockInstance{rwMutex: dsync.NewDRWMutex(&dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, "late-commit" }, Timeouts: dsync.DefaultTimeouts}, bucket+"/"+object), opsID: mustGetUUID()}
|
||||
}
|
||||
oldLock := newLock()
|
||||
oldCtx, err := oldLock.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer oldLock.Unlock(oldCtx)
|
||||
disks, swap := lateCommitSwitch(set)
|
||||
defer swap(disks)
|
||||
earlyDone, renameReady := make(chan struct{}, 16), make(chan struct{}, 16)
|
||||
releaseRename, releaseUndo := make(chan struct{}), make(chan struct{})
|
||||
var renameOnce, undoOnce sync.Once
|
||||
unblockRename := func() { renameOnce.Do(func() { close(releaseRename) }) }
|
||||
unblockUndo := func() { undoOnce.Do(func() { close(releaseUndo) }) }
|
||||
defer unblockUndo()
|
||||
defer unblockRename()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = lateCommitDisk{d, i, earlyDone, releaseRename, renameReady, releaseUndo}
|
||||
}
|
||||
swap(wrapped)
|
||||
aDone := make(chan error, 1)
|
||||
go func() { _, e := put(oldCtx.Context(), bodyA); aDone <- e }()
|
||||
for range 1 {
|
||||
select {
|
||||
case <-earlyDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A early commit barrier")
|
||||
}
|
||||
}
|
||||
// Expire actual dsync lease records, allowing another holder to take over.
|
||||
for _, l := range lockers {
|
||||
l.mutex.Lock()
|
||||
for key, entries := range l.lockMap {
|
||||
for i := range entries {
|
||||
entries[i].TimeLastRefresh = time.Now().Add(-lockValidityDuration - time.Second).UnixNano()
|
||||
}
|
||||
l.lockMap[key] = entries
|
||||
}
|
||||
l.mutex.Unlock()
|
||||
l.expireOldLocks(lockValidityDuration)
|
||||
}
|
||||
newHolder := newLock()
|
||||
newCtx, err := newHolder.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
swap(disks)
|
||||
b, err := put(newCtx.Context(), bodyB)
|
||||
newHolder.Unlock(newCtx)
|
||||
if err != nil {
|
||||
t.Fatalf("B must acknowledge success: %v", err)
|
||||
}
|
||||
counts := func() (older, newer, unknown int) {
|
||||
for _, d := range disks {
|
||||
fi, e := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if e != nil {
|
||||
unknown++
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case fi.ModTime.Equal(b.ModTime):
|
||||
newer++
|
||||
case fi.ModTime.Before(b.ModTime):
|
||||
older++
|
||||
default:
|
||||
t.Fatalf("unexpected generation %v", fi.ModTime)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
o, n, u := counts()
|
||||
if o != 0 || n != 16 || u != 0 {
|
||||
t.Fatalf("B ACK evidence: %d %d %d", o, n, u)
|
||||
}
|
||||
t.Logf("B successfully committed; observed B=16/16, all disks online; ModTime=%s", b.ModTime)
|
||||
unblockRename()
|
||||
for range len(disks) {
|
||||
select {
|
||||
case <-renameReady:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("rename completion barrier")
|
||||
}
|
||||
}
|
||||
o, n, u = counts()
|
||||
|
||||
fi, e := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
qr, qw := fi.Erasure.DataBlocks, fi.Erasure.DataBlocks
|
||||
if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
|
||||
qw++
|
||||
}
|
||||
if qr != 12 || qw != 12 {
|
||||
t.Fatalf("unexpected quorum %d/%d", qr, qw)
|
||||
}
|
||||
t.Logf("Before A undo: old A=%d, newer acknowledged B=%d, U=%d, Qr=%d, Qw=%d", o, n, u, qr, qw)
|
||||
get := func() (string, error) {
|
||||
rd, e := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if e != nil {
|
||||
return "", e
|
||||
}
|
||||
defer rd.Close()
|
||||
body, e := io.ReadAll(rd)
|
||||
return string(body), e
|
||||
}
|
||||
first, firstErr := get()
|
||||
t.Logf("R1 after B ACK and before A undo: body=%q error=%v", label(first), firstErr)
|
||||
if firstErr != nil && !isErrReadQuorum(firstErr) {
|
||||
t.Errorf("unexpected first read error: %v", firstErr)
|
||||
}
|
||||
unblockUndo()
|
||||
select {
|
||||
case e := <-aDone:
|
||||
if e == nil {
|
||||
t.Fatal("A unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("A failed after completing UndoWrite: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A undo did not finish")
|
||||
}
|
||||
second, secondErr := get()
|
||||
if secondErr != nil || second != bodyB {
|
||||
t.Fatalf("B must remain after A fails: %q %v", label(second), secondErr)
|
||||
}
|
||||
o, n, u = counts()
|
||||
t.Logf("R2 after A failure, with no new PUT: body=%q; old A=%d, B=%d, U=%d", label(second), o, n, u)
|
||||
if first == bodyA {
|
||||
t.Error("NON-LINEARIZABLE: R1 exposed A, then R2 returned B with no second B PUT; B completed before R1, and R1 completed before R2. Current fragment counts do not prove B was never committed.")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
type scopeLeaseLocker struct {
|
||||
*localLocker
|
||||
partition *atomic.Bool
|
||||
refresh chan<- struct{}
|
||||
}
|
||||
|
||||
func (l scopeLeaseLocker) Refresh(ctx context.Context, a dsync.LockArgs) (bool, error) {
|
||||
if l.partition.Load() {
|
||||
select {
|
||||
case l.refresh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return false, errors.New("injected lock-plane partition")
|
||||
}
|
||||
return l.localLocker.Refresh(ctx, a)
|
||||
}
|
||||
|
||||
func (l scopeLeaseLocker) ForceUnlock(ctx context.Context, a dsync.LockArgs) (bool, error) {
|
||||
if l.partition.Load() {
|
||||
return false, errors.New("partition")
|
||||
}
|
||||
return l.localLocker.ForceUnlock(ctx, a)
|
||||
}
|
||||
|
||||
type scopeStagedContext struct {
|
||||
context.Context
|
||||
meta string
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
once sync.Once
|
||||
tracker healthDiskCtxValue
|
||||
}
|
||||
|
||||
func (c *scopeStagedContext) Value(k any) any {
|
||||
if _, ok := k.(healthDiskCtxKey); ok {
|
||||
if _, e := os.Stat(c.meta); e == nil {
|
||||
c.once.Do(func() { c.entered <- struct{}{}; <-c.release })
|
||||
}
|
||||
return &c.tracker
|
||||
}
|
||||
return c.Context.Value(k)
|
||||
}
|
||||
|
||||
type scopeRPCDisk struct {
|
||||
StorageAPI
|
||||
client *storageRESTClient
|
||||
}
|
||||
|
||||
func (d scopeRPCDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
return d.client.RenameData(ctx, sv, sp, fi, dv, dp, o)
|
||||
}
|
||||
|
||||
// Controlled fault schedule: real dsync lockers lose their expired lease,
|
||||
// while a real disk RenameData is paused after reading and staging xl.meta.
|
||||
// Only the clock age of lease records and the disk completion schedule are injected.
|
||||
// No object data or metadata is rewritten by the test after the seed PUT.
|
||||
func TestMutationLeaseLossAndPartition(t *testing.T) {
|
||||
for _, partitioned := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("network-partition-%t", partitioned), func(t *testing.T) {
|
||||
size, _ := strconv.Atoi(os.Getenv("SILO_TEST_MUTATION_BYTES"))
|
||||
payload := func(s string) string {
|
||||
if size <= len(s) {
|
||||
return s
|
||||
}
|
||||
return s + strings.Repeat(".", size-len(s))
|
||||
}
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "lease-key"
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, payload("initial-0"), base)
|
||||
var partition atomic.Bool
|
||||
refreshed := make(chan struct{}, 16)
|
||||
locals := make([]*localLocker, 4)
|
||||
oldClients := make([]dsync.NetLocker, 4)
|
||||
newClients := make([]dsync.NetLocker, 4)
|
||||
for i := range locals {
|
||||
locals[i] = newLocker()
|
||||
oldClients[i] = scopeLeaseLocker{locals[i], &partition, refreshed}
|
||||
newClients[i] = locals[i]
|
||||
}
|
||||
mklock := func(clients []dsync.NetLocker) *distLockInstance {
|
||||
return &distLockInstance{rwMutex: dsync.NewDRWMutex(&dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, "scope-owner" }, Timeouts: dsync.DefaultTimeouts}, bucket+"/"+object), opsID: mustGetUUID()}
|
||||
}
|
||||
a := mklock(oldClients)
|
||||
requestCtx, cancelRequest := context.WithCancel(context.Background())
|
||||
defer cancelRequest()
|
||||
alc, err := a.GetLock(requestCtx, newDynamicTimeout(time.Second, time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Unlock(alc)
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered, release := make(chan struct{}, 8), make(chan struct{})
|
||||
physicalDone := make(chan error, 8)
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
route := fmt.Sprintf("/lease/%d", i)
|
||||
handler := func(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
var success int64
|
||||
gate := &scopeStagedContext{Context: context.Background(), meta: filepath.Join(disk.Endpoint().Path, p.SrcVolume, p.SrcPath, xlStorageFormatFile), entered: entered, release: release, tracker: healthDiskCtxValue{lastSuccess: &success}}
|
||||
resp, e := disk.RenameData(gate, p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath, p.Opts)
|
||||
physicalDone <- e
|
||||
return &resp, grid.NewRemoteErr(e)
|
||||
}
|
||||
if e := storageRenameDataRPC.Register(tg.Managers[1], handler, route); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
return handler(&p.RenameDataHandlerParams)
|
||||
}, route); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
wrapped[i] = scopeRPCDisk{disk, client}
|
||||
}
|
||||
swap(wrapped)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
body := []byte(payload("pending-1000"))
|
||||
_, e := z.serverPools[0].PutObject(alc.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
|
||||
done <- e
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case e := <-done:
|
||||
t.Fatalf("A returned before disk barrier: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("disk barrier timeout")
|
||||
}
|
||||
}
|
||||
t.Log("OBS A staged old metadata on all eight real disks; no final rename yet")
|
||||
partition.Store(partitioned)
|
||||
for _, l := range locals {
|
||||
l.mutex.Lock()
|
||||
for key, entries := range l.lockMap {
|
||||
for i := range entries {
|
||||
entries[i].TimeLastRefresh = time.Now().Add(-lockValidityDuration - time.Second).UnixNano()
|
||||
}
|
||||
l.lockMap[key] = entries
|
||||
}
|
||||
l.mutex.Unlock()
|
||||
l.expireOldLocks(lockValidityDuration)
|
||||
}
|
||||
if partitioned {
|
||||
select {
|
||||
case <-refreshed:
|
||||
case <-time.After(13 * time.Second):
|
||||
t.Fatal("refresh not attempted")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
t.Logf("OBS after failed refresh old context error=%v", alc.Context().Err())
|
||||
} else {
|
||||
select {
|
||||
case <-alc.Context().Done():
|
||||
case <-time.After(13 * time.Second):
|
||||
t.Fatal("confirmed lost lock did not cancel")
|
||||
}
|
||||
t.Logf("OBS confirmed lease loss canceled old context: %v", alc.Context().Err())
|
||||
}
|
||||
b := mklock(newClients)
|
||||
blc, e := b.GetLock(context.Background(), newDynamicTimeout(time.Second, time.Second))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
swap(disks)
|
||||
bdone := make(chan error, 1)
|
||||
go func() {
|
||||
body := []byte(payload("ack-9999"))
|
||||
_, e := z.serverPools[0].PutObject(blc.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(2 * time.Minute)})
|
||||
b.Unlock(blc)
|
||||
bdone <- e
|
||||
}()
|
||||
bearly := false
|
||||
select {
|
||||
case e := <-bdone:
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
bearly = true
|
||||
t.Log("OBS B acquired the same lock and ACKed while A remained physically staged")
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
t.Log("OBS B is waiting for the earlier disk transaction")
|
||||
}
|
||||
if partitioned {
|
||||
cancelRequest()
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case e := <-done:
|
||||
t.Logf("OBS A finished: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
if !bearly {
|
||||
select {
|
||||
case e := <-bdone:
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Log("OBS B ACKed after disk serialization")
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("B did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
for range disks {
|
||||
select {
|
||||
case e := <-physicalDone:
|
||||
t.Logf("OBS physical handler completed: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("physical handler did not finish")
|
||||
}
|
||||
}
|
||||
for _, fast := range []bool{false, true} {
|
||||
r, e := z.GetObjectNInfo(context.Background(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: fast})
|
||||
if e != nil {
|
||||
t.Errorf("OBS latest acknowledged payload unreadable fast=%t error=%v", fast, e)
|
||||
continue
|
||||
}
|
||||
data, e := io.ReadAll(r)
|
||||
r.Close()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
shown := string(data)
|
||||
if len(shown) > 32 {
|
||||
shown = shown[:32]
|
||||
}
|
||||
t.Logf("OBS final GET fast=%t bytes=%d prefix=%q", fast, len(data), shown)
|
||||
if string(data) != payload("ack-9999") && string(data) != payload("pending-1000") {
|
||||
t.Errorf("STRICT STALE VALUE: prefix %q is neither latest ACK nor pending write", shown)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMutationReadPreservesAcknowledgedTags(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "tag-read"
|
||||
mutationPut(t, z, bucket, object, 0, "payload", time.Now())
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
partial := append([]StorageAPI(nil), disks...)
|
||||
partial[0] = nil
|
||||
swap(partial)
|
||||
const tags = "owner=acknowledged-B"
|
||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
swap(disks)
|
||||
before, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := disks[1].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("old/new tags=%q/%q; equal tuple=%t", before.Metadata[xhttp.AmzObjectTagging], after.Metadata[xhttp.AmzObjectTagging], before.ModTime.Equal(after.ModTime) && before.DataDir == after.DataDir && before.Metadata["etag"] == after.Metadata["etag"])
|
||||
for _, fast := range []bool{false, true} {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if oi.UserTags != tags {
|
||||
t.Fatalf("acknowledged tags replaced by stale metadata (fast=%t version=%q): %q", fast, version, oi.UserTags)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationVersionHistoryRead(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "history-read"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
var versions []string
|
||||
for i := range 12 {
|
||||
id := mustGetUUID()
|
||||
body := "version-" + strconv.Itoa(i)
|
||||
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, VersionID: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
versions = append(versions, id)
|
||||
}
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
uniform := func() bool {
|
||||
raw := make([]RawFileInfo, len(disks))
|
||||
meta := make([]FileInfo, len(disks))
|
||||
errs := make([]error, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
raw[i], err = disk.ReadXL(t.Context(), bucket, object, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta[i], errs[i] = fileInfoFromRaw(raw[i], bucket, object, false, false)
|
||||
}
|
||||
return uniformLatestVersion(meta, errs)
|
||||
}
|
||||
if !uniform() {
|
||||
t.Fatal("healthy version history requires redundant full metadata decoding")
|
||||
}
|
||||
partial := append([]StorageAPI(nil), disks...)
|
||||
partial[0] = nil
|
||||
swap(partial)
|
||||
const tags = "owner=latest-acknowledged"
|
||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
swap(disks)
|
||||
if uniform() {
|
||||
t.Fatal("stale same-version metadata incorrectly accepted as uniform")
|
||||
}
|
||||
for _, fast := range []bool{false, true} {
|
||||
for _, version := range []string{"", versions[len(versions)-1]} {
|
||||
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
||||
if err != nil || oi.VersionID != versions[len(versions)-1] || oi.UserTags != tags {
|
||||
t.Fatalf("latest version/tags changed: version=%q tags=%q err=%v", oi.VersionID, oi.UserTags, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, id := range versions {
|
||||
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != "version-"+strconv.Itoa(i) {
|
||||
t.Fatalf("history version %d changed: %q err=%v", i, body, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationCrossDiskOrderHasStableRead(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cross-disk-order"
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, "initial", base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
earlyDone, release := make(chan struct{}, 4), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
aDisks := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
aDisks[i] = orderedRenameDisk{d, i < 4, earlyDone, release}
|
||||
}
|
||||
swap(aDisks)
|
||||
put := func(body string, mtime time.Time) error {
|
||||
_, e := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
||||
ObjectOptions{NoLock: true, MTime: mtime})
|
||||
return e
|
||||
}
|
||||
aDone := make(chan error, 1)
|
||||
go func() { aDone <- put("writer-A", base.Add(time.Minute)) }()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-earlyDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not reach first-half barrier")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
if err := put("writer-B", base.Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("B failed: %v", err)
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-aDone:
|
||||
if err != nil {
|
||||
t.Fatalf("A failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
t.Log("both PUTs succeeded; final per-disk order is A→B on four disks and B→A on four disks; no writer remains")
|
||||
swap(disks)
|
||||
for range 8 {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: true, VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatalf("quiescent successful writes became unreadable: %v", err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != "writer-B" {
|
||||
t.Fatalf("completed reads changed value: %q %v", body, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func uniformLatestVersion(meta []FileInfo, errs []error) bool {
|
||||
if len(meta) == 0 {
|
||||
return false
|
||||
}
|
||||
first := meta[0]
|
||||
for i, fi := range meta {
|
||||
if errs[i] != nil || !fi.IsValid() || fi.XLV1 || fi.Deleted || !fi.IsLatest || fi.NumVersions < 1 ||
|
||||
!fi.ModTime.Equal(first.ModTime) || !sameFileInfoVersion(first, fi) ||
|
||||
fi.NumVersions != first.NumVersions || !fi.SuccessorModTime.Equal(first.SuccessorModTime) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type orderedRenameDisk struct {
|
||||
StorageAPI
|
||||
early bool
|
||||
earlyDone chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d orderedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if !d.early {
|
||||
<-d.release
|
||||
}
|
||||
r, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
if d.early {
|
||||
d.earlyDone <- struct{}{}
|
||||
<-d.release
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
@@ -244,6 +244,26 @@ func TestPoolsObjectLockOverwrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
defer conditionalPutPool(t, z, key, 0)()
|
||||
var source string
|
||||
var copyInfo ObjectInfo
|
||||
if op == "copy" || op == "self-copy" {
|
||||
source = key
|
||||
if op == "copy" {
|
||||
source = "source-" + state
|
||||
putConsistencyObject(t, z, bucket, source, 0, "replacement", ObjectOptions{Versioned: true})
|
||||
}
|
||||
// Resolve the source before the pool becomes unreadable.
|
||||
// The destination must still reject a previously read source.
|
||||
gr, gerr := z.GetObjectNInfo(ctx, bucket, source, nil, nil, ObjectOptions{})
|
||||
if gerr != nil {
|
||||
t.Fatal(gerr)
|
||||
}
|
||||
defer gr.Close()
|
||||
copyInfo = gr.ObjInfo
|
||||
copyInfo.metadataOnly = op == "self-copy"
|
||||
copyInfo.PutObjReader = mustGetPutObjReader(t, gr, copyInfo.Size, "", "")
|
||||
copyInfo.Reader = copyInfo.PutObjReader.Reader
|
||||
}
|
||||
var restore func()
|
||||
if state == "unreadable" {
|
||||
restore = conditionalPutSwapDisks(z.serverPools[1], key, func(d StorageAPI) StorageAPI {
|
||||
@@ -257,22 +277,8 @@ func TestPoolsObjectLockOverwrite(t *testing.T) {
|
||||
case "put":
|
||||
_, writeErr = z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), opts)
|
||||
case "copy", "self-copy":
|
||||
source := key
|
||||
if op == "copy" {
|
||||
source = "source-" + state
|
||||
putConsistencyObject(t, z, bucket, source, 0, "replacement", ObjectOptions{Versioned: true})
|
||||
}
|
||||
gr, gerr := z.GetObjectNInfo(ctx, bucket, source, nil, nil, ObjectOptions{})
|
||||
if gerr != nil {
|
||||
t.Fatal(gerr)
|
||||
}
|
||||
defer gr.Close()
|
||||
info := gr.ObjInfo
|
||||
info.UserDefined = opts.UserDefined
|
||||
info.metadataOnly = op == "self-copy"
|
||||
info.PutObjReader = mustGetPutObjReader(t, gr, info.Size, "", "")
|
||||
info.Reader = info.PutObjReader.Reader
|
||||
_, writeErr = z.CopyObject(ctx, bucket, source, bucket, key, info, ObjectOptions{VersionID: nullVersionID}, opts)
|
||||
copyInfo.UserDefined = opts.UserDefined
|
||||
_, writeErr = z.CopyObject(ctx, bucket, source, bucket, key, copyInfo, ObjectOptions{VersionID: nullVersionID}, opts)
|
||||
case "multipart":
|
||||
id, parts := multipartConditionUpload(t, z, bucket, key, 0, opts)
|
||||
_, writeErr = z.CompleteMultipartUpload(ctx, bucket, key, id, parts, ObjectOptions{Versioned: true})
|
||||
|
||||
@@ -0,0 +1,522 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
// Pause at the real disk health callback after staging xl.meta. Unlike a sleep,
|
||||
// this fixes cancellation between staging and the final metadata rename.
|
||||
type stagedRenameContext struct {
|
||||
context.Context
|
||||
meta string
|
||||
entered, release chan struct{}
|
||||
once sync.Once
|
||||
tracker healthDiskCtxValue
|
||||
}
|
||||
|
||||
func (c *stagedRenameContext) Value(key any) any {
|
||||
if _, ok := key.(healthDiskCtxKey); ok {
|
||||
if _, err := os.Stat(c.meta); err == nil {
|
||||
c.once.Do(func() {
|
||||
close(c.entered)
|
||||
<-c.release
|
||||
})
|
||||
}
|
||||
return &c.tracker
|
||||
}
|
||||
return c.Context.Value(key)
|
||||
}
|
||||
|
||||
func TestRenameCancellationLocalWaitsForDisk(t *testing.T) {
|
||||
disk, root, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var lastSuccess int64
|
||||
gate := &stagedRenameContext{
|
||||
Context: ctx, meta: filepath.Join(root, minioMetaTmpBucket, "staged", xlStorageFormatFile),
|
||||
entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
done <- err
|
||||
}()
|
||||
<-gate.entered
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Errorf("RenameData returned %v while its disk worker was still running", err)
|
||||
default:
|
||||
}
|
||||
close(gate.release)
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("canceled staged write: %v", err)
|
||||
}
|
||||
default:
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "bucket", "object", xlStorageFormatFile)); !os.IsNotExist(err) {
|
||||
t.Fatalf("canceled staged write committed metadata: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRenameCancellationRPCWaitsForDisk(t *testing.T) {
|
||||
for _, operation := range []string{"regular", "inline", "write-metadata", "update-metadata", "undo"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
var committed atomic.Bool
|
||||
handle := func(*RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
close(entered)
|
||||
<-release
|
||||
committed.Store(true)
|
||||
return &RenameDataResp{OldDataDir: "previous-generation"}, nil
|
||||
}
|
||||
const route = "/cancel-commit"
|
||||
if err := storageRenameDataRPC.Register(tg.Managers[1], handle, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
return handle(&p.RenameDataHandlerParams)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := func(*MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
_, err := handle(nil)
|
||||
return grid.NewNoPayload(), err
|
||||
}
|
||||
if err := storageWriteMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageUpdateMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], func(*DeleteVersionHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
return meta(nil)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
fi := FileInfo{}
|
||||
if operation == "inline" {
|
||||
fi.Data = []byte("payload")
|
||||
}
|
||||
type result struct {
|
||||
resp RenameDataResp
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
var resp RenameDataResp
|
||||
var err error
|
||||
switch operation {
|
||||
case "write-metadata":
|
||||
err = client.WriteMetadata(ctx, "", "bucket", "object", fi)
|
||||
case "update-metadata":
|
||||
err = client.UpdateMetadata(ctx, "bucket", "object", fi, UpdateMetadataOpts{})
|
||||
case "undo":
|
||||
err = client.DeleteVersion(ctx, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true})
|
||||
default:
|
||||
resp, err = client.RenameData(ctx, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
}
|
||||
done <- result{resp, err}
|
||||
}()
|
||||
<-entered
|
||||
cancel()
|
||||
var early *result
|
||||
select {
|
||||
case r := <-done:
|
||||
early = &r
|
||||
t.Errorf("RPC returned %v before the remote disk completed", r.err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
if early == nil {
|
||||
select {
|
||||
case r := <-done:
|
||||
wantOld := ""
|
||||
if operation == "regular" || operation == "inline" {
|
||||
wantOld = "previous-generation"
|
||||
}
|
||||
if r.err != nil || r.resp.OldDataDir != wantOld || !committed.Load() {
|
||||
t.Errorf("lost completed result needed for rollback: %+v", r)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RPC did not finish after disk completion")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameCancellationAlreadyCanceledRPC(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
// A canceled call must not access the connection or send a new mutation.
|
||||
client := &storageRESTClient{}
|
||||
client.diskID.Store(new(string))
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
t.Errorf("canceled mutation reached the connection: %v", p)
|
||||
}
|
||||
}()
|
||||
_, err := client.RenameData(ctx, minioMetaTmpBucket, "staged", FileInfo{}, "bucket", "object", RenameOptions{})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("pre-canceled RenameData: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type gatedRenameDisk struct {
|
||||
StorageAPI
|
||||
client *storageRESTClient
|
||||
}
|
||||
|
||||
func (d gatedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
return d.client.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
}
|
||||
|
||||
func (d gatedRenameDisk) WriteMetadata(ctx context.Context, ov, volume, path string, fi FileInfo) error {
|
||||
return d.client.WriteMetadata(ctx, ov, volume, path, fi)
|
||||
}
|
||||
|
||||
func (d gatedRenameDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
|
||||
return d.client.DeleteVersion(ctx, volume, path, fi, force, opts)
|
||||
}
|
||||
|
||||
// Use real disk commits behind the real RPC client. The barrier fixes the
|
||||
// order: every commit has started, then cancellation, then remote completion.
|
||||
func TestRenameCancellationKeepsLockUntilRollback(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
size int
|
||||
versioned bool
|
||||
copy bool
|
||||
shared bool
|
||||
}{
|
||||
{32, false, false, false},
|
||||
{2 << 20, false, false, false},
|
||||
{32, true, false, false},
|
||||
{2 << 20, true, false, false},
|
||||
{32, false, true, false},
|
||||
{2 << 20, false, true, false},
|
||||
{32, false, false, true},
|
||||
{2 << 20, false, false, true},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("bytes-%d/other-versions-%t/copy-%t/shared-%t", tc.size, tc.versioned, tc.copy, tc.shared), func(t *testing.T) {
|
||||
size := tc.size
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "cancel-rollback"
|
||||
old := strings.Repeat("a", size)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
var sharedVersions [2]string
|
||||
for pool := range 2 {
|
||||
if tc.versioned {
|
||||
putConsistencyObject(t, z, bucket, object, pool, "retained-version", ObjectOptions{Versioned: true, MTime: base.Add(-time.Minute)})
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, pool, old, ObjectOptions{MTime: base})
|
||||
if tc.shared {
|
||||
info, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info.metadataOnly, info.versionOnly = true, true
|
||||
copied, err := z.serverPools[pool].getHashedSet(object).CopyObject(t.Context(), bucket, object, bucket, object, info,
|
||||
ObjectOptions{}, ObjectOptions{Versioned: true, MTime: base.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sharedVersions[pool] = copied.VersionID
|
||||
}
|
||||
}
|
||||
copyInfo, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
copyInfo.metadataOnly = true
|
||||
copyInfo.UserDefined["copy-test"] = "canceled"
|
||||
// Exercise the production rollback handler and wire options as
|
||||
// well as the client. A local DeleteVersion bypasses that boundary.
|
||||
globalLocalDrivesMu.Lock()
|
||||
previousSets, previousMap := globalLocalSetDrives, globalLocalDrivesMap
|
||||
globalLocalSetDrives = nil
|
||||
globalLocalDrivesMap = make(map[string]StorageAPI)
|
||||
for _, pool := range z.serverPools {
|
||||
for _, disk := range pool.getHashedSet(object).getDisks() {
|
||||
globalLocalDrivesMap[disk.Endpoint().String()] = disk
|
||||
}
|
||||
}
|
||||
globalLocalDrivesMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalLocalDrivesMu.Lock()
|
||||
globalLocalSetDrives, globalLocalDrivesMap = previousSets, previousMap
|
||||
globalLocalDrivesMu.Unlock()
|
||||
})
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered := make(chan struct{}, 32)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
var completed atomic.Int32
|
||||
var injectCommitFailure atomic.Bool
|
||||
injectCommitFailure.Store(true)
|
||||
for pi, pool := range z.serverPools {
|
||||
set := pool.getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
route := fmt.Sprintf("/cancel-rollback/%d/%d", pi, i)
|
||||
server := storageRESTServer{endpoint: disk.Endpoint()}
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], server.DeleteVersionHandler, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handle := func(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
entered <- struct{}{}
|
||||
<-release
|
||||
// Cancellation alone must not undo a successful quorum.
|
||||
// This rollback fixture also fails half the disks, so its
|
||||
// original payload/lock assertions require a real failed commit.
|
||||
defer completed.Add(1)
|
||||
if i >= len(disks)/2 && injectCommitFailure.Load() {
|
||||
return nil, grid.NewRemoteErr(errDiskNotFound)
|
||||
}
|
||||
resp, err := disk.RenameData(context.Background(), p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath, p.Opts)
|
||||
if err != nil {
|
||||
return nil, grid.NewRemoteErr(err)
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
if err := storageRenameDataRPC.Register(tg.Managers[1], handle, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
return handle(&p.RenameDataHandlerParams)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageWriteMetadataRPC.Register(tg.Managers[1], func(p *MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
if p.Volume == bucket && p.FilePath == object {
|
||||
entered <- struct{}{}
|
||||
<-release
|
||||
defer completed.Add(1)
|
||||
}
|
||||
err := disk.WriteMetadata(context.Background(), p.OrigVolume, p.Volume, p.FilePath, p.FI)
|
||||
return grid.NewNoPayload(), grid.NewRemoteErr(err)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
wrapped[i] = gatedRenameDisk{disk, client}
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
pending := bytes.Repeat([]byte("b"), size)
|
||||
reader := mustGetPutObjReader(t, bytes.NewReader(pending), int64(size), "", "")
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
if tc.copy {
|
||||
_, err = z.CopyObject(ctx, bucket, object, bucket, object, copyInfo, ObjectOptions{}, ObjectOptions{})
|
||||
} else {
|
||||
_, err = z.PutObject(ctx, bucket, object, reader, ObjectOptions{})
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
n := z.serverPools[0].getHashedSet(object).setDriveCount
|
||||
for range n {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("write ended before commit barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("write did not reach every disk")
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
var early bool
|
||||
select {
|
||||
case err := <-done:
|
||||
early = true
|
||||
t.Errorf("mutation returned before remote commits completed: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
lock := z.NewNSLock(bucket, object)
|
||||
lockCtx, cancelLock := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||
lc, lockErr := lock.GetLock(lockCtx, newDynamicTimeout(50*time.Millisecond, 50*time.Millisecond))
|
||||
cancelLock()
|
||||
if lockErr == nil {
|
||||
lock.Unlock(lc)
|
||||
t.Error("namespace lock released before remote commits and rollback")
|
||||
}
|
||||
unblock()
|
||||
if early {
|
||||
// Keep the old-code failure deterministic; don't race cleanup
|
||||
// against handlers intentionally still running at the barrier.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for completed.Load() != int32(n) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("remote handlers did not finish")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("canceled write unexpectedly acknowledged")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("canceled write did not finish rollback")
|
||||
}
|
||||
if completed.Load() != int32(n) {
|
||||
t.Fatal("returned without accounting for every remote commit")
|
||||
}
|
||||
injectCommitFailure.Store(false)
|
||||
for pi, pool := range z.serverPools {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
r, err := pool.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatalf("pool %d lost its acknowledged copy (version %q): %v", pi, version, err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != old {
|
||||
t.Errorf("pool %d rollback lost acknowledged payload (version %q): bytes=%d err=%v", pi, version, len(body), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
got, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(got)
|
||||
got.Close()
|
||||
if err != nil || string(body) != old {
|
||||
t.Fatalf("rollback lost acknowledged payload: bytes=%d err=%v", len(body), err)
|
||||
}
|
||||
newBody := bytes.Repeat([]byte("c"), size)
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(newBody), int64(size), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err = z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err = io.ReadAll(got)
|
||||
got.Close()
|
||||
if err != nil || !bytes.Equal(body, newBody) {
|
||||
t.Fatalf("late commit replaced subsequent successful PUT: bytes=%d err=%v", len(body), err)
|
||||
}
|
||||
if tc.shared {
|
||||
for pi, pool := range z.serverPools {
|
||||
r, err := pool.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: sharedVersions[pi]})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != old {
|
||||
t.Errorf("commit cleanup destroyed shared version on pool %d: bytes=%d err=%v", pi, len(body), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameCancellationStaleRollbackPreservesNewWrite(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "stale-rollback"
|
||||
putConsistencyObject(t, z, bucket, object, 0, "canceled-generation", ObjectOptions{})
|
||||
disks := z.serverPools[0].getHashedSet(object).getDisks()
|
||||
stale := make([]FileInfo, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
stale[i], err = disk.ReadVersion(t.Context(), "", bucket, object, nullVersionID, ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 0, "new-acknowledged-generation", ObjectOptions{})
|
||||
for i, disk := range disks {
|
||||
err := disk.DeleteVersion(t.Context(), bucket, object, stale[i], false, DeleteOptions{UndoWrite: true})
|
||||
if err != nil && !errors.Is(err, errFileVersionNotFound) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("stale rollback removed the new acknowledged write: %v", err)
|
||||
}
|
||||
defer got.Close()
|
||||
body, err := io.ReadAll(got)
|
||||
if err != nil || string(body) != "new-acknowledged-generation" {
|
||||
t.Fatalf("stale rollback changed the new acknowledged payload: %q, %v", body, err)
|
||||
}
|
||||
}
|
||||
@@ -102,14 +102,9 @@ func TestGuardedStorageCoversEveryPathMethod(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuardChecksUnreachableFields pins the guard on path fields that exist on
|
||||
// the wire but that no handler currently reads, so an end-to-end test cannot
|
||||
// observe them. DeleteVersionHandler hardcodes `opts := DeleteOptions{}` and
|
||||
// discards p.Opts, which means DeleteOptions.OldDataDir - a value that reaches
|
||||
// renameAll() with no containment of its own - is unreachable by accident
|
||||
// rather than by design. If someone later plumbs p.Opts through, the guard is
|
||||
// already there; this test is what stops it from being removed as "dead".
|
||||
func TestGuardChecksUnreachableFields(t *testing.T) {
|
||||
// OldDataDir can reach renameAll during rollback. Guard it independently of
|
||||
// which handlers currently forward deletion options.
|
||||
func TestGuardChecksDeleteOptionPaths(t *testing.T) {
|
||||
g := guardedStorage{poisonStorage{}}
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -415,8 +415,7 @@ func (s *storageRESTServer) DeleteVersionHandler(p *DeleteVersionHandlerParams)
|
||||
filePath := p.FilePath
|
||||
forceDelMarker := p.ForceDelMarker
|
||||
|
||||
opts := DeleteOptions{}
|
||||
err := s.getStorage().DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker, opts)
|
||||
err := s.getStorage().DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker, p.Opts)
|
||||
return np, grid.NewRemoteErr(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -231,12 +231,9 @@ func TestStorageRESTTraversalRejected(t *testing.T) {
|
||||
{"DeleteVersion-DataDir", func() error {
|
||||
return restClient.DeleteVersion(ctx, "foo", "obj", badFI(p), false, DeleteOptions{})
|
||||
}},
|
||||
// DeleteOptions.OldDataDir is guarded too, but cannot be exercised
|
||||
// from here: DeleteVersionHandler hardcodes `opts := DeleteOptions{}`
|
||||
// and never reads the wire value, so the field is unreachable today.
|
||||
// That is an accidental mitigation, not a control - see
|
||||
// TestGuardChecksUnreachableFields for the unit-level assertion that
|
||||
// the guard is ready if anyone ever plumbs it through.
|
||||
{"DeleteVersion-OldDataDir", func() error {
|
||||
return restClient.DeleteVersion(ctx, "foo", "obj", badFI(""), false, DeleteOptions{UndoWrite: true, OldDataDir: p})
|
||||
}},
|
||||
{"RenameData-srcPath", func() error {
|
||||
_, err := restClient.RenameData(ctx, "foo", p, badFI(""), "bar", "dst", RenameOptions{})
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
type cancelableRenameDisk struct {
|
||||
StorageAPI
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d cancelableRenameDisk) RenameData(ctx context.Context, _, _ string, _ FileInfo, _, _ string, _ RenameOptions) (RenameDataResp, error) {
|
||||
d.entered <- struct{}{}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return RenameDataResp{}, ctx.Err()
|
||||
case <-d.release:
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageTimeoutCoordinatorCancellation(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
defer close(release)
|
||||
disks := make([]StorageAPI, 4)
|
||||
metadata := make([]FileInfo, 4)
|
||||
for i := range disks {
|
||||
disks[i] = cancelableRenameDisk{entered: entered, release: release}
|
||||
metadata[i] = FileInfo{Deleted: true}
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := renameData(ctx, disks, minioMetaTmpBucket, "staged", metadata, "bucket", "object", 3)
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
<-entered
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("canceled commit unexpectedly succeeded")
|
||||
}
|
||||
default:
|
||||
t.Fatal("coordinator did not propagate cancellation to disk calls")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type rollbackOptionsDisk struct {
|
||||
StorageAPI
|
||||
mu sync.Mutex
|
||||
opts DeleteOptions
|
||||
}
|
||||
|
||||
func (d *rollbackOptionsDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, opts DeleteOptions) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.opts = opts
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestStorageRollbackRPCOptions(t *testing.T) {
|
||||
endpoint, err := NewEndpoint(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disk := &rollbackOptionsDisk{}
|
||||
globalLocalDrivesMu.Lock()
|
||||
oldDrives, oldSets := globalLocalDrivesMap, globalLocalSetDrives
|
||||
globalLocalDrivesMap = map[string]StorageAPI{endpoint.String(): disk}
|
||||
globalLocalSetDrives = nil
|
||||
globalLocalDrivesMu.Unlock()
|
||||
defer func() {
|
||||
globalLocalDrivesMu.Lock()
|
||||
globalLocalDrivesMap, globalLocalSetDrives = oldDrives, oldSets
|
||||
globalLocalDrivesMu.Unlock()
|
||||
}()
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
server := storageRESTServer{endpoint: endpoint}
|
||||
const route = "/rollback-options"
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], server.DeleteVersionHandler, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
want := DeleteOptions{UndoWrite: true, OldDataDir: mustGetUUID()}
|
||||
if err := client.DeleteVersion(t.Context(), "bucket", "object", FileInfo{}, false, want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disk.mu.Lock()
|
||||
defer disk.mu.Unlock()
|
||||
if disk.opts != want {
|
||||
t.Fatalf("remote handler lost rollback options: got %+v, want %+v", disk.opts, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoragePrereleaseMetadataHidden(t *testing.T) {
|
||||
disk, _, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Metadata = map[string]string{rollbackIDKey: mustGetUUID(), "user-key": "value"}
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
public := cleanMetadata(got.Metadata)
|
||||
if _, ok := public[rollbackIDKey]; ok || public["user-key"] != "value" {
|
||||
t.Fatalf("pre-release metadata exposed or user metadata lost: %+v", public)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
// TrackDiskHealth makes the first lookup. Pause the real disk worker at its
|
||||
// next health callback, without requiring a broken drive or a wall-clock sleep.
|
||||
type timeoutDiskContext struct {
|
||||
context.Context
|
||||
calls atomic.Int32
|
||||
entered, release chan struct{}
|
||||
tracker healthDiskCtxValue
|
||||
pauseDeadline bool
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (c *timeoutDiskContext) Deadline() (time.Time, bool) {
|
||||
if c.pauseDeadline {
|
||||
// Metadata operations create a nested deadline in their disk worker.
|
||||
c.once.Do(func() { close(c.entered); <-c.release })
|
||||
}
|
||||
return c.Context.Deadline()
|
||||
}
|
||||
|
||||
func (c *timeoutDiskContext) Value(key any) any {
|
||||
if _, ok := key.(healthDiskCtxKey); ok {
|
||||
if !c.pauseDeadline && c.calls.Add(1) == 2 {
|
||||
close(c.entered)
|
||||
<-c.release
|
||||
}
|
||||
return &c.tracker
|
||||
}
|
||||
return c.Context.Value(key)
|
||||
}
|
||||
|
||||
func TestStorageTimeoutLocal(t *testing.T) {
|
||||
for _, operation := range []string{"rename", "rename-cancel", "write-metadata", "update-metadata", "undo"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
disk, _, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Metadata = map[string]string{"test": "original"}
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var lastSuccess int64
|
||||
gate := &timeoutDiskContext{
|
||||
Context: ctx, entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
pauseDeadline: operation != "rename" && operation != "rename-cancel",
|
||||
}
|
||||
unblock := sync.OnceFunc(func() { close(gate.release) })
|
||||
defer unblock()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
switch operation {
|
||||
case "rename", "rename-cancel":
|
||||
_, err = disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
case "write-metadata":
|
||||
err = disk.WriteMetadata(gate, "", "bucket", "object", fi)
|
||||
case "update-metadata":
|
||||
err = disk.UpdateMetadata(gate, "bucket", "object", fi, UpdateMetadataOpts{})
|
||||
case "undo":
|
||||
err = disk.DeleteVersion(gate, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true})
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("operation did not reach the disk worker: %v", err)
|
||||
}
|
||||
want := context.DeadlineExceeded
|
||||
if operation == "rename-cancel" {
|
||||
cancel()
|
||||
want = context.Canceled
|
||||
} else {
|
||||
time.Sleep(globalDriveConfig.GetMaxTimeout() + time.Second)
|
||||
}
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("expected %v, got %v", want, err)
|
||||
}
|
||||
default:
|
||||
t.Fatal("operation still waits for the blocked worker after its deadline")
|
||||
}
|
||||
// A timed-out worker must not leave a new physical mutation lock
|
||||
// preventing subsequent work on this object.
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatalf("subsequent metadata write: %v", err)
|
||||
}
|
||||
unblock()
|
||||
// synctest waits for the released worker before disk teardown.
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageTimeoutRPC(t *testing.T) {
|
||||
for _, operation := range []string{"rename", "inline", "write-metadata", "update-metadata", "undo"} {
|
||||
for _, deadline := range []bool{false, true} {
|
||||
mode := "cancel"
|
||||
if deadline {
|
||||
mode = "deadline"
|
||||
}
|
||||
t.Run(operation+"/"+mode, func(t *testing.T) {
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered, release, finished := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
handle := func() {
|
||||
close(entered)
|
||||
<-release
|
||||
close(finished)
|
||||
}
|
||||
const route = "/storage-timeout"
|
||||
if err := storageRenameDataRPC.Register(tg.Managers[1], func(*RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
handle()
|
||||
return &RenameDataResp{}, nil
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
handle()
|
||||
return &RenameDataResp{}, nil
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := func(*MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
handle()
|
||||
return grid.NewNoPayload(), nil
|
||||
}
|
||||
if err := storageWriteMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageUpdateMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], func(p *DeleteVersionHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
if !p.Opts.UndoWrite || p.Opts.OldDataDir != "previous" {
|
||||
t.Error("rollback options were lost on the wire")
|
||||
}
|
||||
return meta(nil)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
if deadline {
|
||||
var stop context.CancelFunc
|
||||
ctx, stop = context.WithTimeout(ctx, 500*time.Millisecond)
|
||||
defer stop()
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
fi := FileInfo{}
|
||||
switch operation {
|
||||
case "write-metadata":
|
||||
err = client.WriteMetadata(ctx, "", "bucket", "object", fi)
|
||||
case "update-metadata":
|
||||
err = client.UpdateMetadata(ctx, "bucket", "object", fi, UpdateMetadataOpts{})
|
||||
case "undo":
|
||||
err = client.DeleteVersion(ctx, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true, OldDataDir: "previous"})
|
||||
default:
|
||||
if operation == "inline" {
|
||||
fi.Data = []byte("payload")
|
||||
}
|
||||
_, err = client.RenameData(ctx, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("remote worker did not start")
|
||||
}
|
||||
want := context.DeadlineExceeded
|
||||
if !deadline {
|
||||
cancel()
|
||||
want = context.Canceled
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("expected %v, got %v", want, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RPC still waits for the remote worker")
|
||||
}
|
||||
unblock()
|
||||
<-finished
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -503,6 +503,8 @@ func resetTestGlobals() {
|
||||
resetGlobalHealState()
|
||||
// Reset globalIAMSys to `nil`
|
||||
resetGlobalIAMSys()
|
||||
// Object-layer fixtures must not reuse the preceding IAM suite's etcd store.
|
||||
globalEtcdClient = nil
|
||||
}
|
||||
|
||||
// Configure the server for the test run.
|
||||
|
||||
@@ -4,9 +4,9 @@ go 1.27.3
|
||||
|
||||
// Console and MC retain their historical module paths for best-effort upstream
|
||||
// compatibility. Pin the maintained PGSTY implementations used by SILO.
|
||||
replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20261025140954-258717be077b
|
||||
replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20261026153924-fff1618d9abb
|
||||
|
||||
replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20261025104502-dcc78d506f94
|
||||
replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20261026120214-0ceb298cdcd0
|
||||
|
||||
// v22.7.0 does not compile on NetBSD because its unix implementation uses
|
||||
// CLOCK_MONOTONIC, which is unavailable there. Keep the last portable release
|
||||
@@ -68,7 +68,7 @@ require (
|
||||
github.com/minio/kms-go/kes v0.3.1
|
||||
github.com/minio/kms-go/kms v0.7.0
|
||||
github.com/minio/madmin-go/v3 v3.0.110
|
||||
github.com/minio/minio-go/v7 v7.4.2-0.20261020194015-ab93db7ba435
|
||||
github.com/minio/minio-go/v7 v7.4.2-0.20261025165926-320af050c445
|
||||
github.com/minio/mux v1.10.1
|
||||
github.com/minio/selfupdate v0.6.0
|
||||
github.com/minio/simdjson-go v0.4.5
|
||||
@@ -81,7 +81,7 @@ require (
|
||||
github.com/nats-io/stan.go v0.10.4
|
||||
github.com/ncw/directio v1.0.5
|
||||
github.com/nsqio/go-nsq v1.1.0
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.1
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.2
|
||||
github.com/philhofer/fwd v1.2.1
|
||||
github.com/pierrec/lz4/v4 v4.1.33
|
||||
github.com/pkg/errors v0.9.1
|
||||
|
||||
@@ -442,8 +442,8 @@ github.com/minio/madmin-go/v3 v3.0.110 h1:FIYekj7YPc430ffpXFWiUtyut3qBt/unIAcDzJ
|
||||
github.com/minio/madmin-go/v3 v3.0.110/go.mod h1:WOe2kYmYl1OIlY2DSRHVQ8j1v4OItARQ6jGyQqcCud8=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.4.2-0.20261020194015-ab93db7ba435 h1:BIn1BJi2zZ+z5BIMsr02USFB+Wmipc4SrIAA8nksNH0=
|
||||
github.com/minio/minio-go/v7 v7.4.2-0.20261020194015-ab93db7ba435/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
|
||||
github.com/minio/minio-go/v7 v7.4.2-0.20261025165926-320af050c445 h1:TIdz2CWnaQQDML9I03RY0pdms4zfqpkGHPmnJxfvCrE=
|
||||
github.com/minio/minio-go/v7 v7.4.2-0.20261025165926-320af050c445/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk=
|
||||
github.com/minio/mux v1.10.1 h1:grrK8SwRKbkNFE6qG7WAvFGH09bB46d5teOOtKfQ14s=
|
||||
github.com/minio/mux v1.10.1/go.mod h1:INYT4sMSTJy0QWUEA/E2DZNxJ5sAxIwbnyZjkzNFRfE=
|
||||
github.com/minio/pkg/v3 v3.11.0 h1:WjzWGP7lyuupJgUSRim1RDtNujHtESyMQF7qSR4r5Dg=
|
||||
@@ -506,12 +506,12 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pgsty/mc v0.0.0-20261025104502-dcc78d506f94 h1:M5n0rCPOTyo3RiQBeSCQW/klx08efFpiVYmPq6eoM3w=
|
||||
github.com/pgsty/mc v0.0.0-20261025104502-dcc78d506f94/go.mod h1:H3Uwlm1Qj3kPsbmojhNnv3aj5a+QdjdTKwv4w4gMSF8=
|
||||
github.com/pgsty/silo-console v0.0.0-20261025140954-258717be077b h1:/RQ+D5TRP8/GGZaY5cbwIixd1RFE0eQjEVLBpIl3+8k=
|
||||
github.com/pgsty/silo-console v0.0.0-20261025140954-258717be077b/go.mod h1:fMxNcVxan3mu9mc7HCRiczSyQoFrbZEGumEmpatxtds=
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.1 h1:T5rqY3HR1WGybcjnx/ghBvliEgo7yNCA7JAC3f1LpTk=
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.1/go.mod h1:GmTF6fIEkTrzgM+ysWWfDZIlVRD2DYAJkKjA5bjiWpQ=
|
||||
github.com/pgsty/mc v0.0.0-20261026120214-0ceb298cdcd0 h1:kxlOeGEFoBW/lgLQVBL9pcw3uq5KTMO6VmqdeDebTig=
|
||||
github.com/pgsty/mc v0.0.0-20261026120214-0ceb298cdcd0/go.mod h1:dUu9hAVOMyAC+/9XrMdPxISLwJp537jXuv3bNW5+q5E=
|
||||
github.com/pgsty/silo-console v0.0.0-20261026153924-fff1618d9abb h1:ruI2VVcMcfnpk2tHwcBSeaV4yrfiFQvSqPmA1At92mM=
|
||||
github.com/pgsty/silo-console v0.0.0-20261026153924-fff1618d9abb/go.mod h1:0CxxVsLvy+u91xMHwrLGI1gwc96sBKpB8JTpGWIOm7I=
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.2 h1:PXgFAgkymHXhq8m5Rbs4QofHENOy5M7hLbC0KKowwvs=
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.2/go.mod h1:uk5B2JR5/NwmlGwwzPFUybtvn8z5ikPcxNLtPwBvcZ0=
|
||||
github.com/philhofer/fwd v1.2.1 h1:UtpS73q8bFgy/yH3gCq8LSNvj5gv/FowNOiWZa7uKwQ=
|
||||
github.com/philhofer/fwd v1.2.1/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pierrec/lz4/v4 v4.1.33 h1:GjG1TJ1V4IzKP8L96muuuDNpTwd7D+l2ccXrjAbe014=
|
||||
|
||||
@@ -25,7 +25,7 @@ imagePullSecrets: []
|
||||
##
|
||||
mcImage:
|
||||
repository: pgsty/mc
|
||||
tag: RELEASE.2026-10-18T00-00-00Z
|
||||
tag: RELEASE.2026-10-26T00-00-00Z
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
## Silo mode, i.e. standalone or distributed.
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package grid
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSingleRequestDefaultTimeout(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
conn := &Connection{outQueue: make(chan []byte, 1)}
|
||||
client := newMuxClient(t.Context(), 1, conn)
|
||||
defer client.cancelFn(context.Canceled)
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := client.roundtrip(handlerTest, nil); done <- err }()
|
||||
wire := <-conn.outQueue
|
||||
var request message
|
||||
if _, err := request.UnmarshalMsg(wire); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
PutByteBuffer(wire)
|
||||
if request.DeadlineMS != uint32(defaultSingleRequestTimeout/time.Millisecond) {
|
||||
t.Fatalf("wire deadline: %d", request.DeadlineMS)
|
||||
}
|
||||
time.Sleep(defaultSingleRequestTimeout + time.Second)
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
default:
|
||||
t.Fatal("request outlived the default timeout")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSingleRequestCancellationAndDisconnect(t *testing.T) {
|
||||
for _, disconnect := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "explicit-cancel", true: "disconnect"}[disconnect], func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
conn := &Connection{outQueue: make(chan []byte, 1)}
|
||||
client := newMuxClient(ctx, 1, conn)
|
||||
defer client.cancelFn(context.Canceled)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.roundtrip(handlerTest, nil)
|
||||
done <- err
|
||||
}()
|
||||
PutByteBuffer(<-conn.outQueue)
|
||||
want := context.Canceled
|
||||
if disconnect {
|
||||
client.close()
|
||||
want = ErrDisconnected
|
||||
} else {
|
||||
cancel()
|
||||
}
|
||||
if got := <-done; !errors.Is(got, want) {
|
||||
t.Fatalf("request did not terminate correctly: want %v, got %v", want, got)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package ioutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWithDeadlineReturnsBeforeWorker(t *testing.T) {
|
||||
for _, cancelEarly := range []bool{false, true} {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
defer close(release)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
value, err := WithDeadline(ctx, time.Second, func(context.Context) (int, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
return 42, nil
|
||||
})
|
||||
if value != 0 {
|
||||
t.Errorf("timeout returned a late worker value: %d", value)
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
<-entered
|
||||
want := context.DeadlineExceeded
|
||||
if cancelEarly {
|
||||
cancel()
|
||||
want = context.Canceled
|
||||
} else {
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("expected %v, got %v", want, err)
|
||||
}
|
||||
default:
|
||||
t.Fatal("caller still waits for the blocked worker")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithDeadlineContainsPanic(t *testing.T) {
|
||||
value, err := WithDeadline(t.Context(), time.Second, func(context.Context) (int, error) {
|
||||
panic("deadline-test")
|
||||
})
|
||||
if value != 0 || err == nil || !strings.Contains(err.Error(), "deadline-test") {
|
||||
t.Fatalf("panic was not converted to an error: %d, %v", value, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user