astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-09-12 13:34:05 +03:00
Code Issues Packages Projects Releases Wiki Activity

test: record Linux restart and OIDC release acceptance

Browse Source 
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-09-11 20:02:09 +08:00
parent e59a3d938e
commit 51d41345f7
3 changed files with 991 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/investigations/issue-116/evidence-20260911.json
+534
View File
@@ -0,0 +1,534 @@
{
"date": "2026-09-11",
"runs": [
{
"runid": "silo-v1-0806-9c53f14c",
"version": "0806",
"image": "sha256:29a498b24669cae1fed11c1a2fb2b3d73c68829a0a9c0b14e71b386671d38fac",
"nodes": 4,
"drives": 4,
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
"drive_bytes": 268435456,
"status": "PASS",
"phases": [
{
"phase": "startup-admin",
"seconds_from_start": 4.157,
"first_online_by_coordinator": [
4.037,
4.078,
4.118,
3.737
]
},
{
"phase": "startup-canary",
"attempts": 1,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 0.239,
"seconds_from_start": 4.396,
"first_put_seconds_after_admin_by_coordinator": [
0.096,
0.116,
0.127,
0.138
],
"first_four_reads_seconds_after_admin_by_coordinator": [
0.162,
0.194,
0.217,
0.239
],
"transient_error_count": 0,
"first_transient_errors": []
},
{
"phase": "full-restart-admin",
"seconds_from_start": 2.084,
"first_online_by_coordinator": [
1.937,
1.976,
2.016,
1.643
]
},
{
"phase": "full-restart-canary",
"attempts": 45,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 14.449,
"seconds_from_start": 16.534,
"first_put_seconds_after_admin_by_coordinator": [
14.315,
13.573,
13.953,
0.05
],
"first_four_reads_seconds_after_admin_by_coordinator": [
14.373,
14.396,
14.426,
14.449
],
"transient_error_count": 129,
"first_transient_errors": [
{
"attempt": 1,
"operation": "put",
"node": 0,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
},
{
"attempt": 1,
"operation": "put",
"node": 1,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
},
{
"attempt": 1,
"operation": "put",
"node": 2,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
},
{
"attempt": 2,
"operation": "put",
"node": 0,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
}
]
},
{
"phase": "readback-15s",
"seconds_after_canary": 17.596,
"objects": 55,
"reads": 220,
"errors": []
},
{
"phase": "readback-30s",
"seconds_after_canary": 31.357,
"objects": 55,
"reads": 220,
"errors": []
},
{
"phase": "readback-60s",
"seconds_after_canary": 61.566,
"objects": 55,
"reads": 220,
"errors": []
},
{
"phase": "one-node-outage",
"existing_read": true,
"put": true,
"readers": 3
},
{
"phase": "rejoin-admin",
"seconds_from_start": 1.382,
"first_online_by_coordinator": [
1.27,
1.307,
1.343,
1.382
]
},
{
"phase": "rejoin-canary",
"attempts": 1,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 0.158,
"seconds_from_start": 1.541,
"first_put_seconds_after_admin_by_coordinator": [
0.013,
0.029,
0.039,
0.05
],
"first_four_reads_seconds_after_admin_by_coordinator": [
0.077,
0.102,
0.132,
0.158
],
"transient_error_count": 0,
"first_transient_errors": []
},
{
"phase": "final-readback",
"seconds_after_canary": 1.71,
"objects": 60,
"reads": 240,
"errors": []
}
],
"acknowledged_objects": 60,
"version_ids_recorded": true,
"sha256_recorded": true
},
{
"runid": "silo-v1-0903-a457573c",
"version": "0903",
"image": "sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3",
"nodes": 4,
"drives": 4,
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
"drive_bytes": 268435456,
"status": "PASS",
"phases": [
{
"phase": "startup-admin",
"seconds_from_start": 4.271,
"first_online_by_coordinator": [
4.158,
4.194,
3.827,
3.867
]
},
{
"phase": "startup-canary",
"attempts": 1,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 0.2,
"seconds_from_start": 4.471,
"first_put_seconds_after_admin_by_coordinator": [
0.054,
0.068,
0.081,
0.092
],
"first_four_reads_seconds_after_admin_by_coordinator": [
0.118,
0.146,
0.167,
0.2
],
"transient_error_count": 0,
"first_transient_errors": []
},
{
"phase": "full-restart-admin",
"seconds_from_start": 2.284,
"first_online_by_coordinator": [
2.143,
2.193,
0.849,
0.902
]
},
{
"phase": "full-restart-canary",
"attempts": 1,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 0.191,
"seconds_from_start": 2.476,
"first_put_seconds_after_admin_by_coordinator": [
0.016,
0.029,
0.044,
0.057
],
"first_four_reads_seconds_after_admin_by_coordinator": [
0.093,
0.124,
0.158,
0.192
],
"transient_error_count": 0,
"first_transient_errors": []
},
{
"phase": "readback-15s",
"seconds_after_canary": 15.235,
"objects": 8,
"reads": 32,
"errors": []
},
{
"phase": "readback-30s",
"seconds_after_canary": 30.356,
"objects": 8,
"reads": 32,
"errors": []
},
{
"phase": "readback-60s",
"seconds_after_canary": 60.192,
"objects": 8,
"reads": 32,
"errors": []
},
{
"phase": "one-node-outage",
"existing_read": true,
"put": true,
"readers": 3
},
{
"phase": "rejoin-admin",
"seconds_from_start": 0.903,
"first_online_by_coordinator": [
0.791,
0.829,
0.865,
0.903
]
},
{
"phase": "rejoin-canary",
"attempts": 1,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 0.141,
"seconds_from_start": 1.044,
"first_put_seconds_after_admin_by_coordinator": [
0.012,
0.022,
0.035,
0.045
],
"first_four_reads_seconds_after_admin_by_coordinator": [
0.069,
0.096,
0.12,
0.141
],
"transient_error_count": 0,
"first_transient_errors": []
},
{
"phase": "final-readback",
"seconds_after_canary": 0.301,
"objects": 13,
"reads": 52,
"errors": []
}
],
"acknowledged_objects": 13,
"version_ids_recorded": true,
"sha256_recorded": true
},
{
"runid": "silo-v1-current-6a14eb18",
"version": "current",
"image": "pgsty/d12a:build",
"image_id": "sha256:307af7711e2e04ab75759cb42a1eef45c43c4404894c0e30dd19f742b107b922",
"platform": "linux/arm64",
"nodes": 4,
"drives": 4,
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
"drive_bytes": 268435456,
"status": "PASS",
"binary_sha256": "1e4cd7b78ecfa1b0cf28f1961d48a220a712c6be1fd3a01b60ee99aec62f04a4",
"source_revision": "b32f2d9dd01a383a9991d34ffe248063463a031d+pool-consistency",
"phases": [
{
"phase": "startup-admin",
"seconds_from_start": 7.374,
"first_online_by_coordinator": [
7.207,
6.677,
6.74,
6.8
]
},
{
"phase": "startup-canary",
"attempts": 1,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 0.479,
"seconds_from_start": 7.854,
"first_put_seconds_after_admin_by_coordinator": [
0.266,
0.279,
0.292,
0.305
],
"first_four_reads_seconds_after_admin_by_coordinator": [
0.336,
0.378,
0.431,
0.479
],
"transient_error_count": 0,
"first_transient_errors": []
},
{
"phase": "full-restart-admin",
"seconds_from_start": 2.461,
"first_online_by_coordinator": [
2.264,
1.414,
2.396,
2.461
]
},
{
"phase": "full-restart-canary",
"attempts": 36,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 14.489,
"seconds_from_start": 16.951,
"first_put_seconds_after_admin_by_coordinator": [
0.013,
0.025,
14.301,
0.046
],
"first_four_reads_seconds_after_admin_by_coordinator": [
14.358,
14.394,
14.435,
14.489
],
"transient_error_count": 104,
"first_transient_errors": [
{
"attempt": 1,
"operation": "put",
"node": 2,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
},
{
"attempt": 1,
"operation": "get",
"node": 2,
"key": "full-restart-attempt-1-node-0",
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
},
{
"attempt": 1,
"operation": "get",
"node": 2,
"key": "full-restart-attempt-1-node-3",
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
},
{
"attempt": 2,
"operation": "put",
"node": 2,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
}
]
},
{
"phase": "readback-15s",
"started_seconds_after_canary": 15.0,
"seconds_after_canary": 22.006,
"objects": 113,
"reads": 452,
"errors": []
},
{
"phase": "readback-30s",
"started_seconds_after_canary": 30.003,
"seconds_after_canary": 36.571,
"objects": 113,
"reads": 452,
"errors": []
},
{
"phase": "readback-60s",
"started_seconds_after_canary": 60.0,
"seconds_after_canary": 65.743,
"objects": 113,
"reads": 452,
"errors": []
},
{
"phase": "one-node-outage",
"existing_read": true,
"put": true,
"readers": 3
},
{
"phase": "rejoin-admin",
"seconds_from_start": 2.49,
"first_online_by_coordinator": [
1.165,
2.417,
1.646,
2.087
]
},
{
"phase": "rejoin-canary",
"attempts": 37,
"puts": 4,
"gets": 16,
"hard_deadline_seconds": 60,
"gate_seconds": 13.977,
"seconds_from_start": 16.468,
"first_put_seconds_after_admin_by_coordinator": [
0.013,
0.024,
0.035,
13.889
],
"first_four_reads_seconds_after_admin_by_coordinator": [
13.909,
13.927,
13.953,
13.977
],
"transient_error_count": 36,
"first_transient_errors": [
{
"attempt": 1,
"operation": "put",
"node": 3,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
},
{
"attempt": 2,
"operation": "put",
"node": 3,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
},
{
"attempt": 3,
"operation": "put",
"node": 3,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
},
{
"attempt": 4,
"operation": "put",
"node": 3,
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
}
]
},
{
"phase": "final-readback",
"started_seconds_after_canary": 0.0,
"seconds_after_canary": 6.507,
"objects": 226,
"reads": 904,
"errors": []
}
],
"acknowledged_objects": 226,
"version_ids_recorded": true,
"sha256_recorded": true
}
]
}
docs/investigations/issue-116/run-linux.py
+327
View File
@@ -0,0 +1,327 @@
#!/usr/bin/env python3
"""Bounded four-container restart/readback acceptance for pgsty/silo#116."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import secrets
import shutil
import signal
import socket
import subprocess
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import boto3
from botocore.config import Config
ROOT = None
MCLI = None
CURRENT_BINARY = None
SOURCE_REVISION = None
IMAGES = {
'0806': 'pgsty/silo:RELEASE.2026-08-06T00-00-00Z',
'0903': 'pgsty/silo:RELEASE.2026-09-03T13-18-01Z',
'current': 'pgsty/d12a:build',
}
def docker(*args, timeout=40, check=True):
return subprocess.run(['docker', *args], check=check, capture_output=True, text=True, timeout=timeout)
class Deadline(BaseException):
pass
def bounded(seconds, operation):
def expired(*_):
raise Deadline(f'hard deadline of {seconds}s exceeded')
old = signal.signal(signal.SIGALRM, expired)
signal.setitimer(signal.ITIMER_REAL, seconds)
try:
return operation(time.monotonic() + seconds)
finally:
signal.setitimer(signal.ITIMER_REAL, 0)
signal.signal(signal.SIGALRM, old)
def remaining(deadline):
value = deadline - time.monotonic()
if value <= 0:
raise Deadline('absolute deadline exceeded')
return value
def run(version):
image_info = json.loads(docker('image', 'inspect', IMAGES[version]).stdout)[0]
runid = f'silo-v1-{version}-{uuid.uuid4().hex[:8]}'
out = ROOT / runid
out.mkdir(mode=0o700)
user, password = 'local116', secrets.token_urlsafe(24)
envfile = out / 'credentials.env'
envfile.write_text(f'MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\nMINIO_CI_CD=1\nMINIO_BROWSER=off\nGOMAXPROCS=2\n')
envfile.chmod(0o600)
env = {k: v for k, v in os.environ.items() if not k.startswith(('MINIO_', 'SILO_', 'MC_'))
and k.lower() not in {'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy'}}
nodes = [f'{runid}-n{i}' for i in range(4)]
sockets = [socket.socket() for _ in nodes]
for sock in sockets:
sock.bind(('127.0.0.1', 0))
ports = [sock.getsockname()[1] for sock in sockets]
for sock in sockets:
sock.close()
volumes = [n + '-data' for n in nodes]
endpoints, ledger = [], []
result = {'runid': runid, 'version': version, 'image': IMAGES[version],
'image_id': image_info['Id'], 'platform': image_info['Os']+'/'+image_info['Architecture'],
'nodes': 4,
'drives': 4, 'filesystem': 'Linux tmpfs named volumes, held mounted across server restarts',
'drive_bytes': 268435456, 'phases': [], 'status': 'RUNNING'}
if version == 'current':
result['binary_sha256'] = hashlib.sha256(CURRENT_BINARY.read_bytes()).hexdigest()
result['source_revision'] = SOURCE_REVISION
bucket = 'canary-' + uuid.uuid4().hex[:10]
def save(event=None):
if event is not None:
result['phases'].append(event)
compact = {k: (len(v) if k in ('transient_errors', 'errors') else v) for k, v in event.items()}
print(json.dumps({'version': version, **compact}), flush=True)
(out / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
(out / 'acknowledged.json').write_text(json.dumps(ledger, indent=2) + '\n')
def parallel(fn, values):
with ThreadPoolExecutor(max_workers=4) as pool:
return list(pool.map(fn, values))
def client(i, deadline):
timeout = remaining(deadline)
return boto3.client('s3', endpoint_url=endpoints[i], aws_access_key_id=user,
aws_secret_access_key=password, region_name='us-east-1',
config=Config(proxies={}, signature_version='s3v4', s3={'addressing_style': 'path'},
retries={'total_max_attempts': 1}, connect_timeout=timeout,
read_timeout=timeout, request_checksum_calculation='when_required',
response_checksum_validation='when_required'))
def admin_gate(origin, phase):
def check(deadline):
first = [None] * 4
while True:
states = []
for i in range(4):
try:
p = subprocess.run([str(MCLI), '--config-dir', str(out / 'mcli'), '--json',
'admin', 'info', f'n{i}'], env=env, capture_output=True,
text=True, timeout=min(5, remaining(deadline)))
info = json.loads(p.stdout)['info']
servers = info['servers']
ok = len(servers) == 4 and all(s.get('state') == 'online' and s.get('drives')
and all(d.get('state') == 'ok' for d in s['drives'])
and all(v == 'online' for v in s.get('network', {}).values()) for s in servers)
if ok:
(out / f'{phase}-admin-{i}.json').write_text(json.dumps(info, indent=2) + '\n')
if first[i] is None:
first[i] = round(time.monotonic() - origin, 3)
states.append(ok)
except (Exception,):
states.append(False)
if all(states):
event = {'phase': phase + '-admin', 'seconds_from_start': round(time.monotonic()-origin, 3),
'first_online_by_coordinator': first}
save(event)
return time.monotonic()
time.sleep(min(.25, remaining(deadline)))
return bounded(90, check)
def canary(phase, origin, admin_time, setup=False):
def check(deadline):
started = time.monotonic()
first_put, first_reads = [None] * 4, [None] * 4
errors, attempt, setup_done = [], 0, not setup
result['active_canary'] = {'phase': phase, 'errors': errors}
while True:
attempt += 1
remaining(deadline)
if not setup_done:
try:
try:
client(0, deadline).create_bucket(Bucket=bucket)
except Exception as e:
if 'BucketAlreadyOwnedByYou' not in str(e):
raise
client(0, deadline).put_bucket_versioning(Bucket=bucket, VersioningConfiguration={'Status': 'Enabled'})
setup_done = True
except Exception as e:
errors.append({'attempt': attempt, 'operation': 'setup', 'error': str(e)[:250]})
time.sleep(min(.25, remaining(deadline)))
continue
acked = []
for i in range(4):
key = f'{phase}-attempt-{attempt}-node-{i}'
payload = (key + '\n').encode() * 16384
try:
vid = client(i, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
entry = {'phase': phase, 'key': key, 'version': vid, 'bytes': len(payload),
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': i,
'ack_seconds_from_start': round(time.monotonic()-origin, 3)}
ledger.append(entry)
save() # Persist every acknowledged write, including failed rounds.
acked.append(entry)
if first_put[i] is None:
first_put[i] = round(time.monotonic()-admin_time, 3)
except Exception as e:
errors.append({'attempt': attempt, 'operation': 'put', 'node': i, 'error': str(e)[:250]})
reads = 0
for i in range(4):
own_reads = 0
for entry in acked:
try:
verify(i, entry, deadline)
reads += 1
own_reads += 1
except Exception as e:
errors.append({'attempt': attempt, 'operation': 'get', 'node': i,
'key': entry['key'], 'error': str(e)[:250]})
if own_reads == 4 and first_reads[i] is None:
first_reads[i] = round(time.monotonic()-admin_time, 3)
remaining(deadline)
if len(acked) == 4 and reads == 16:
result.pop('active_canary', None)
save({'phase': phase + '-canary', 'attempts': attempt, 'puts': 4, 'gets': 16,
'hard_deadline_seconds': 60, 'gate_seconds': round(time.monotonic()-started, 3),
'seconds_from_start': round(time.monotonic()-origin, 3),
'first_put_seconds_after_admin_by_coordinator': first_put,
'first_four_reads_seconds_after_admin_by_coordinator': first_reads, 'transient_errors': errors})
return time.monotonic()
(out / f'{phase}-canary-errors.json').write_text(json.dumps(errors, indent=2) + '\n')
if attempt == 1:
print(json.dumps({'version': version, 'phase': phase, 'first_attempt_errors': errors}), flush=True)
time.sleep(min(.25, remaining(deadline)))
return bounded(60, check)
def verify(i, entry, deadline):
got = client(i, deadline).get_object(Bucket=bucket, Key=entry['key'], VersionId=entry['version'])
try:
data = got['Body'].read()
finally:
got['Body'].close()
assert len(data) == entry['bytes'] and hashlib.sha256(data).hexdigest() == entry['sha256'], entry['key']
assert got.get('VersionId') == entry['version'], entry['key']
remaining(deadline)
def readback(label, origin):
def check(deadline):
started = time.monotonic()
errors = []
for entry in ledger:
for i in range(4):
try:
verify(i, entry, deadline)
except Exception as e:
errors.append({'key': entry['key'], 'node': i, 'error': str(e)[:250]})
save({'phase': label, 'started_seconds_after_canary': round(started-origin, 3),
'seconds_after_canary': round(time.monotonic()-origin, 3),
'objects': len(ledger), 'reads': len(ledger)*4, 'errors': errors})
return errors
return bounded(60, check)
try:
docker('network', 'create', runid)
for volume in volumes:
docker('volume', 'create', '--driver', 'local', '--opt', 'type=tmpfs',
'--opt', 'device=tmpfs', '--opt', 'o=size=256m', volume)
mounts = [arg for i, v in enumerate(volumes) for arg in ('--mount', f'type=volume,source={v},target=/keep/{i}')]
docker('run', '-d', '--pull=never', '--network', 'none', '--name', runid + '-keeper',
*mounts, '--entrypoint', 'sleep', 'alpine:3.23', '1800')
urls = [f'http://{n}:9000/data' for n in nodes]
def create(i):
extra = []
if version == 'current':
extra = ['--mount', f'type=bind,source={CURRENT_BINARY},target=/lab/silo,readonly',
'--entrypoint', '/lab/silo']
docker('create', '--pull=never', '--name', nodes[i], '--network', runid,
'--hostname', nodes[i], '--cpus', '2', '--memory', '3g', '--env-file', str(envfile),
'--mount', f'type=volume,source={volumes[i]},target=/data',
'-p', f'127.0.0.1:{ports[i]}:9000', *extra, image_info['Id'], 'server', '--address', ':9000',
'--console-address', ':9001', *urls)
parallel(create, range(4))
start = time.monotonic()
parallel(lambda n: docker('start', n), nodes)
for i, n in enumerate(nodes):
endpoint = 'http://' + docker('port', n, '9000/tcp').stdout.strip()
endpoints.append(endpoint)
env[f'MC_HOST_n{i}'] = endpoint.replace('http://', f'http://{user}:{password}@')
admin = admin_gate(start, 'startup')
canary('startup', start, admin, setup=True)
parallel(lambda n: docker('stop', '-t', '10', n), nodes)
start = time.monotonic()
parallel(lambda n: docker('start', n), nodes)
admin = admin_gate(start, 'full-restart')
gate = canary('full-restart', start, admin)
read_errors = []
for after in (15, 30, 60):
time.sleep(max(0, gate + after - time.monotonic()))
read_errors.extend(readback(f'readback-{after}s', gate))
assert not read_errors, 'acknowledged object readback failure; see result.json'
docker('stop', '-t', '10', nodes[3])
def outage(deadline):
verify(0, ledger[0], deadline)
payload = b'acknowledged with one Linux node offline' * 16384
key = 'one-node-outage'
vid = client(0, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
entry = {'phase': 'outage', 'key': key, 'version': vid, 'bytes': len(payload),
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': 0}
ledger.append(entry)
save()
for i in range(3):
verify(i, entry, deadline)
save({'phase': 'one-node-outage', 'existing_read': True, 'put': True, 'readers': 3})
bounded(60, outage)
start = time.monotonic()
docker('start', nodes[3])
admin = admin_gate(start, 'rejoin')
gate = canary('rejoin', start, admin)
assert not readback('final-readback', gate)
result['status'] = 'PASS'
except BaseException as e:
result['status'] = 'FAIL'
result['error'] = f'{type(e).__name__}: {e}'
raise
finally:
save()
for n in nodes:
log = docker('logs', n, check=False)
(out / (n + '.log')).write_text(log.stdout + log.stderr)
docker('rm', '-f', n, check=False)
docker('rm', '-f', runid + '-keeper', check=False)
for v in volumes:
docker('volume', 'rm', v, check=False)
docker('network', 'rm', runid, check=False)
envfile.unlink(missing_ok=True)
print(json.dumps({'version': version, 'status': result['status'], 'evidence': str(out)}), flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('versions', nargs='+', choices=list(IMAGES))
parser.add_argument('--output', type=Path, required=True, help='directory for retained evidence')
parser.add_argument('--mcli', default=shutil.which('mcli'), help='native mcli executable')
parser.add_argument('--current-binary', type=Path, help='Linux binary matching the Docker architecture')
parser.add_argument('--source-revision', help='Git revision of --current-binary')
parser.add_argument('--current-image', default=IMAGES['current'], help='cached Linux base image for current binary')
args = parser.parse_args()
if not args.mcli or not Path(args.mcli).is_file():
parser.error('--mcli must point to an executable file')
if 'current' in args.versions and (not args.current_binary or not args.current_binary.is_file()):
parser.error('current requires --current-binary')
ROOT = args.output.resolve()
ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
MCLI = Path(args.mcli).resolve()
CURRENT_BINARY = args.current_binary.resolve() if args.current_binary else None
SOURCE_REVISION = args.source_revision
IMAGES['current'] = args.current_image
for version in args.versions:
run(version)
docs/investigations/release-readiness-20260911.md
+130
View File
@@ -0,0 +1,130 @@
# Remaining release correctness work, 2026-09-11
This records the three work items agreed after the branch/PR consolidation:
OIDC #154, Linux restart/readback #116, and the related multi-pool defects
#133/#144. The maintained target is SILO with the PGSTY Console, mcli and
silo-pkg dependencies in the repository's current `go.mod`.
## Multi-pool writes and conditional deletion (#133, #144)
The pools layer now holds its object write lock across PUT and multipart
completion, as it already does for metadata updates and DELETE. Multipart
completion acquires the object lock before the upload lock. Queued healing and
drive healing use this namespace too; healing under a caller-owned lock retains
that lock's cancellation context. The destination-pool allocation policy is
unchanged.
Trusted replica writes resolve the addressed version in every pool, including
draining and rebalancing pools. Retention, legal hold and tags retain their
independent ordering timestamps. A timestamp-only removal survives a stale
retransmit. Completion resolves the version persisted in the upload, including
the null version, rather than taking a later latest version's metadata.
Successful replica replacement retires competing copies of that exact version,
so an equal-ModTime copy in an earlier pool cannot shadow the reconciled result.
Metadata updates evaluate their callback once against the merged version and
update all its copies. Replica metadata COPY rechecks ordering under the lock.
Cleanup failures propagate; a replacement may already have committed when
cleanup fails, and a retry can finish cleanup. Data movement keeps ownership of
its source cleanup.
Conditional DELETE evaluates its precondition against the logical latest or
explicitly addressed version. It checks all pools before mutation, removes
secondary copies before the authoritative one, and returns cleanup errors.
Versioned DELETE without a version ID creates a delete marker and preserves
version history. Retention and replication callbacks evaluate the reconciled
logical version. The existing all-pool delete helper also propagates errors
from non-first pools.
The deterministic two-pool, 32-drive fixtures cover version selection,
duplicate removal, delete failure propagation, PUT/DELETE and completion/DELETE
interleavings, independent lock winners, draining/rebalancing owners, null
versions, metadata COPY, metadata/healing serialization and cleanup retry.
The original branch reproduced the wrong-version lookup, surviving duplicate,
suppressed delete error, PUT/DELETE race, and PUT/completion lock-state failures
before the fixes were applied.
Validation completed locally: the full `cmd` suite (255.720 s), all `internal`
tests, `go vet ./...`, generated-file checks and the branding/entrypoint checks.
Focused race checks cover the pooled interleavings, SSE-C lock regressions,
conditional deletion, access-tier movement and TLS defaults. The two-pool and
related replica/delete/movement tests also passed as a Linux/arm64 test binary
in an isolated container with an 8 GiB `/tmp` tmpfs. Its initial 1 GiB tmpfs
was insufficient for the existing single-drive test fixtures' free-space guard.
## Linux restart/readback (#116)
The [runner](issue-116/run-linux.py) creates four Linux/arm64 server containers
on one Docker Desktop Linux VM, with separate network identities and one drive
per node (EC 2+2). A holder container keeps four 256 MiB Linux tmpfs named
volumes mounted across full server stops. Docker's ordinary filesystem had
only 3.8 GiB free out of 2 TiB and correctly hit the server's free-space limit;
the test uses the separate tmpfs filesystems without changing that threshold.
This is the four-containers-on-one-Linux-host option explicitly accepted in
the [#116 V1 plan](https://github.com/pgsty/silo/issues/116).
This covers process/container restart and TCP peer reconnection on one Linux
host. It does not establish independent-host, host-reboot or physical-media
durability. Every resource created by the runner is removed in its cleanup.
The [retained summary](issue-116/evidence-20260911.json) records image identities,
per-coordinator observations and readback counts.
| Binary | Full restart to all admin views online | Admin gate to complete 4 PUT / 16 GET round | Timed readbacks | Final readback |
|---|---:|---:|---:|---:|
| 0806 release | 2.084 s | 14.449 s | 660 / 660 | 240 / 240 |
| 0903 release | 2.284 s | 0.191 s | 96 / 96 | 52 / 52 |
| Current fix candidate | 2.461 s | 14.489 s | 1356 / 1356 | 904 / 904 |
Each canary has one hard 60-second deadline covering setup, requests, response
body reads and sleeps, with SDK retries disabled. Every acknowledged PUT uses
a unique versioned key and immediately records its VersionId, size and SHA-256.
The timed checks reread these same objects through all four coordinators at
15, 30 and 60 seconds **after the data canary succeeds**. They do not replace
early acknowledgements with later writes. The final check also includes writes
made during the one-node outage and subsequent rejoin canary.
All three runs passed the bounded canary, scheduled readbacks, one-node-outage
read/write and rejoin readback. The candidate also showed a rejoin window:
admin online at 2.490 s, complete data canary 13.977 s later. These are individual
observations, not a latency guarantee or a comparison proving one version
faster. Admin/health readiness still must be followed by a data-path check.
To repeat with cached images and a native mcli executable:
```sh
uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \
0806 0903 --output /tmp/silo-linux-acceptance --mcli /path/to/mcli
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -tags kqueue -o /tmp/silo-linux .
uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \
current --output /tmp/silo-linux-candidate --mcli /path/to/mcli \
--current-binary /tmp/silo-linux --source-revision "$(git rev-parse HEAD)"
```
The runner uses `--pull=never`; preload the two release images, `alpine:3.23`,
and the Linux base image selected with `--current-image`. Match the candidate
binary architecture to that image. Detailed local evidence, including the
acknowledgement ledger and full logs, remains in the requested output directory.
## OIDC (#154)
The TLS implementation fix was already merged in `48e1846525cc`: transports
honor Go's key-exchange defaults, so `GODEBUG=tlsmlkem=0` can opt out of ML-KEM
for an ingress that rejects it. It does not disable certificate validation or
force an automatic protocol downgrade. See the existing
[Go 1.27 investigation](go127-stack.md) and [issue investigation](issue-154.md).
A Linux build of main `b32f2d9dd01a` passed fresh isolated fixture checks:
- ML-KEM-intolerant IdP with `tlsmlkem=0`: discovery, IAM, Console login (204),
authenticated bucket listing (200).
- Normal TLS 1.3 IdP without that override: the same complete login chain.
- Add OIDC through the real administration API with the compatibility setting.
- Invalid JWT signature and audience: no session cookie and bucket access 403.
- Untrusted CA: discovery fails and cluster readiness remains 503.
The affected customer's discovery URL and ingress configuration are still
unavailable. This establishes the supported local fix and its negative
controls, not the root cause or recovery of that hidden deployment. #154 stays
open for an affected-environment retest; no release date or published artifact
is implied by these checks.