From 51d41345f7ac532f9c6fea2b7dba5f29da930b8f Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Fri, 11 Sep 2026 20:02:09 +0800 Subject: [PATCH] test: record Linux restart and OIDC release acceptance Signed-off-by: Feng Ruohang --- .../issue-116/evidence-20260911.json | 534 ++++++++++++++++++ docs/investigations/issue-116/run-linux.py | 327 +++++++++++ .../release-readiness-20260911.md | 130 +++++ 3 files changed, 991 insertions(+) create mode 100644 docs/investigations/issue-116/evidence-20260911.json create mode 100644 docs/investigations/issue-116/run-linux.py create mode 100644 docs/investigations/release-readiness-20260911.md diff --git a/docs/investigations/issue-116/evidence-20260911.json b/docs/investigations/issue-116/evidence-20260911.json new file mode 100644 index 000000000..ab2c005a1 --- /dev/null +++ b/docs/investigations/issue-116/evidence-20260911.json @@ -0,0 +1,534 @@ +{ + "date": "2026-09-11", + "runs": [ + { + "runid": "silo-v1-0806-9c53f14c", + "version": "0806", + "image": "sha256:29a498b24669cae1fed11c1a2fb2b3d73c68829a0a9c0b14e71b386671d38fac", + "nodes": 4, + "drives": 4, + "filesystem": "Linux tmpfs named volumes, held mounted across server restarts", + "drive_bytes": 268435456, + "status": "PASS", + "phases": [ + { + "phase": "startup-admin", + "seconds_from_start": 4.157, + "first_online_by_coordinator": [ + 4.037, + 4.078, + 4.118, + 3.737 + ] + }, + { + "phase": "startup-canary", + "attempts": 1, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 0.239, + "seconds_from_start": 4.396, + "first_put_seconds_after_admin_by_coordinator": [ + 0.096, + 0.116, + 0.127, + 0.138 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 0.162, + 0.194, + 0.217, + 0.239 + ], + "transient_error_count": 0, + "first_transient_errors": [] + }, + { + "phase": "full-restart-admin", + "seconds_from_start": 2.084, + "first_online_by_coordinator": [ + 1.937, + 1.976, + 2.016, + 1.643 + ] + }, + { + "phase": "full-restart-canary", + "attempts": 45, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 14.449, + "seconds_from_start": 16.534, + "first_put_seconds_after_admin_by_coordinator": [ + 14.315, + 13.573, + 13.953, + 0.05 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 14.373, + 14.396, + 14.426, + 14.449 + ], + "transient_error_count": 129, + "first_transient_errors": [ + { + "attempt": 1, + "operation": "put", + "node": 0, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + }, + { + "attempt": 1, + "operation": "put", + "node": 1, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + }, + { + "attempt": 1, + "operation": "put", + "node": 2, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + }, + { + "attempt": 2, + "operation": "put", + "node": 0, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + } + ] + }, + { + "phase": "readback-15s", + "seconds_after_canary": 17.596, + "objects": 55, + "reads": 220, + "errors": [] + }, + { + "phase": "readback-30s", + "seconds_after_canary": 31.357, + "objects": 55, + "reads": 220, + "errors": [] + }, + { + "phase": "readback-60s", + "seconds_after_canary": 61.566, + "objects": 55, + "reads": 220, + "errors": [] + }, + { + "phase": "one-node-outage", + "existing_read": true, + "put": true, + "readers": 3 + }, + { + "phase": "rejoin-admin", + "seconds_from_start": 1.382, + "first_online_by_coordinator": [ + 1.27, + 1.307, + 1.343, + 1.382 + ] + }, + { + "phase": "rejoin-canary", + "attempts": 1, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 0.158, + "seconds_from_start": 1.541, + "first_put_seconds_after_admin_by_coordinator": [ + 0.013, + 0.029, + 0.039, + 0.05 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 0.077, + 0.102, + 0.132, + 0.158 + ], + "transient_error_count": 0, + "first_transient_errors": [] + }, + { + "phase": "final-readback", + "seconds_after_canary": 1.71, + "objects": 60, + "reads": 240, + "errors": [] + } + ], + "acknowledged_objects": 60, + "version_ids_recorded": true, + "sha256_recorded": true + }, + { + "runid": "silo-v1-0903-a457573c", + "version": "0903", + "image": "sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3", + "nodes": 4, + "drives": 4, + "filesystem": "Linux tmpfs named volumes, held mounted across server restarts", + "drive_bytes": 268435456, + "status": "PASS", + "phases": [ + { + "phase": "startup-admin", + "seconds_from_start": 4.271, + "first_online_by_coordinator": [ + 4.158, + 4.194, + 3.827, + 3.867 + ] + }, + { + "phase": "startup-canary", + "attempts": 1, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 0.2, + "seconds_from_start": 4.471, + "first_put_seconds_after_admin_by_coordinator": [ + 0.054, + 0.068, + 0.081, + 0.092 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 0.118, + 0.146, + 0.167, + 0.2 + ], + "transient_error_count": 0, + "first_transient_errors": [] + }, + { + "phase": "full-restart-admin", + "seconds_from_start": 2.284, + "first_online_by_coordinator": [ + 2.143, + 2.193, + 0.849, + 0.902 + ] + }, + { + "phase": "full-restart-canary", + "attempts": 1, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 0.191, + "seconds_from_start": 2.476, + "first_put_seconds_after_admin_by_coordinator": [ + 0.016, + 0.029, + 0.044, + 0.057 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 0.093, + 0.124, + 0.158, + 0.192 + ], + "transient_error_count": 0, + "first_transient_errors": [] + }, + { + "phase": "readback-15s", + "seconds_after_canary": 15.235, + "objects": 8, + "reads": 32, + "errors": [] + }, + { + "phase": "readback-30s", + "seconds_after_canary": 30.356, + "objects": 8, + "reads": 32, + "errors": [] + }, + { + "phase": "readback-60s", + "seconds_after_canary": 60.192, + "objects": 8, + "reads": 32, + "errors": [] + }, + { + "phase": "one-node-outage", + "existing_read": true, + "put": true, + "readers": 3 + }, + { + "phase": "rejoin-admin", + "seconds_from_start": 0.903, + "first_online_by_coordinator": [ + 0.791, + 0.829, + 0.865, + 0.903 + ] + }, + { + "phase": "rejoin-canary", + "attempts": 1, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 0.141, + "seconds_from_start": 1.044, + "first_put_seconds_after_admin_by_coordinator": [ + 0.012, + 0.022, + 0.035, + 0.045 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 0.069, + 0.096, + 0.12, + 0.141 + ], + "transient_error_count": 0, + "first_transient_errors": [] + }, + { + "phase": "final-readback", + "seconds_after_canary": 0.301, + "objects": 13, + "reads": 52, + "errors": [] + } + ], + "acknowledged_objects": 13, + "version_ids_recorded": true, + "sha256_recorded": true + }, + { + "runid": "silo-v1-current-6a14eb18", + "version": "current", + "image": "pgsty/d12a:build", + "image_id": "sha256:307af7711e2e04ab75759cb42a1eef45c43c4404894c0e30dd19f742b107b922", + "platform": "linux/arm64", + "nodes": 4, + "drives": 4, + "filesystem": "Linux tmpfs named volumes, held mounted across server restarts", + "drive_bytes": 268435456, + "status": "PASS", + "binary_sha256": "1e4cd7b78ecfa1b0cf28f1961d48a220a712c6be1fd3a01b60ee99aec62f04a4", + "source_revision": "b32f2d9dd01a383a9991d34ffe248063463a031d+pool-consistency", + "phases": [ + { + "phase": "startup-admin", + "seconds_from_start": 7.374, + "first_online_by_coordinator": [ + 7.207, + 6.677, + 6.74, + 6.8 + ] + }, + { + "phase": "startup-canary", + "attempts": 1, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 0.479, + "seconds_from_start": 7.854, + "first_put_seconds_after_admin_by_coordinator": [ + 0.266, + 0.279, + 0.292, + 0.305 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 0.336, + 0.378, + 0.431, + 0.479 + ], + "transient_error_count": 0, + "first_transient_errors": [] + }, + { + "phase": "full-restart-admin", + "seconds_from_start": 2.461, + "first_online_by_coordinator": [ + 2.264, + 1.414, + 2.396, + 2.461 + ] + }, + { + "phase": "full-restart-canary", + "attempts": 36, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 14.489, + "seconds_from_start": 16.951, + "first_put_seconds_after_admin_by_coordinator": [ + 0.013, + 0.025, + 14.301, + 0.046 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 14.358, + 14.394, + 14.435, + 14.489 + ], + "transient_error_count": 104, + "first_transient_errors": [ + { + "attempt": 1, + "operation": "put", + "node": 2, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + }, + { + "attempt": 1, + "operation": "get", + "node": 2, + "key": "full-restart-attempt-1-node-0", + "error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate" + }, + { + "attempt": 1, + "operation": "get", + "node": 2, + "key": "full-restart-attempt-1-node-3", + "error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate" + }, + { + "attempt": 2, + "operation": "put", + "node": 2, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + } + ] + }, + { + "phase": "readback-15s", + "started_seconds_after_canary": 15.0, + "seconds_after_canary": 22.006, + "objects": 113, + "reads": 452, + "errors": [] + }, + { + "phase": "readback-30s", + "started_seconds_after_canary": 30.003, + "seconds_after_canary": 36.571, + "objects": 113, + "reads": 452, + "errors": [] + }, + { + "phase": "readback-60s", + "started_seconds_after_canary": 60.0, + "seconds_after_canary": 65.743, + "objects": 113, + "reads": 452, + "errors": [] + }, + { + "phase": "one-node-outage", + "existing_read": true, + "put": true, + "readers": 3 + }, + { + "phase": "rejoin-admin", + "seconds_from_start": 2.49, + "first_online_by_coordinator": [ + 1.165, + 2.417, + 1.646, + 2.087 + ] + }, + { + "phase": "rejoin-canary", + "attempts": 37, + "puts": 4, + "gets": 16, + "hard_deadline_seconds": 60, + "gate_seconds": 13.977, + "seconds_from_start": 16.468, + "first_put_seconds_after_admin_by_coordinator": [ + 0.013, + 0.024, + 0.035, + 13.889 + ], + "first_four_reads_seconds_after_admin_by_coordinator": [ + 13.909, + 13.927, + 13.953, + 13.977 + ], + "transient_error_count": 36, + "first_transient_errors": [ + { + "attempt": 1, + "operation": "put", + "node": 3, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + }, + { + "attempt": 2, + "operation": "put", + "node": 3, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + }, + { + "attempt": 3, + "operation": "put", + "node": 3, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + }, + { + "attempt": 4, + "operation": "put", + "node": 3, + "error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate" + } + ] + }, + { + "phase": "final-readback", + "started_seconds_after_canary": 0.0, + "seconds_after_canary": 6.507, + "objects": 226, + "reads": 904, + "errors": [] + } + ], + "acknowledged_objects": 226, + "version_ids_recorded": true, + "sha256_recorded": true + } + ] +} diff --git a/docs/investigations/issue-116/run-linux.py b/docs/investigations/issue-116/run-linux.py new file mode 100644 index 000000000..f34e12be6 --- /dev/null +++ b/docs/investigations/issue-116/run-linux.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Bounded four-container restart/readback acceptance for pgsty/silo#116.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import secrets +import shutil +import signal +import socket +import subprocess +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import boto3 +from botocore.config import Config + +ROOT = None +MCLI = None +CURRENT_BINARY = None +SOURCE_REVISION = None +IMAGES = { + '0806': 'pgsty/silo:RELEASE.2026-08-06T00-00-00Z', + '0903': 'pgsty/silo:RELEASE.2026-09-03T13-18-01Z', + 'current': 'pgsty/d12a:build', +} + + +def docker(*args, timeout=40, check=True): + return subprocess.run(['docker', *args], check=check, capture_output=True, text=True, timeout=timeout) + + +class Deadline(BaseException): + pass + + +def bounded(seconds, operation): + def expired(*_): + raise Deadline(f'hard deadline of {seconds}s exceeded') + old = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + return operation(time.monotonic() + seconds) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, old) + + +def remaining(deadline): + value = deadline - time.monotonic() + if value <= 0: + raise Deadline('absolute deadline exceeded') + return value + + +def run(version): + image_info = json.loads(docker('image', 'inspect', IMAGES[version]).stdout)[0] + runid = f'silo-v1-{version}-{uuid.uuid4().hex[:8]}' + out = ROOT / runid + out.mkdir(mode=0o700) + user, password = 'local116', secrets.token_urlsafe(24) + envfile = out / 'credentials.env' + envfile.write_text(f'MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\nMINIO_CI_CD=1\nMINIO_BROWSER=off\nGOMAXPROCS=2\n') + envfile.chmod(0o600) + env = {k: v for k, v in os.environ.items() if not k.startswith(('MINIO_', 'SILO_', 'MC_')) + and k.lower() not in {'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy'}} + nodes = [f'{runid}-n{i}' for i in range(4)] + sockets = [socket.socket() for _ in nodes] + for sock in sockets: + sock.bind(('127.0.0.1', 0)) + ports = [sock.getsockname()[1] for sock in sockets] + for sock in sockets: + sock.close() + volumes = [n + '-data' for n in nodes] + endpoints, ledger = [], [] + result = {'runid': runid, 'version': version, 'image': IMAGES[version], + 'image_id': image_info['Id'], 'platform': image_info['Os']+'/'+image_info['Architecture'], + 'nodes': 4, + 'drives': 4, 'filesystem': 'Linux tmpfs named volumes, held mounted across server restarts', + 'drive_bytes': 268435456, 'phases': [], 'status': 'RUNNING'} + if version == 'current': + result['binary_sha256'] = hashlib.sha256(CURRENT_BINARY.read_bytes()).hexdigest() + result['source_revision'] = SOURCE_REVISION + bucket = 'canary-' + uuid.uuid4().hex[:10] + + def save(event=None): + if event is not None: + result['phases'].append(event) + compact = {k: (len(v) if k in ('transient_errors', 'errors') else v) for k, v in event.items()} + print(json.dumps({'version': version, **compact}), flush=True) + (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') + (out / 'acknowledged.json').write_text(json.dumps(ledger, indent=2) + '\n') + + def parallel(fn, values): + with ThreadPoolExecutor(max_workers=4) as pool: + return list(pool.map(fn, values)) + + def client(i, deadline): + timeout = remaining(deadline) + return boto3.client('s3', endpoint_url=endpoints[i], aws_access_key_id=user, + aws_secret_access_key=password, region_name='us-east-1', + config=Config(proxies={}, signature_version='s3v4', s3={'addressing_style': 'path'}, + retries={'total_max_attempts': 1}, connect_timeout=timeout, + read_timeout=timeout, request_checksum_calculation='when_required', + response_checksum_validation='when_required')) + + def admin_gate(origin, phase): + def check(deadline): + first = [None] * 4 + while True: + states = [] + for i in range(4): + try: + p = subprocess.run([str(MCLI), '--config-dir', str(out / 'mcli'), '--json', + 'admin', 'info', f'n{i}'], env=env, capture_output=True, + text=True, timeout=min(5, remaining(deadline))) + info = json.loads(p.stdout)['info'] + servers = info['servers'] + ok = len(servers) == 4 and all(s.get('state') == 'online' and s.get('drives') + and all(d.get('state') == 'ok' for d in s['drives']) + and all(v == 'online' for v in s.get('network', {}).values()) for s in servers) + if ok: + (out / f'{phase}-admin-{i}.json').write_text(json.dumps(info, indent=2) + '\n') + if first[i] is None: + first[i] = round(time.monotonic() - origin, 3) + states.append(ok) + except (Exception,): + states.append(False) + if all(states): + event = {'phase': phase + '-admin', 'seconds_from_start': round(time.monotonic()-origin, 3), + 'first_online_by_coordinator': first} + save(event) + return time.monotonic() + time.sleep(min(.25, remaining(deadline))) + return bounded(90, check) + + def canary(phase, origin, admin_time, setup=False): + def check(deadline): + started = time.monotonic() + first_put, first_reads = [None] * 4, [None] * 4 + errors, attempt, setup_done = [], 0, not setup + result['active_canary'] = {'phase': phase, 'errors': errors} + while True: + attempt += 1 + remaining(deadline) + if not setup_done: + try: + try: + client(0, deadline).create_bucket(Bucket=bucket) + except Exception as e: + if 'BucketAlreadyOwnedByYou' not in str(e): + raise + client(0, deadline).put_bucket_versioning(Bucket=bucket, VersioningConfiguration={'Status': 'Enabled'}) + setup_done = True + except Exception as e: + errors.append({'attempt': attempt, 'operation': 'setup', 'error': str(e)[:250]}) + time.sleep(min(.25, remaining(deadline))) + continue + acked = [] + for i in range(4): + key = f'{phase}-attempt-{attempt}-node-{i}' + payload = (key + '\n').encode() * 16384 + try: + vid = client(i, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId'] + entry = {'phase': phase, 'key': key, 'version': vid, 'bytes': len(payload), + 'sha256': hashlib.sha256(payload).hexdigest(), 'writer': i, + 'ack_seconds_from_start': round(time.monotonic()-origin, 3)} + ledger.append(entry) + save() # Persist every acknowledged write, including failed rounds. + acked.append(entry) + if first_put[i] is None: + first_put[i] = round(time.monotonic()-admin_time, 3) + except Exception as e: + errors.append({'attempt': attempt, 'operation': 'put', 'node': i, 'error': str(e)[:250]}) + reads = 0 + for i in range(4): + own_reads = 0 + for entry in acked: + try: + verify(i, entry, deadline) + reads += 1 + own_reads += 1 + except Exception as e: + errors.append({'attempt': attempt, 'operation': 'get', 'node': i, + 'key': entry['key'], 'error': str(e)[:250]}) + if own_reads == 4 and first_reads[i] is None: + first_reads[i] = round(time.monotonic()-admin_time, 3) + remaining(deadline) + if len(acked) == 4 and reads == 16: + result.pop('active_canary', None) + save({'phase': phase + '-canary', 'attempts': attempt, 'puts': 4, 'gets': 16, + 'hard_deadline_seconds': 60, 'gate_seconds': round(time.monotonic()-started, 3), + 'seconds_from_start': round(time.monotonic()-origin, 3), + 'first_put_seconds_after_admin_by_coordinator': first_put, + 'first_four_reads_seconds_after_admin_by_coordinator': first_reads, 'transient_errors': errors}) + return time.monotonic() + (out / f'{phase}-canary-errors.json').write_text(json.dumps(errors, indent=2) + '\n') + if attempt == 1: + print(json.dumps({'version': version, 'phase': phase, 'first_attempt_errors': errors}), flush=True) + time.sleep(min(.25, remaining(deadline))) + return bounded(60, check) + + def verify(i, entry, deadline): + got = client(i, deadline).get_object(Bucket=bucket, Key=entry['key'], VersionId=entry['version']) + try: + data = got['Body'].read() + finally: + got['Body'].close() + assert len(data) == entry['bytes'] and hashlib.sha256(data).hexdigest() == entry['sha256'], entry['key'] + assert got.get('VersionId') == entry['version'], entry['key'] + remaining(deadline) + + def readback(label, origin): + def check(deadline): + started = time.monotonic() + errors = [] + for entry in ledger: + for i in range(4): + try: + verify(i, entry, deadline) + except Exception as e: + errors.append({'key': entry['key'], 'node': i, 'error': str(e)[:250]}) + save({'phase': label, 'started_seconds_after_canary': round(started-origin, 3), + 'seconds_after_canary': round(time.monotonic()-origin, 3), + 'objects': len(ledger), 'reads': len(ledger)*4, 'errors': errors}) + return errors + return bounded(60, check) + + try: + docker('network', 'create', runid) + for volume in volumes: + docker('volume', 'create', '--driver', 'local', '--opt', 'type=tmpfs', + '--opt', 'device=tmpfs', '--opt', 'o=size=256m', volume) + mounts = [arg for i, v in enumerate(volumes) for arg in ('--mount', f'type=volume,source={v},target=/keep/{i}')] + docker('run', '-d', '--pull=never', '--network', 'none', '--name', runid + '-keeper', + *mounts, '--entrypoint', 'sleep', 'alpine:3.23', '1800') + urls = [f'http://{n}:9000/data' for n in nodes] + def create(i): + extra = [] + if version == 'current': + extra = ['--mount', f'type=bind,source={CURRENT_BINARY},target=/lab/silo,readonly', + '--entrypoint', '/lab/silo'] + docker('create', '--pull=never', '--name', nodes[i], '--network', runid, + '--hostname', nodes[i], '--cpus', '2', '--memory', '3g', '--env-file', str(envfile), + '--mount', f'type=volume,source={volumes[i]},target=/data', + '-p', f'127.0.0.1:{ports[i]}:9000', *extra, image_info['Id'], 'server', '--address', ':9000', + '--console-address', ':9001', *urls) + parallel(create, range(4)) + start = time.monotonic() + parallel(lambda n: docker('start', n), nodes) + for i, n in enumerate(nodes): + endpoint = 'http://' + docker('port', n, '9000/tcp').stdout.strip() + endpoints.append(endpoint) + env[f'MC_HOST_n{i}'] = endpoint.replace('http://', f'http://{user}:{password}@') + admin = admin_gate(start, 'startup') + canary('startup', start, admin, setup=True) + parallel(lambda n: docker('stop', '-t', '10', n), nodes) + start = time.monotonic() + parallel(lambda n: docker('start', n), nodes) + admin = admin_gate(start, 'full-restart') + gate = canary('full-restart', start, admin) + read_errors = [] + for after in (15, 30, 60): + time.sleep(max(0, gate + after - time.monotonic())) + read_errors.extend(readback(f'readback-{after}s', gate)) + assert not read_errors, 'acknowledged object readback failure; see result.json' + docker('stop', '-t', '10', nodes[3]) + def outage(deadline): + verify(0, ledger[0], deadline) + payload = b'acknowledged with one Linux node offline' * 16384 + key = 'one-node-outage' + vid = client(0, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId'] + entry = {'phase': 'outage', 'key': key, 'version': vid, 'bytes': len(payload), + 'sha256': hashlib.sha256(payload).hexdigest(), 'writer': 0} + ledger.append(entry) + save() + for i in range(3): + verify(i, entry, deadline) + save({'phase': 'one-node-outage', 'existing_read': True, 'put': True, 'readers': 3}) + bounded(60, outage) + start = time.monotonic() + docker('start', nodes[3]) + admin = admin_gate(start, 'rejoin') + gate = canary('rejoin', start, admin) + assert not readback('final-readback', gate) + result['status'] = 'PASS' + except BaseException as e: + result['status'] = 'FAIL' + result['error'] = f'{type(e).__name__}: {e}' + raise + finally: + save() + for n in nodes: + log = docker('logs', n, check=False) + (out / (n + '.log')).write_text(log.stdout + log.stderr) + docker('rm', '-f', n, check=False) + docker('rm', '-f', runid + '-keeper', check=False) + for v in volumes: + docker('volume', 'rm', v, check=False) + docker('network', 'rm', runid, check=False) + envfile.unlink(missing_ok=True) + print(json.dumps({'version': version, 'status': result['status'], 'evidence': str(out)}), flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('versions', nargs='+', choices=list(IMAGES)) + parser.add_argument('--output', type=Path, required=True, help='directory for retained evidence') + parser.add_argument('--mcli', default=shutil.which('mcli'), help='native mcli executable') + parser.add_argument('--current-binary', type=Path, help='Linux binary matching the Docker architecture') + parser.add_argument('--source-revision', help='Git revision of --current-binary') + parser.add_argument('--current-image', default=IMAGES['current'], help='cached Linux base image for current binary') + args = parser.parse_args() + if not args.mcli or not Path(args.mcli).is_file(): + parser.error('--mcli must point to an executable file') + if 'current' in args.versions and (not args.current_binary or not args.current_binary.is_file()): + parser.error('current requires --current-binary') + ROOT = args.output.resolve() + ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) + MCLI = Path(args.mcli).resolve() + CURRENT_BINARY = args.current_binary.resolve() if args.current_binary else None + SOURCE_REVISION = args.source_revision + IMAGES['current'] = args.current_image + for version in args.versions: + run(version) diff --git a/docs/investigations/release-readiness-20260911.md b/docs/investigations/release-readiness-20260911.md new file mode 100644 index 000000000..70bee061d --- /dev/null +++ b/docs/investigations/release-readiness-20260911.md @@ -0,0 +1,130 @@ +# Remaining release correctness work, 2026-09-11 + +This records the three work items agreed after the branch/PR consolidation: +OIDC #154, Linux restart/readback #116, and the related multi-pool defects +#133/#144. The maintained target is SILO with the PGSTY Console, mcli and +silo-pkg dependencies in the repository's current `go.mod`. + +## Multi-pool writes and conditional deletion (#133, #144) + +The pools layer now holds its object write lock across PUT and multipart +completion, as it already does for metadata updates and DELETE. Multipart +completion acquires the object lock before the upload lock. Queued healing and +drive healing use this namespace too; healing under a caller-owned lock retains +that lock's cancellation context. The destination-pool allocation policy is +unchanged. + +Trusted replica writes resolve the addressed version in every pool, including +draining and rebalancing pools. Retention, legal hold and tags retain their +independent ordering timestamps. A timestamp-only removal survives a stale +retransmit. Completion resolves the version persisted in the upload, including +the null version, rather than taking a later latest version's metadata. + +Successful replica replacement retires competing copies of that exact version, +so an equal-ModTime copy in an earlier pool cannot shadow the reconciled result. +Metadata updates evaluate their callback once against the merged version and +update all its copies. Replica metadata COPY rechecks ordering under the lock. +Cleanup failures propagate; a replacement may already have committed when +cleanup fails, and a retry can finish cleanup. Data movement keeps ownership of +its source cleanup. + +Conditional DELETE evaluates its precondition against the logical latest or +explicitly addressed version. It checks all pools before mutation, removes +secondary copies before the authoritative one, and returns cleanup errors. +Versioned DELETE without a version ID creates a delete marker and preserves +version history. Retention and replication callbacks evaluate the reconciled +logical version. The existing all-pool delete helper also propagates errors +from non-first pools. + +The deterministic two-pool, 32-drive fixtures cover version selection, +duplicate removal, delete failure propagation, PUT/DELETE and completion/DELETE +interleavings, independent lock winners, draining/rebalancing owners, null +versions, metadata COPY, metadata/healing serialization and cleanup retry. +The original branch reproduced the wrong-version lookup, surviving duplicate, +suppressed delete error, PUT/DELETE race, and PUT/completion lock-state failures +before the fixes were applied. + +Validation completed locally: the full `cmd` suite (255.720 s), all `internal` +tests, `go vet ./...`, generated-file checks and the branding/entrypoint checks. +Focused race checks cover the pooled interleavings, SSE-C lock regressions, +conditional deletion, access-tier movement and TLS defaults. The two-pool and +related replica/delete/movement tests also passed as a Linux/arm64 test binary +in an isolated container with an 8 GiB `/tmp` tmpfs. Its initial 1 GiB tmpfs +was insufficient for the existing single-drive test fixtures' free-space guard. + +## Linux restart/readback (#116) + +The [runner](issue-116/run-linux.py) creates four Linux/arm64 server containers +on one Docker Desktop Linux VM, with separate network identities and one drive +per node (EC 2+2). A holder container keeps four 256 MiB Linux tmpfs named +volumes mounted across full server stops. Docker's ordinary filesystem had +only 3.8 GiB free out of 2 TiB and correctly hit the server's free-space limit; +the test uses the separate tmpfs filesystems without changing that threshold. +This is the four-containers-on-one-Linux-host option explicitly accepted in +the [#116 V1 plan](https://github.com/pgsty/silo/issues/116). + +This covers process/container restart and TCP peer reconnection on one Linux +host. It does not establish independent-host, host-reboot or physical-media +durability. Every resource created by the runner is removed in its cleanup. +The [retained summary](issue-116/evidence-20260911.json) records image identities, +per-coordinator observations and readback counts. + +| Binary | Full restart to all admin views online | Admin gate to complete 4 PUT / 16 GET round | Timed readbacks | Final readback | +|---|---:|---:|---:|---:| +| 0806 release | 2.084 s | 14.449 s | 660 / 660 | 240 / 240 | +| 0903 release | 2.284 s | 0.191 s | 96 / 96 | 52 / 52 | +| Current fix candidate | 2.461 s | 14.489 s | 1356 / 1356 | 904 / 904 | + +Each canary has one hard 60-second deadline covering setup, requests, response +body reads and sleeps, with SDK retries disabled. Every acknowledged PUT uses +a unique versioned key and immediately records its VersionId, size and SHA-256. +The timed checks reread these same objects through all four coordinators at +15, 30 and 60 seconds **after the data canary succeeds**. They do not replace +early acknowledgements with later writes. The final check also includes writes +made during the one-node outage and subsequent rejoin canary. + +All three runs passed the bounded canary, scheduled readbacks, one-node-outage +read/write and rejoin readback. The candidate also showed a rejoin window: +admin online at 2.490 s, complete data canary 13.977 s later. These are individual +observations, not a latency guarantee or a comparison proving one version +faster. Admin/health readiness still must be followed by a data-path check. + +To repeat with cached images and a native mcli executable: + +```sh +uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \ + 0806 0903 --output /tmp/silo-linux-acceptance --mcli /path/to/mcli + +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -tags kqueue -o /tmp/silo-linux . +uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \ + current --output /tmp/silo-linux-candidate --mcli /path/to/mcli \ + --current-binary /tmp/silo-linux --source-revision "$(git rev-parse HEAD)" +``` + +The runner uses `--pull=never`; preload the two release images, `alpine:3.23`, +and the Linux base image selected with `--current-image`. Match the candidate +binary architecture to that image. Detailed local evidence, including the +acknowledgement ledger and full logs, remains in the requested output directory. + +## OIDC (#154) + +The TLS implementation fix was already merged in `48e1846525cc`: transports +honor Go's key-exchange defaults, so `GODEBUG=tlsmlkem=0` can opt out of ML-KEM +for an ingress that rejects it. It does not disable certificate validation or +force an automatic protocol downgrade. See the existing +[Go 1.27 investigation](go127-stack.md) and [issue investigation](issue-154.md). + +A Linux build of main `b32f2d9dd01a` passed fresh isolated fixture checks: + +- ML-KEM-intolerant IdP with `tlsmlkem=0`: discovery, IAM, Console login (204), + authenticated bucket listing (200). +- Normal TLS 1.3 IdP without that override: the same complete login chain. +- Add OIDC through the real administration API with the compatibility setting. +- Invalid JWT signature and audience: no session cookie and bucket access 403. +- Untrusted CA: discovery fails and cluster readiness remains 503. + +The affected customer's discovery URL and ingress configuration are still +unavailable. This establishes the supported local fix and its negative +controls, not the root cause or recovery of that hidden deployment. #154 stays +open for an affected-environment retest; no release date or published artifact +is implied by these checks.