mirror of
https://github.com/pgsty/minio.git
synced 2026-09-12 21:44:04 +03:00
test: record Linux restart and OIDC release acceptance
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
@@ -0,0 +1,534 @@
|
||||
{
|
||||
"date": "2026-09-11",
|
||||
"runs": [
|
||||
{
|
||||
"runid": "silo-v1-0806-9c53f14c",
|
||||
"version": "0806",
|
||||
"image": "sha256:29a498b24669cae1fed11c1a2fb2b3d73c68829a0a9c0b14e71b386671d38fac",
|
||||
"nodes": 4,
|
||||
"drives": 4,
|
||||
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||
"drive_bytes": 268435456,
|
||||
"status": "PASS",
|
||||
"phases": [
|
||||
{
|
||||
"phase": "startup-admin",
|
||||
"seconds_from_start": 4.157,
|
||||
"first_online_by_coordinator": [
|
||||
4.037,
|
||||
4.078,
|
||||
4.118,
|
||||
3.737
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "startup-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.239,
|
||||
"seconds_from_start": 4.396,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.096,
|
||||
0.116,
|
||||
0.127,
|
||||
0.138
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.162,
|
||||
0.194,
|
||||
0.217,
|
||||
0.239
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-admin",
|
||||
"seconds_from_start": 2.084,
|
||||
"first_online_by_coordinator": [
|
||||
1.937,
|
||||
1.976,
|
||||
2.016,
|
||||
1.643
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-canary",
|
||||
"attempts": 45,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 14.449,
|
||||
"seconds_from_start": 16.534,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
14.315,
|
||||
13.573,
|
||||
13.953,
|
||||
0.05
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
14.373,
|
||||
14.396,
|
||||
14.426,
|
||||
14.449
|
||||
],
|
||||
"transient_error_count": 129,
|
||||
"first_transient_errors": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 0,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 1,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 2,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"operation": "put",
|
||||
"node": 0,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "readback-15s",
|
||||
"seconds_after_canary": 17.596,
|
||||
"objects": 55,
|
||||
"reads": 220,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-30s",
|
||||
"seconds_after_canary": 31.357,
|
||||
"objects": 55,
|
||||
"reads": 220,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-60s",
|
||||
"seconds_after_canary": 61.566,
|
||||
"objects": 55,
|
||||
"reads": 220,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "one-node-outage",
|
||||
"existing_read": true,
|
||||
"put": true,
|
||||
"readers": 3
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-admin",
|
||||
"seconds_from_start": 1.382,
|
||||
"first_online_by_coordinator": [
|
||||
1.27,
|
||||
1.307,
|
||||
1.343,
|
||||
1.382
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.158,
|
||||
"seconds_from_start": 1.541,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.013,
|
||||
0.029,
|
||||
0.039,
|
||||
0.05
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.077,
|
||||
0.102,
|
||||
0.132,
|
||||
0.158
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "final-readback",
|
||||
"seconds_after_canary": 1.71,
|
||||
"objects": 60,
|
||||
"reads": 240,
|
||||
"errors": []
|
||||
}
|
||||
],
|
||||
"acknowledged_objects": 60,
|
||||
"version_ids_recorded": true,
|
||||
"sha256_recorded": true
|
||||
},
|
||||
{
|
||||
"runid": "silo-v1-0903-a457573c",
|
||||
"version": "0903",
|
||||
"image": "sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3",
|
||||
"nodes": 4,
|
||||
"drives": 4,
|
||||
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||
"drive_bytes": 268435456,
|
||||
"status": "PASS",
|
||||
"phases": [
|
||||
{
|
||||
"phase": "startup-admin",
|
||||
"seconds_from_start": 4.271,
|
||||
"first_online_by_coordinator": [
|
||||
4.158,
|
||||
4.194,
|
||||
3.827,
|
||||
3.867
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "startup-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.2,
|
||||
"seconds_from_start": 4.471,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.054,
|
||||
0.068,
|
||||
0.081,
|
||||
0.092
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.118,
|
||||
0.146,
|
||||
0.167,
|
||||
0.2
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-admin",
|
||||
"seconds_from_start": 2.284,
|
||||
"first_online_by_coordinator": [
|
||||
2.143,
|
||||
2.193,
|
||||
0.849,
|
||||
0.902
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.191,
|
||||
"seconds_from_start": 2.476,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.016,
|
||||
0.029,
|
||||
0.044,
|
||||
0.057
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.093,
|
||||
0.124,
|
||||
0.158,
|
||||
0.192
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-15s",
|
||||
"seconds_after_canary": 15.235,
|
||||
"objects": 8,
|
||||
"reads": 32,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-30s",
|
||||
"seconds_after_canary": 30.356,
|
||||
"objects": 8,
|
||||
"reads": 32,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-60s",
|
||||
"seconds_after_canary": 60.192,
|
||||
"objects": 8,
|
||||
"reads": 32,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "one-node-outage",
|
||||
"existing_read": true,
|
||||
"put": true,
|
||||
"readers": 3
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-admin",
|
||||
"seconds_from_start": 0.903,
|
||||
"first_online_by_coordinator": [
|
||||
0.791,
|
||||
0.829,
|
||||
0.865,
|
||||
0.903
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.141,
|
||||
"seconds_from_start": 1.044,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.012,
|
||||
0.022,
|
||||
0.035,
|
||||
0.045
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.069,
|
||||
0.096,
|
||||
0.12,
|
||||
0.141
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "final-readback",
|
||||
"seconds_after_canary": 0.301,
|
||||
"objects": 13,
|
||||
"reads": 52,
|
||||
"errors": []
|
||||
}
|
||||
],
|
||||
"acknowledged_objects": 13,
|
||||
"version_ids_recorded": true,
|
||||
"sha256_recorded": true
|
||||
},
|
||||
{
|
||||
"runid": "silo-v1-current-6a14eb18",
|
||||
"version": "current",
|
||||
"image": "pgsty/d12a:build",
|
||||
"image_id": "sha256:307af7711e2e04ab75759cb42a1eef45c43c4404894c0e30dd19f742b107b922",
|
||||
"platform": "linux/arm64",
|
||||
"nodes": 4,
|
||||
"drives": 4,
|
||||
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||
"drive_bytes": 268435456,
|
||||
"status": "PASS",
|
||||
"binary_sha256": "1e4cd7b78ecfa1b0cf28f1961d48a220a712c6be1fd3a01b60ee99aec62f04a4",
|
||||
"source_revision": "b32f2d9dd01a383a9991d34ffe248063463a031d+pool-consistency",
|
||||
"phases": [
|
||||
{
|
||||
"phase": "startup-admin",
|
||||
"seconds_from_start": 7.374,
|
||||
"first_online_by_coordinator": [
|
||||
7.207,
|
||||
6.677,
|
||||
6.74,
|
||||
6.8
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "startup-canary",
|
||||
"attempts": 1,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 0.479,
|
||||
"seconds_from_start": 7.854,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.266,
|
||||
0.279,
|
||||
0.292,
|
||||
0.305
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
0.336,
|
||||
0.378,
|
||||
0.431,
|
||||
0.479
|
||||
],
|
||||
"transient_error_count": 0,
|
||||
"first_transient_errors": []
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-admin",
|
||||
"seconds_from_start": 2.461,
|
||||
"first_online_by_coordinator": [
|
||||
2.264,
|
||||
1.414,
|
||||
2.396,
|
||||
2.461
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "full-restart-canary",
|
||||
"attempts": 36,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 14.489,
|
||||
"seconds_from_start": 16.951,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.013,
|
||||
0.025,
|
||||
14.301,
|
||||
0.046
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
14.358,
|
||||
14.394,
|
||||
14.435,
|
||||
14.489
|
||||
],
|
||||
"transient_error_count": 104,
|
||||
"first_transient_errors": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 2,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "get",
|
||||
"node": 2,
|
||||
"key": "full-restart-attempt-1-node-0",
|
||||
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "get",
|
||||
"node": 2,
|
||||
"key": "full-restart-attempt-1-node-3",
|
||||
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"operation": "put",
|
||||
"node": 2,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "readback-15s",
|
||||
"started_seconds_after_canary": 15.0,
|
||||
"seconds_after_canary": 22.006,
|
||||
"objects": 113,
|
||||
"reads": 452,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-30s",
|
||||
"started_seconds_after_canary": 30.003,
|
||||
"seconds_after_canary": 36.571,
|
||||
"objects": 113,
|
||||
"reads": 452,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "readback-60s",
|
||||
"started_seconds_after_canary": 60.0,
|
||||
"seconds_after_canary": 65.743,
|
||||
"objects": 113,
|
||||
"reads": 452,
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"phase": "one-node-outage",
|
||||
"existing_read": true,
|
||||
"put": true,
|
||||
"readers": 3
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-admin",
|
||||
"seconds_from_start": 2.49,
|
||||
"first_online_by_coordinator": [
|
||||
1.165,
|
||||
2.417,
|
||||
1.646,
|
||||
2.087
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "rejoin-canary",
|
||||
"attempts": 37,
|
||||
"puts": 4,
|
||||
"gets": 16,
|
||||
"hard_deadline_seconds": 60,
|
||||
"gate_seconds": 13.977,
|
||||
"seconds_from_start": 16.468,
|
||||
"first_put_seconds_after_admin_by_coordinator": [
|
||||
0.013,
|
||||
0.024,
|
||||
0.035,
|
||||
13.889
|
||||
],
|
||||
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||
13.909,
|
||||
13.927,
|
||||
13.953,
|
||||
13.977
|
||||
],
|
||||
"transient_error_count": 36,
|
||||
"first_transient_errors": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 3,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
},
|
||||
{
|
||||
"attempt": 4,
|
||||
"operation": "put",
|
||||
"node": 3,
|
||||
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": "final-readback",
|
||||
"started_seconds_after_canary": 0.0,
|
||||
"seconds_after_canary": 6.507,
|
||||
"objects": 226,
|
||||
"reads": 904,
|
||||
"errors": []
|
||||
}
|
||||
],
|
||||
"acknowledged_objects": 226,
|
||||
"version_ids_recorded": true,
|
||||
"sha256_recorded": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded four-container restart/readback acceptance for pgsty/silo#116."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
ROOT = None
|
||||
MCLI = None
|
||||
CURRENT_BINARY = None
|
||||
SOURCE_REVISION = None
|
||||
IMAGES = {
|
||||
'0806': 'pgsty/silo:RELEASE.2026-08-06T00-00-00Z',
|
||||
'0903': 'pgsty/silo:RELEASE.2026-09-03T13-18-01Z',
|
||||
'current': 'pgsty/d12a:build',
|
||||
}
|
||||
|
||||
|
||||
def docker(*args, timeout=40, check=True):
|
||||
return subprocess.run(['docker', *args], check=check, capture_output=True, text=True, timeout=timeout)
|
||||
|
||||
|
||||
class Deadline(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
def bounded(seconds, operation):
|
||||
def expired(*_):
|
||||
raise Deadline(f'hard deadline of {seconds}s exceeded')
|
||||
old = signal.signal(signal.SIGALRM, expired)
|
||||
signal.setitimer(signal.ITIMER_REAL, seconds)
|
||||
try:
|
||||
return operation(time.monotonic() + seconds)
|
||||
finally:
|
||||
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||
signal.signal(signal.SIGALRM, old)
|
||||
|
||||
|
||||
def remaining(deadline):
|
||||
value = deadline - time.monotonic()
|
||||
if value <= 0:
|
||||
raise Deadline('absolute deadline exceeded')
|
||||
return value
|
||||
|
||||
|
||||
def run(version):
|
||||
image_info = json.loads(docker('image', 'inspect', IMAGES[version]).stdout)[0]
|
||||
runid = f'silo-v1-{version}-{uuid.uuid4().hex[:8]}'
|
||||
out = ROOT / runid
|
||||
out.mkdir(mode=0o700)
|
||||
user, password = 'local116', secrets.token_urlsafe(24)
|
||||
envfile = out / 'credentials.env'
|
||||
envfile.write_text(f'MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\nMINIO_CI_CD=1\nMINIO_BROWSER=off\nGOMAXPROCS=2\n')
|
||||
envfile.chmod(0o600)
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith(('MINIO_', 'SILO_', 'MC_'))
|
||||
and k.lower() not in {'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy'}}
|
||||
nodes = [f'{runid}-n{i}' for i in range(4)]
|
||||
sockets = [socket.socket() for _ in nodes]
|
||||
for sock in sockets:
|
||||
sock.bind(('127.0.0.1', 0))
|
||||
ports = [sock.getsockname()[1] for sock in sockets]
|
||||
for sock in sockets:
|
||||
sock.close()
|
||||
volumes = [n + '-data' for n in nodes]
|
||||
endpoints, ledger = [], []
|
||||
result = {'runid': runid, 'version': version, 'image': IMAGES[version],
|
||||
'image_id': image_info['Id'], 'platform': image_info['Os']+'/'+image_info['Architecture'],
|
||||
'nodes': 4,
|
||||
'drives': 4, 'filesystem': 'Linux tmpfs named volumes, held mounted across server restarts',
|
||||
'drive_bytes': 268435456, 'phases': [], 'status': 'RUNNING'}
|
||||
if version == 'current':
|
||||
result['binary_sha256'] = hashlib.sha256(CURRENT_BINARY.read_bytes()).hexdigest()
|
||||
result['source_revision'] = SOURCE_REVISION
|
||||
bucket = 'canary-' + uuid.uuid4().hex[:10]
|
||||
|
||||
def save(event=None):
|
||||
if event is not None:
|
||||
result['phases'].append(event)
|
||||
compact = {k: (len(v) if k in ('transient_errors', 'errors') else v) for k, v in event.items()}
|
||||
print(json.dumps({'version': version, **compact}), flush=True)
|
||||
(out / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
(out / 'acknowledged.json').write_text(json.dumps(ledger, indent=2) + '\n')
|
||||
|
||||
def parallel(fn, values):
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
return list(pool.map(fn, values))
|
||||
|
||||
def client(i, deadline):
|
||||
timeout = remaining(deadline)
|
||||
return boto3.client('s3', endpoint_url=endpoints[i], aws_access_key_id=user,
|
||||
aws_secret_access_key=password, region_name='us-east-1',
|
||||
config=Config(proxies={}, signature_version='s3v4', s3={'addressing_style': 'path'},
|
||||
retries={'total_max_attempts': 1}, connect_timeout=timeout,
|
||||
read_timeout=timeout, request_checksum_calculation='when_required',
|
||||
response_checksum_validation='when_required'))
|
||||
|
||||
def admin_gate(origin, phase):
|
||||
def check(deadline):
|
||||
first = [None] * 4
|
||||
while True:
|
||||
states = []
|
||||
for i in range(4):
|
||||
try:
|
||||
p = subprocess.run([str(MCLI), '--config-dir', str(out / 'mcli'), '--json',
|
||||
'admin', 'info', f'n{i}'], env=env, capture_output=True,
|
||||
text=True, timeout=min(5, remaining(deadline)))
|
||||
info = json.loads(p.stdout)['info']
|
||||
servers = info['servers']
|
||||
ok = len(servers) == 4 and all(s.get('state') == 'online' and s.get('drives')
|
||||
and all(d.get('state') == 'ok' for d in s['drives'])
|
||||
and all(v == 'online' for v in s.get('network', {}).values()) for s in servers)
|
||||
if ok:
|
||||
(out / f'{phase}-admin-{i}.json').write_text(json.dumps(info, indent=2) + '\n')
|
||||
if first[i] is None:
|
||||
first[i] = round(time.monotonic() - origin, 3)
|
||||
states.append(ok)
|
||||
except (Exception,):
|
||||
states.append(False)
|
||||
if all(states):
|
||||
event = {'phase': phase + '-admin', 'seconds_from_start': round(time.monotonic()-origin, 3),
|
||||
'first_online_by_coordinator': first}
|
||||
save(event)
|
||||
return time.monotonic()
|
||||
time.sleep(min(.25, remaining(deadline)))
|
||||
return bounded(90, check)
|
||||
|
||||
def canary(phase, origin, admin_time, setup=False):
|
||||
def check(deadline):
|
||||
started = time.monotonic()
|
||||
first_put, first_reads = [None] * 4, [None] * 4
|
||||
errors, attempt, setup_done = [], 0, not setup
|
||||
result['active_canary'] = {'phase': phase, 'errors': errors}
|
||||
while True:
|
||||
attempt += 1
|
||||
remaining(deadline)
|
||||
if not setup_done:
|
||||
try:
|
||||
try:
|
||||
client(0, deadline).create_bucket(Bucket=bucket)
|
||||
except Exception as e:
|
||||
if 'BucketAlreadyOwnedByYou' not in str(e):
|
||||
raise
|
||||
client(0, deadline).put_bucket_versioning(Bucket=bucket, VersioningConfiguration={'Status': 'Enabled'})
|
||||
setup_done = True
|
||||
except Exception as e:
|
||||
errors.append({'attempt': attempt, 'operation': 'setup', 'error': str(e)[:250]})
|
||||
time.sleep(min(.25, remaining(deadline)))
|
||||
continue
|
||||
acked = []
|
||||
for i in range(4):
|
||||
key = f'{phase}-attempt-{attempt}-node-{i}'
|
||||
payload = (key + '\n').encode() * 16384
|
||||
try:
|
||||
vid = client(i, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
|
||||
entry = {'phase': phase, 'key': key, 'version': vid, 'bytes': len(payload),
|
||||
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': i,
|
||||
'ack_seconds_from_start': round(time.monotonic()-origin, 3)}
|
||||
ledger.append(entry)
|
||||
save() # Persist every acknowledged write, including failed rounds.
|
||||
acked.append(entry)
|
||||
if first_put[i] is None:
|
||||
first_put[i] = round(time.monotonic()-admin_time, 3)
|
||||
except Exception as e:
|
||||
errors.append({'attempt': attempt, 'operation': 'put', 'node': i, 'error': str(e)[:250]})
|
||||
reads = 0
|
||||
for i in range(4):
|
||||
own_reads = 0
|
||||
for entry in acked:
|
||||
try:
|
||||
verify(i, entry, deadline)
|
||||
reads += 1
|
||||
own_reads += 1
|
||||
except Exception as e:
|
||||
errors.append({'attempt': attempt, 'operation': 'get', 'node': i,
|
||||
'key': entry['key'], 'error': str(e)[:250]})
|
||||
if own_reads == 4 and first_reads[i] is None:
|
||||
first_reads[i] = round(time.monotonic()-admin_time, 3)
|
||||
remaining(deadline)
|
||||
if len(acked) == 4 and reads == 16:
|
||||
result.pop('active_canary', None)
|
||||
save({'phase': phase + '-canary', 'attempts': attempt, 'puts': 4, 'gets': 16,
|
||||
'hard_deadline_seconds': 60, 'gate_seconds': round(time.monotonic()-started, 3),
|
||||
'seconds_from_start': round(time.monotonic()-origin, 3),
|
||||
'first_put_seconds_after_admin_by_coordinator': first_put,
|
||||
'first_four_reads_seconds_after_admin_by_coordinator': first_reads, 'transient_errors': errors})
|
||||
return time.monotonic()
|
||||
(out / f'{phase}-canary-errors.json').write_text(json.dumps(errors, indent=2) + '\n')
|
||||
if attempt == 1:
|
||||
print(json.dumps({'version': version, 'phase': phase, 'first_attempt_errors': errors}), flush=True)
|
||||
time.sleep(min(.25, remaining(deadline)))
|
||||
return bounded(60, check)
|
||||
|
||||
def verify(i, entry, deadline):
|
||||
got = client(i, deadline).get_object(Bucket=bucket, Key=entry['key'], VersionId=entry['version'])
|
||||
try:
|
||||
data = got['Body'].read()
|
||||
finally:
|
||||
got['Body'].close()
|
||||
assert len(data) == entry['bytes'] and hashlib.sha256(data).hexdigest() == entry['sha256'], entry['key']
|
||||
assert got.get('VersionId') == entry['version'], entry['key']
|
||||
remaining(deadline)
|
||||
|
||||
def readback(label, origin):
|
||||
def check(deadline):
|
||||
started = time.monotonic()
|
||||
errors = []
|
||||
for entry in ledger:
|
||||
for i in range(4):
|
||||
try:
|
||||
verify(i, entry, deadline)
|
||||
except Exception as e:
|
||||
errors.append({'key': entry['key'], 'node': i, 'error': str(e)[:250]})
|
||||
save({'phase': label, 'started_seconds_after_canary': round(started-origin, 3),
|
||||
'seconds_after_canary': round(time.monotonic()-origin, 3),
|
||||
'objects': len(ledger), 'reads': len(ledger)*4, 'errors': errors})
|
||||
return errors
|
||||
return bounded(60, check)
|
||||
|
||||
try:
|
||||
docker('network', 'create', runid)
|
||||
for volume in volumes:
|
||||
docker('volume', 'create', '--driver', 'local', '--opt', 'type=tmpfs',
|
||||
'--opt', 'device=tmpfs', '--opt', 'o=size=256m', volume)
|
||||
mounts = [arg for i, v in enumerate(volumes) for arg in ('--mount', f'type=volume,source={v},target=/keep/{i}')]
|
||||
docker('run', '-d', '--pull=never', '--network', 'none', '--name', runid + '-keeper',
|
||||
*mounts, '--entrypoint', 'sleep', 'alpine:3.23', '1800')
|
||||
urls = [f'http://{n}:9000/data' for n in nodes]
|
||||
def create(i):
|
||||
extra = []
|
||||
if version == 'current':
|
||||
extra = ['--mount', f'type=bind,source={CURRENT_BINARY},target=/lab/silo,readonly',
|
||||
'--entrypoint', '/lab/silo']
|
||||
docker('create', '--pull=never', '--name', nodes[i], '--network', runid,
|
||||
'--hostname', nodes[i], '--cpus', '2', '--memory', '3g', '--env-file', str(envfile),
|
||||
'--mount', f'type=volume,source={volumes[i]},target=/data',
|
||||
'-p', f'127.0.0.1:{ports[i]}:9000', *extra, image_info['Id'], 'server', '--address', ':9000',
|
||||
'--console-address', ':9001', *urls)
|
||||
parallel(create, range(4))
|
||||
start = time.monotonic()
|
||||
parallel(lambda n: docker('start', n), nodes)
|
||||
for i, n in enumerate(nodes):
|
||||
endpoint = 'http://' + docker('port', n, '9000/tcp').stdout.strip()
|
||||
endpoints.append(endpoint)
|
||||
env[f'MC_HOST_n{i}'] = endpoint.replace('http://', f'http://{user}:{password}@')
|
||||
admin = admin_gate(start, 'startup')
|
||||
canary('startup', start, admin, setup=True)
|
||||
parallel(lambda n: docker('stop', '-t', '10', n), nodes)
|
||||
start = time.monotonic()
|
||||
parallel(lambda n: docker('start', n), nodes)
|
||||
admin = admin_gate(start, 'full-restart')
|
||||
gate = canary('full-restart', start, admin)
|
||||
read_errors = []
|
||||
for after in (15, 30, 60):
|
||||
time.sleep(max(0, gate + after - time.monotonic()))
|
||||
read_errors.extend(readback(f'readback-{after}s', gate))
|
||||
assert not read_errors, 'acknowledged object readback failure; see result.json'
|
||||
docker('stop', '-t', '10', nodes[3])
|
||||
def outage(deadline):
|
||||
verify(0, ledger[0], deadline)
|
||||
payload = b'acknowledged with one Linux node offline' * 16384
|
||||
key = 'one-node-outage'
|
||||
vid = client(0, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
|
||||
entry = {'phase': 'outage', 'key': key, 'version': vid, 'bytes': len(payload),
|
||||
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': 0}
|
||||
ledger.append(entry)
|
||||
save()
|
||||
for i in range(3):
|
||||
verify(i, entry, deadline)
|
||||
save({'phase': 'one-node-outage', 'existing_read': True, 'put': True, 'readers': 3})
|
||||
bounded(60, outage)
|
||||
start = time.monotonic()
|
||||
docker('start', nodes[3])
|
||||
admin = admin_gate(start, 'rejoin')
|
||||
gate = canary('rejoin', start, admin)
|
||||
assert not readback('final-readback', gate)
|
||||
result['status'] = 'PASS'
|
||||
except BaseException as e:
|
||||
result['status'] = 'FAIL'
|
||||
result['error'] = f'{type(e).__name__}: {e}'
|
||||
raise
|
||||
finally:
|
||||
save()
|
||||
for n in nodes:
|
||||
log = docker('logs', n, check=False)
|
||||
(out / (n + '.log')).write_text(log.stdout + log.stderr)
|
||||
docker('rm', '-f', n, check=False)
|
||||
docker('rm', '-f', runid + '-keeper', check=False)
|
||||
for v in volumes:
|
||||
docker('volume', 'rm', v, check=False)
|
||||
docker('network', 'rm', runid, check=False)
|
||||
envfile.unlink(missing_ok=True)
|
||||
print(json.dumps({'version': version, 'status': result['status'], 'evidence': str(out)}), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('versions', nargs='+', choices=list(IMAGES))
|
||||
parser.add_argument('--output', type=Path, required=True, help='directory for retained evidence')
|
||||
parser.add_argument('--mcli', default=shutil.which('mcli'), help='native mcli executable')
|
||||
parser.add_argument('--current-binary', type=Path, help='Linux binary matching the Docker architecture')
|
||||
parser.add_argument('--source-revision', help='Git revision of --current-binary')
|
||||
parser.add_argument('--current-image', default=IMAGES['current'], help='cached Linux base image for current binary')
|
||||
args = parser.parse_args()
|
||||
if not args.mcli or not Path(args.mcli).is_file():
|
||||
parser.error('--mcli must point to an executable file')
|
||||
if 'current' in args.versions and (not args.current_binary or not args.current_binary.is_file()):
|
||||
parser.error('current requires --current-binary')
|
||||
ROOT = args.output.resolve()
|
||||
ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
MCLI = Path(args.mcli).resolve()
|
||||
CURRENT_BINARY = args.current_binary.resolve() if args.current_binary else None
|
||||
SOURCE_REVISION = args.source_revision
|
||||
IMAGES['current'] = args.current_image
|
||||
for version in args.versions:
|
||||
run(version)
|
||||
@@ -0,0 +1,130 @@
|
||||
# Remaining release correctness work, 2026-09-11
|
||||
|
||||
This records the three work items agreed after the branch/PR consolidation:
|
||||
OIDC #154, Linux restart/readback #116, and the related multi-pool defects
|
||||
#133/#144. The maintained target is SILO with the PGSTY Console, mcli and
|
||||
silo-pkg dependencies in the repository's current `go.mod`.
|
||||
|
||||
## Multi-pool writes and conditional deletion (#133, #144)
|
||||
|
||||
The pools layer now holds its object write lock across PUT and multipart
|
||||
completion, as it already does for metadata updates and DELETE. Multipart
|
||||
completion acquires the object lock before the upload lock. Queued healing and
|
||||
drive healing use this namespace too; healing under a caller-owned lock retains
|
||||
that lock's cancellation context. The destination-pool allocation policy is
|
||||
unchanged.
|
||||
|
||||
Trusted replica writes resolve the addressed version in every pool, including
|
||||
draining and rebalancing pools. Retention, legal hold and tags retain their
|
||||
independent ordering timestamps. A timestamp-only removal survives a stale
|
||||
retransmit. Completion resolves the version persisted in the upload, including
|
||||
the null version, rather than taking a later latest version's metadata.
|
||||
|
||||
Successful replica replacement retires competing copies of that exact version,
|
||||
so an equal-ModTime copy in an earlier pool cannot shadow the reconciled result.
|
||||
Metadata updates evaluate their callback once against the merged version and
|
||||
update all its copies. Replica metadata COPY rechecks ordering under the lock.
|
||||
Cleanup failures propagate; a replacement may already have committed when
|
||||
cleanup fails, and a retry can finish cleanup. Data movement keeps ownership of
|
||||
its source cleanup.
|
||||
|
||||
Conditional DELETE evaluates its precondition against the logical latest or
|
||||
explicitly addressed version. It checks all pools before mutation, removes
|
||||
secondary copies before the authoritative one, and returns cleanup errors.
|
||||
Versioned DELETE without a version ID creates a delete marker and preserves
|
||||
version history. Retention and replication callbacks evaluate the reconciled
|
||||
logical version. The existing all-pool delete helper also propagates errors
|
||||
from non-first pools.
|
||||
|
||||
The deterministic two-pool, 32-drive fixtures cover version selection,
|
||||
duplicate removal, delete failure propagation, PUT/DELETE and completion/DELETE
|
||||
interleavings, independent lock winners, draining/rebalancing owners, null
|
||||
versions, metadata COPY, metadata/healing serialization and cleanup retry.
|
||||
The original branch reproduced the wrong-version lookup, surviving duplicate,
|
||||
suppressed delete error, PUT/DELETE race, and PUT/completion lock-state failures
|
||||
before the fixes were applied.
|
||||
|
||||
Validation completed locally: the full `cmd` suite (255.720 s), all `internal`
|
||||
tests, `go vet ./...`, generated-file checks and the branding/entrypoint checks.
|
||||
Focused race checks cover the pooled interleavings, SSE-C lock regressions,
|
||||
conditional deletion, access-tier movement and TLS defaults. The two-pool and
|
||||
related replica/delete/movement tests also passed as a Linux/arm64 test binary
|
||||
in an isolated container with an 8 GiB `/tmp` tmpfs. Its initial 1 GiB tmpfs
|
||||
was insufficient for the existing single-drive test fixtures' free-space guard.
|
||||
|
||||
## Linux restart/readback (#116)
|
||||
|
||||
The [runner](issue-116/run-linux.py) creates four Linux/arm64 server containers
|
||||
on one Docker Desktop Linux VM, with separate network identities and one drive
|
||||
per node (EC 2+2). A holder container keeps four 256 MiB Linux tmpfs named
|
||||
volumes mounted across full server stops. Docker's ordinary filesystem had
|
||||
only 3.8 GiB free out of 2 TiB and correctly hit the server's free-space limit;
|
||||
the test uses the separate tmpfs filesystems without changing that threshold.
|
||||
This is the four-containers-on-one-Linux-host option explicitly accepted in
|
||||
the [#116 V1 plan](https://github.com/pgsty/silo/issues/116).
|
||||
|
||||
This covers process/container restart and TCP peer reconnection on one Linux
|
||||
host. It does not establish independent-host, host-reboot or physical-media
|
||||
durability. Every resource created by the runner is removed in its cleanup.
|
||||
The [retained summary](issue-116/evidence-20260911.json) records image identities,
|
||||
per-coordinator observations and readback counts.
|
||||
|
||||
| Binary | Full restart to all admin views online | Admin gate to complete 4 PUT / 16 GET round | Timed readbacks | Final readback |
|
||||
|---|---:|---:|---:|---:|
|
||||
| 0806 release | 2.084 s | 14.449 s | 660 / 660 | 240 / 240 |
|
||||
| 0903 release | 2.284 s | 0.191 s | 96 / 96 | 52 / 52 |
|
||||
| Current fix candidate | 2.461 s | 14.489 s | 1356 / 1356 | 904 / 904 |
|
||||
|
||||
Each canary has one hard 60-second deadline covering setup, requests, response
|
||||
body reads and sleeps, with SDK retries disabled. Every acknowledged PUT uses
|
||||
a unique versioned key and immediately records its VersionId, size and SHA-256.
|
||||
The timed checks reread these same objects through all four coordinators at
|
||||
15, 30 and 60 seconds **after the data canary succeeds**. They do not replace
|
||||
early acknowledgements with later writes. The final check also includes writes
|
||||
made during the one-node outage and subsequent rejoin canary.
|
||||
|
||||
All three runs passed the bounded canary, scheduled readbacks, one-node-outage
|
||||
read/write and rejoin readback. The candidate also showed a rejoin window:
|
||||
admin online at 2.490 s, complete data canary 13.977 s later. These are individual
|
||||
observations, not a latency guarantee or a comparison proving one version
|
||||
faster. Admin/health readiness still must be followed by a data-path check.
|
||||
|
||||
To repeat with cached images and a native mcli executable:
|
||||
|
||||
```sh
|
||||
uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \
|
||||
0806 0903 --output /tmp/silo-linux-acceptance --mcli /path/to/mcli
|
||||
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -tags kqueue -o /tmp/silo-linux .
|
||||
uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \
|
||||
current --output /tmp/silo-linux-candidate --mcli /path/to/mcli \
|
||||
--current-binary /tmp/silo-linux --source-revision "$(git rev-parse HEAD)"
|
||||
```
|
||||
|
||||
The runner uses `--pull=never`; preload the two release images, `alpine:3.23`,
|
||||
and the Linux base image selected with `--current-image`. Match the candidate
|
||||
binary architecture to that image. Detailed local evidence, including the
|
||||
acknowledgement ledger and full logs, remains in the requested output directory.
|
||||
|
||||
## OIDC (#154)
|
||||
|
||||
The TLS implementation fix was already merged in `48e1846525cc`: transports
|
||||
honor Go's key-exchange defaults, so `GODEBUG=tlsmlkem=0` can opt out of ML-KEM
|
||||
for an ingress that rejects it. It does not disable certificate validation or
|
||||
force an automatic protocol downgrade. See the existing
|
||||
[Go 1.27 investigation](go127-stack.md) and [issue investigation](issue-154.md).
|
||||
|
||||
A Linux build of main `b32f2d9dd01a` passed fresh isolated fixture checks:
|
||||
|
||||
- ML-KEM-intolerant IdP with `tlsmlkem=0`: discovery, IAM, Console login (204),
|
||||
authenticated bucket listing (200).
|
||||
- Normal TLS 1.3 IdP without that override: the same complete login chain.
|
||||
- Add OIDC through the real administration API with the compatibility setting.
|
||||
- Invalid JWT signature and audience: no session cookie and bucket access 403.
|
||||
- Untrusted CA: discovery fails and cluster readiness remains 503.
|
||||
|
||||
The affected customer's discovery URL and ingress configuration are still
|
||||
unavailable. This establishes the supported local fix and its negative
|
||||
controls, not the root cause or recovery of that hidden deployment. #154 stays
|
||||
open for an affected-environment retest; no release date or published artifact
|
||||
is implied by these checks.
|
||||
Reference in New Issue
Block a user