mirror of
https://github.com/pgsty/minio.git
synced 2026-10-30 20:41:03 +03:00
test: record Linux restart and OIDC release acceptance
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
@@ -0,0 +1,534 @@
|
|||||||
|
{
|
||||||
|
"date": "2026-09-11",
|
||||||
|
"runs": [
|
||||||
|
{
|
||||||
|
"runid": "silo-v1-0806-9c53f14c",
|
||||||
|
"version": "0806",
|
||||||
|
"image": "sha256:29a498b24669cae1fed11c1a2fb2b3d73c68829a0a9c0b14e71b386671d38fac",
|
||||||
|
"nodes": 4,
|
||||||
|
"drives": 4,
|
||||||
|
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||||
|
"drive_bytes": 268435456,
|
||||||
|
"status": "PASS",
|
||||||
|
"phases": [
|
||||||
|
{
|
||||||
|
"phase": "startup-admin",
|
||||||
|
"seconds_from_start": 4.157,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
4.037,
|
||||||
|
4.078,
|
||||||
|
4.118,
|
||||||
|
3.737
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "startup-canary",
|
||||||
|
"attempts": 1,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 0.239,
|
||||||
|
"seconds_from_start": 4.396,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.096,
|
||||||
|
0.116,
|
||||||
|
0.127,
|
||||||
|
0.138
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
0.162,
|
||||||
|
0.194,
|
||||||
|
0.217,
|
||||||
|
0.239
|
||||||
|
],
|
||||||
|
"transient_error_count": 0,
|
||||||
|
"first_transient_errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "full-restart-admin",
|
||||||
|
"seconds_from_start": 2.084,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
1.937,
|
||||||
|
1.976,
|
||||||
|
2.016,
|
||||||
|
1.643
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "full-restart-canary",
|
||||||
|
"attempts": 45,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 14.449,
|
||||||
|
"seconds_from_start": 16.534,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
14.315,
|
||||||
|
13.573,
|
||||||
|
13.953,
|
||||||
|
0.05
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
14.373,
|
||||||
|
14.396,
|
||||||
|
14.426,
|
||||||
|
14.449
|
||||||
|
],
|
||||||
|
"transient_error_count": 129,
|
||||||
|
"first_transient_errors": [
|
||||||
|
{
|
||||||
|
"attempt": 1,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 0,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 1,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 1,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 1,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 2,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 2,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 0,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-15s",
|
||||||
|
"seconds_after_canary": 17.596,
|
||||||
|
"objects": 55,
|
||||||
|
"reads": 220,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-30s",
|
||||||
|
"seconds_after_canary": 31.357,
|
||||||
|
"objects": 55,
|
||||||
|
"reads": 220,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-60s",
|
||||||
|
"seconds_after_canary": 61.566,
|
||||||
|
"objects": 55,
|
||||||
|
"reads": 220,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "one-node-outage",
|
||||||
|
"existing_read": true,
|
||||||
|
"put": true,
|
||||||
|
"readers": 3
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "rejoin-admin",
|
||||||
|
"seconds_from_start": 1.382,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
1.27,
|
||||||
|
1.307,
|
||||||
|
1.343,
|
||||||
|
1.382
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "rejoin-canary",
|
||||||
|
"attempts": 1,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 0.158,
|
||||||
|
"seconds_from_start": 1.541,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.013,
|
||||||
|
0.029,
|
||||||
|
0.039,
|
||||||
|
0.05
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
0.077,
|
||||||
|
0.102,
|
||||||
|
0.132,
|
||||||
|
0.158
|
||||||
|
],
|
||||||
|
"transient_error_count": 0,
|
||||||
|
"first_transient_errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "final-readback",
|
||||||
|
"seconds_after_canary": 1.71,
|
||||||
|
"objects": 60,
|
||||||
|
"reads": 240,
|
||||||
|
"errors": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"acknowledged_objects": 60,
|
||||||
|
"version_ids_recorded": true,
|
||||||
|
"sha256_recorded": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"runid": "silo-v1-0903-a457573c",
|
||||||
|
"version": "0903",
|
||||||
|
"image": "sha256:b616a0cf8cb281e7e6bb3c9b1fb53875b4016a2878223925541c18f82d6c5ca3",
|
||||||
|
"nodes": 4,
|
||||||
|
"drives": 4,
|
||||||
|
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||||
|
"drive_bytes": 268435456,
|
||||||
|
"status": "PASS",
|
||||||
|
"phases": [
|
||||||
|
{
|
||||||
|
"phase": "startup-admin",
|
||||||
|
"seconds_from_start": 4.271,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
4.158,
|
||||||
|
4.194,
|
||||||
|
3.827,
|
||||||
|
3.867
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "startup-canary",
|
||||||
|
"attempts": 1,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 0.2,
|
||||||
|
"seconds_from_start": 4.471,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.054,
|
||||||
|
0.068,
|
||||||
|
0.081,
|
||||||
|
0.092
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
0.118,
|
||||||
|
0.146,
|
||||||
|
0.167,
|
||||||
|
0.2
|
||||||
|
],
|
||||||
|
"transient_error_count": 0,
|
||||||
|
"first_transient_errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "full-restart-admin",
|
||||||
|
"seconds_from_start": 2.284,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
2.143,
|
||||||
|
2.193,
|
||||||
|
0.849,
|
||||||
|
0.902
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "full-restart-canary",
|
||||||
|
"attempts": 1,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 0.191,
|
||||||
|
"seconds_from_start": 2.476,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.016,
|
||||||
|
0.029,
|
||||||
|
0.044,
|
||||||
|
0.057
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
0.093,
|
||||||
|
0.124,
|
||||||
|
0.158,
|
||||||
|
0.192
|
||||||
|
],
|
||||||
|
"transient_error_count": 0,
|
||||||
|
"first_transient_errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-15s",
|
||||||
|
"seconds_after_canary": 15.235,
|
||||||
|
"objects": 8,
|
||||||
|
"reads": 32,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-30s",
|
||||||
|
"seconds_after_canary": 30.356,
|
||||||
|
"objects": 8,
|
||||||
|
"reads": 32,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-60s",
|
||||||
|
"seconds_after_canary": 60.192,
|
||||||
|
"objects": 8,
|
||||||
|
"reads": 32,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "one-node-outage",
|
||||||
|
"existing_read": true,
|
||||||
|
"put": true,
|
||||||
|
"readers": 3
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "rejoin-admin",
|
||||||
|
"seconds_from_start": 0.903,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
0.791,
|
||||||
|
0.829,
|
||||||
|
0.865,
|
||||||
|
0.903
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "rejoin-canary",
|
||||||
|
"attempts": 1,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 0.141,
|
||||||
|
"seconds_from_start": 1.044,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.012,
|
||||||
|
0.022,
|
||||||
|
0.035,
|
||||||
|
0.045
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
0.069,
|
||||||
|
0.096,
|
||||||
|
0.12,
|
||||||
|
0.141
|
||||||
|
],
|
||||||
|
"transient_error_count": 0,
|
||||||
|
"first_transient_errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "final-readback",
|
||||||
|
"seconds_after_canary": 0.301,
|
||||||
|
"objects": 13,
|
||||||
|
"reads": 52,
|
||||||
|
"errors": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"acknowledged_objects": 13,
|
||||||
|
"version_ids_recorded": true,
|
||||||
|
"sha256_recorded": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"runid": "silo-v1-current-6a14eb18",
|
||||||
|
"version": "current",
|
||||||
|
"image": "pgsty/d12a:build",
|
||||||
|
"image_id": "sha256:307af7711e2e04ab75759cb42a1eef45c43c4404894c0e30dd19f742b107b922",
|
||||||
|
"platform": "linux/arm64",
|
||||||
|
"nodes": 4,
|
||||||
|
"drives": 4,
|
||||||
|
"filesystem": "Linux tmpfs named volumes, held mounted across server restarts",
|
||||||
|
"drive_bytes": 268435456,
|
||||||
|
"status": "PASS",
|
||||||
|
"binary_sha256": "1e4cd7b78ecfa1b0cf28f1961d48a220a712c6be1fd3a01b60ee99aec62f04a4",
|
||||||
|
"source_revision": "b32f2d9dd01a383a9991d34ffe248063463a031d+pool-consistency",
|
||||||
|
"phases": [
|
||||||
|
{
|
||||||
|
"phase": "startup-admin",
|
||||||
|
"seconds_from_start": 7.374,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
7.207,
|
||||||
|
6.677,
|
||||||
|
6.74,
|
||||||
|
6.8
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "startup-canary",
|
||||||
|
"attempts": 1,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 0.479,
|
||||||
|
"seconds_from_start": 7.854,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.266,
|
||||||
|
0.279,
|
||||||
|
0.292,
|
||||||
|
0.305
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
0.336,
|
||||||
|
0.378,
|
||||||
|
0.431,
|
||||||
|
0.479
|
||||||
|
],
|
||||||
|
"transient_error_count": 0,
|
||||||
|
"first_transient_errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "full-restart-admin",
|
||||||
|
"seconds_from_start": 2.461,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
2.264,
|
||||||
|
1.414,
|
||||||
|
2.396,
|
||||||
|
2.461
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "full-restart-canary",
|
||||||
|
"attempts": 36,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 14.489,
|
||||||
|
"seconds_from_start": 16.951,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.013,
|
||||||
|
0.025,
|
||||||
|
14.301,
|
||||||
|
0.046
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
14.358,
|
||||||
|
14.394,
|
||||||
|
14.435,
|
||||||
|
14.489
|
||||||
|
],
|
||||||
|
"transient_error_count": 104,
|
||||||
|
"first_transient_errors": [
|
||||||
|
{
|
||||||
|
"attempt": 1,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 2,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 1,
|
||||||
|
"operation": "get",
|
||||||
|
"node": 2,
|
||||||
|
"key": "full-restart-attempt-1-node-0",
|
||||||
|
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 1,
|
||||||
|
"operation": "get",
|
||||||
|
"node": 2,
|
||||||
|
"key": "full-restart-attempt-1-node-3",
|
||||||
|
"error": "An error occurred (SlowDownRead) when calling the GetObject operation (reached max retries: 0): Resource requested is unreadable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 2,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 2,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-15s",
|
||||||
|
"started_seconds_after_canary": 15.0,
|
||||||
|
"seconds_after_canary": 22.006,
|
||||||
|
"objects": 113,
|
||||||
|
"reads": 452,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-30s",
|
||||||
|
"started_seconds_after_canary": 30.003,
|
||||||
|
"seconds_after_canary": 36.571,
|
||||||
|
"objects": 113,
|
||||||
|
"reads": 452,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "readback-60s",
|
||||||
|
"started_seconds_after_canary": 60.0,
|
||||||
|
"seconds_after_canary": 65.743,
|
||||||
|
"objects": 113,
|
||||||
|
"reads": 452,
|
||||||
|
"errors": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "one-node-outage",
|
||||||
|
"existing_read": true,
|
||||||
|
"put": true,
|
||||||
|
"readers": 3
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "rejoin-admin",
|
||||||
|
"seconds_from_start": 2.49,
|
||||||
|
"first_online_by_coordinator": [
|
||||||
|
1.165,
|
||||||
|
2.417,
|
||||||
|
1.646,
|
||||||
|
2.087
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "rejoin-canary",
|
||||||
|
"attempts": 37,
|
||||||
|
"puts": 4,
|
||||||
|
"gets": 16,
|
||||||
|
"hard_deadline_seconds": 60,
|
||||||
|
"gate_seconds": 13.977,
|
||||||
|
"seconds_from_start": 16.468,
|
||||||
|
"first_put_seconds_after_admin_by_coordinator": [
|
||||||
|
0.013,
|
||||||
|
0.024,
|
||||||
|
0.035,
|
||||||
|
13.889
|
||||||
|
],
|
||||||
|
"first_four_reads_seconds_after_admin_by_coordinator": [
|
||||||
|
13.909,
|
||||||
|
13.927,
|
||||||
|
13.953,
|
||||||
|
13.977
|
||||||
|
],
|
||||||
|
"transient_error_count": 36,
|
||||||
|
"first_transient_errors": [
|
||||||
|
{
|
||||||
|
"attempt": 1,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 3,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 2,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 3,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 3,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 3,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"attempt": 4,
|
||||||
|
"operation": "put",
|
||||||
|
"node": 3,
|
||||||
|
"error": "An error occurred (SlowDownWrite) when calling the PutObject operation (reached max retries: 0): Resource requested is unwritable, please reduce your request rate"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase": "final-readback",
|
||||||
|
"started_seconds_after_canary": 0.0,
|
||||||
|
"seconds_after_canary": 6.507,
|
||||||
|
"objects": 226,
|
||||||
|
"reads": 904,
|
||||||
|
"errors": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"acknowledged_objects": 226,
|
||||||
|
"version_ids_recorded": true,
|
||||||
|
"sha256_recorded": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,327 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Bounded four-container restart/readback acceptance for pgsty/silo#116."""
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import secrets
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
ROOT = None
|
||||||
|
MCLI = None
|
||||||
|
CURRENT_BINARY = None
|
||||||
|
SOURCE_REVISION = None
|
||||||
|
IMAGES = {
|
||||||
|
'0806': 'pgsty/silo:RELEASE.2026-08-06T00-00-00Z',
|
||||||
|
'0903': 'pgsty/silo:RELEASE.2026-09-03T13-18-01Z',
|
||||||
|
'current': 'pgsty/d12a:build',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def docker(*args, timeout=40, check=True):
|
||||||
|
return subprocess.run(['docker', *args], check=check, capture_output=True, text=True, timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
|
class Deadline(BaseException):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def bounded(seconds, operation):
|
||||||
|
def expired(*_):
|
||||||
|
raise Deadline(f'hard deadline of {seconds}s exceeded')
|
||||||
|
old = signal.signal(signal.SIGALRM, expired)
|
||||||
|
signal.setitimer(signal.ITIMER_REAL, seconds)
|
||||||
|
try:
|
||||||
|
return operation(time.monotonic() + seconds)
|
||||||
|
finally:
|
||||||
|
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||||
|
signal.signal(signal.SIGALRM, old)
|
||||||
|
|
||||||
|
|
||||||
|
def remaining(deadline):
|
||||||
|
value = deadline - time.monotonic()
|
||||||
|
if value <= 0:
|
||||||
|
raise Deadline('absolute deadline exceeded')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def run(version):
|
||||||
|
image_info = json.loads(docker('image', 'inspect', IMAGES[version]).stdout)[0]
|
||||||
|
runid = f'silo-v1-{version}-{uuid.uuid4().hex[:8]}'
|
||||||
|
out = ROOT / runid
|
||||||
|
out.mkdir(mode=0o700)
|
||||||
|
user, password = 'local116', secrets.token_urlsafe(24)
|
||||||
|
envfile = out / 'credentials.env'
|
||||||
|
envfile.write_text(f'MINIO_ROOT_USER={user}\nMINIO_ROOT_PASSWORD={password}\nMINIO_CI_CD=1\nMINIO_BROWSER=off\nGOMAXPROCS=2\n')
|
||||||
|
envfile.chmod(0o600)
|
||||||
|
env = {k: v for k, v in os.environ.items() if not k.startswith(('MINIO_', 'SILO_', 'MC_'))
|
||||||
|
and k.lower() not in {'http_proxy', 'https_proxy', 'all_proxy', 'no_proxy'}}
|
||||||
|
nodes = [f'{runid}-n{i}' for i in range(4)]
|
||||||
|
sockets = [socket.socket() for _ in nodes]
|
||||||
|
for sock in sockets:
|
||||||
|
sock.bind(('127.0.0.1', 0))
|
||||||
|
ports = [sock.getsockname()[1] for sock in sockets]
|
||||||
|
for sock in sockets:
|
||||||
|
sock.close()
|
||||||
|
volumes = [n + '-data' for n in nodes]
|
||||||
|
endpoints, ledger = [], []
|
||||||
|
result = {'runid': runid, 'version': version, 'image': IMAGES[version],
|
||||||
|
'image_id': image_info['Id'], 'platform': image_info['Os']+'/'+image_info['Architecture'],
|
||||||
|
'nodes': 4,
|
||||||
|
'drives': 4, 'filesystem': 'Linux tmpfs named volumes, held mounted across server restarts',
|
||||||
|
'drive_bytes': 268435456, 'phases': [], 'status': 'RUNNING'}
|
||||||
|
if version == 'current':
|
||||||
|
result['binary_sha256'] = hashlib.sha256(CURRENT_BINARY.read_bytes()).hexdigest()
|
||||||
|
result['source_revision'] = SOURCE_REVISION
|
||||||
|
bucket = 'canary-' + uuid.uuid4().hex[:10]
|
||||||
|
|
||||||
|
def save(event=None):
|
||||||
|
if event is not None:
|
||||||
|
result['phases'].append(event)
|
||||||
|
compact = {k: (len(v) if k in ('transient_errors', 'errors') else v) for k, v in event.items()}
|
||||||
|
print(json.dumps({'version': version, **compact}), flush=True)
|
||||||
|
(out / 'result.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||||
|
(out / 'acknowledged.json').write_text(json.dumps(ledger, indent=2) + '\n')
|
||||||
|
|
||||||
|
def parallel(fn, values):
|
||||||
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||||
|
return list(pool.map(fn, values))
|
||||||
|
|
||||||
|
def client(i, deadline):
|
||||||
|
timeout = remaining(deadline)
|
||||||
|
return boto3.client('s3', endpoint_url=endpoints[i], aws_access_key_id=user,
|
||||||
|
aws_secret_access_key=password, region_name='us-east-1',
|
||||||
|
config=Config(proxies={}, signature_version='s3v4', s3={'addressing_style': 'path'},
|
||||||
|
retries={'total_max_attempts': 1}, connect_timeout=timeout,
|
||||||
|
read_timeout=timeout, request_checksum_calculation='when_required',
|
||||||
|
response_checksum_validation='when_required'))
|
||||||
|
|
||||||
|
def admin_gate(origin, phase):
|
||||||
|
def check(deadline):
|
||||||
|
first = [None] * 4
|
||||||
|
while True:
|
||||||
|
states = []
|
||||||
|
for i in range(4):
|
||||||
|
try:
|
||||||
|
p = subprocess.run([str(MCLI), '--config-dir', str(out / 'mcli'), '--json',
|
||||||
|
'admin', 'info', f'n{i}'], env=env, capture_output=True,
|
||||||
|
text=True, timeout=min(5, remaining(deadline)))
|
||||||
|
info = json.loads(p.stdout)['info']
|
||||||
|
servers = info['servers']
|
||||||
|
ok = len(servers) == 4 and all(s.get('state') == 'online' and s.get('drives')
|
||||||
|
and all(d.get('state') == 'ok' for d in s['drives'])
|
||||||
|
and all(v == 'online' for v in s.get('network', {}).values()) for s in servers)
|
||||||
|
if ok:
|
||||||
|
(out / f'{phase}-admin-{i}.json').write_text(json.dumps(info, indent=2) + '\n')
|
||||||
|
if first[i] is None:
|
||||||
|
first[i] = round(time.monotonic() - origin, 3)
|
||||||
|
states.append(ok)
|
||||||
|
except (Exception,):
|
||||||
|
states.append(False)
|
||||||
|
if all(states):
|
||||||
|
event = {'phase': phase + '-admin', 'seconds_from_start': round(time.monotonic()-origin, 3),
|
||||||
|
'first_online_by_coordinator': first}
|
||||||
|
save(event)
|
||||||
|
return time.monotonic()
|
||||||
|
time.sleep(min(.25, remaining(deadline)))
|
||||||
|
return bounded(90, check)
|
||||||
|
|
||||||
|
def canary(phase, origin, admin_time, setup=False):
|
||||||
|
def check(deadline):
|
||||||
|
started = time.monotonic()
|
||||||
|
first_put, first_reads = [None] * 4, [None] * 4
|
||||||
|
errors, attempt, setup_done = [], 0, not setup
|
||||||
|
result['active_canary'] = {'phase': phase, 'errors': errors}
|
||||||
|
while True:
|
||||||
|
attempt += 1
|
||||||
|
remaining(deadline)
|
||||||
|
if not setup_done:
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
client(0, deadline).create_bucket(Bucket=bucket)
|
||||||
|
except Exception as e:
|
||||||
|
if 'BucketAlreadyOwnedByYou' not in str(e):
|
||||||
|
raise
|
||||||
|
client(0, deadline).put_bucket_versioning(Bucket=bucket, VersioningConfiguration={'Status': 'Enabled'})
|
||||||
|
setup_done = True
|
||||||
|
except Exception as e:
|
||||||
|
errors.append({'attempt': attempt, 'operation': 'setup', 'error': str(e)[:250]})
|
||||||
|
time.sleep(min(.25, remaining(deadline)))
|
||||||
|
continue
|
||||||
|
acked = []
|
||||||
|
for i in range(4):
|
||||||
|
key = f'{phase}-attempt-{attempt}-node-{i}'
|
||||||
|
payload = (key + '\n').encode() * 16384
|
||||||
|
try:
|
||||||
|
vid = client(i, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
|
||||||
|
entry = {'phase': phase, 'key': key, 'version': vid, 'bytes': len(payload),
|
||||||
|
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': i,
|
||||||
|
'ack_seconds_from_start': round(time.monotonic()-origin, 3)}
|
||||||
|
ledger.append(entry)
|
||||||
|
save() # Persist every acknowledged write, including failed rounds.
|
||||||
|
acked.append(entry)
|
||||||
|
if first_put[i] is None:
|
||||||
|
first_put[i] = round(time.monotonic()-admin_time, 3)
|
||||||
|
except Exception as e:
|
||||||
|
errors.append({'attempt': attempt, 'operation': 'put', 'node': i, 'error': str(e)[:250]})
|
||||||
|
reads = 0
|
||||||
|
for i in range(4):
|
||||||
|
own_reads = 0
|
||||||
|
for entry in acked:
|
||||||
|
try:
|
||||||
|
verify(i, entry, deadline)
|
||||||
|
reads += 1
|
||||||
|
own_reads += 1
|
||||||
|
except Exception as e:
|
||||||
|
errors.append({'attempt': attempt, 'operation': 'get', 'node': i,
|
||||||
|
'key': entry['key'], 'error': str(e)[:250]})
|
||||||
|
if own_reads == 4 and first_reads[i] is None:
|
||||||
|
first_reads[i] = round(time.monotonic()-admin_time, 3)
|
||||||
|
remaining(deadline)
|
||||||
|
if len(acked) == 4 and reads == 16:
|
||||||
|
result.pop('active_canary', None)
|
||||||
|
save({'phase': phase + '-canary', 'attempts': attempt, 'puts': 4, 'gets': 16,
|
||||||
|
'hard_deadline_seconds': 60, 'gate_seconds': round(time.monotonic()-started, 3),
|
||||||
|
'seconds_from_start': round(time.monotonic()-origin, 3),
|
||||||
|
'first_put_seconds_after_admin_by_coordinator': first_put,
|
||||||
|
'first_four_reads_seconds_after_admin_by_coordinator': first_reads, 'transient_errors': errors})
|
||||||
|
return time.monotonic()
|
||||||
|
(out / f'{phase}-canary-errors.json').write_text(json.dumps(errors, indent=2) + '\n')
|
||||||
|
if attempt == 1:
|
||||||
|
print(json.dumps({'version': version, 'phase': phase, 'first_attempt_errors': errors}), flush=True)
|
||||||
|
time.sleep(min(.25, remaining(deadline)))
|
||||||
|
return bounded(60, check)
|
||||||
|
|
||||||
|
def verify(i, entry, deadline):
|
||||||
|
got = client(i, deadline).get_object(Bucket=bucket, Key=entry['key'], VersionId=entry['version'])
|
||||||
|
try:
|
||||||
|
data = got['Body'].read()
|
||||||
|
finally:
|
||||||
|
got['Body'].close()
|
||||||
|
assert len(data) == entry['bytes'] and hashlib.sha256(data).hexdigest() == entry['sha256'], entry['key']
|
||||||
|
assert got.get('VersionId') == entry['version'], entry['key']
|
||||||
|
remaining(deadline)
|
||||||
|
|
||||||
|
def readback(label, origin):
|
||||||
|
def check(deadline):
|
||||||
|
started = time.monotonic()
|
||||||
|
errors = []
|
||||||
|
for entry in ledger:
|
||||||
|
for i in range(4):
|
||||||
|
try:
|
||||||
|
verify(i, entry, deadline)
|
||||||
|
except Exception as e:
|
||||||
|
errors.append({'key': entry['key'], 'node': i, 'error': str(e)[:250]})
|
||||||
|
save({'phase': label, 'started_seconds_after_canary': round(started-origin, 3),
|
||||||
|
'seconds_after_canary': round(time.monotonic()-origin, 3),
|
||||||
|
'objects': len(ledger), 'reads': len(ledger)*4, 'errors': errors})
|
||||||
|
return errors
|
||||||
|
return bounded(60, check)
|
||||||
|
|
||||||
|
try:
|
||||||
|
docker('network', 'create', runid)
|
||||||
|
for volume in volumes:
|
||||||
|
docker('volume', 'create', '--driver', 'local', '--opt', 'type=tmpfs',
|
||||||
|
'--opt', 'device=tmpfs', '--opt', 'o=size=256m', volume)
|
||||||
|
mounts = [arg for i, v in enumerate(volumes) for arg in ('--mount', f'type=volume,source={v},target=/keep/{i}')]
|
||||||
|
docker('run', '-d', '--pull=never', '--network', 'none', '--name', runid + '-keeper',
|
||||||
|
*mounts, '--entrypoint', 'sleep', 'alpine:3.23', '1800')
|
||||||
|
urls = [f'http://{n}:9000/data' for n in nodes]
|
||||||
|
def create(i):
|
||||||
|
extra = []
|
||||||
|
if version == 'current':
|
||||||
|
extra = ['--mount', f'type=bind,source={CURRENT_BINARY},target=/lab/silo,readonly',
|
||||||
|
'--entrypoint', '/lab/silo']
|
||||||
|
docker('create', '--pull=never', '--name', nodes[i], '--network', runid,
|
||||||
|
'--hostname', nodes[i], '--cpus', '2', '--memory', '3g', '--env-file', str(envfile),
|
||||||
|
'--mount', f'type=volume,source={volumes[i]},target=/data',
|
||||||
|
'-p', f'127.0.0.1:{ports[i]}:9000', *extra, image_info['Id'], 'server', '--address', ':9000',
|
||||||
|
'--console-address', ':9001', *urls)
|
||||||
|
parallel(create, range(4))
|
||||||
|
start = time.monotonic()
|
||||||
|
parallel(lambda n: docker('start', n), nodes)
|
||||||
|
for i, n in enumerate(nodes):
|
||||||
|
endpoint = 'http://' + docker('port', n, '9000/tcp').stdout.strip()
|
||||||
|
endpoints.append(endpoint)
|
||||||
|
env[f'MC_HOST_n{i}'] = endpoint.replace('http://', f'http://{user}:{password}@')
|
||||||
|
admin = admin_gate(start, 'startup')
|
||||||
|
canary('startup', start, admin, setup=True)
|
||||||
|
parallel(lambda n: docker('stop', '-t', '10', n), nodes)
|
||||||
|
start = time.monotonic()
|
||||||
|
parallel(lambda n: docker('start', n), nodes)
|
||||||
|
admin = admin_gate(start, 'full-restart')
|
||||||
|
gate = canary('full-restart', start, admin)
|
||||||
|
read_errors = []
|
||||||
|
for after in (15, 30, 60):
|
||||||
|
time.sleep(max(0, gate + after - time.monotonic()))
|
||||||
|
read_errors.extend(readback(f'readback-{after}s', gate))
|
||||||
|
assert not read_errors, 'acknowledged object readback failure; see result.json'
|
||||||
|
docker('stop', '-t', '10', nodes[3])
|
||||||
|
def outage(deadline):
|
||||||
|
verify(0, ledger[0], deadline)
|
||||||
|
payload = b'acknowledged with one Linux node offline' * 16384
|
||||||
|
key = 'one-node-outage'
|
||||||
|
vid = client(0, deadline).put_object(Bucket=bucket, Key=key, Body=payload)['VersionId']
|
||||||
|
entry = {'phase': 'outage', 'key': key, 'version': vid, 'bytes': len(payload),
|
||||||
|
'sha256': hashlib.sha256(payload).hexdigest(), 'writer': 0}
|
||||||
|
ledger.append(entry)
|
||||||
|
save()
|
||||||
|
for i in range(3):
|
||||||
|
verify(i, entry, deadline)
|
||||||
|
save({'phase': 'one-node-outage', 'existing_read': True, 'put': True, 'readers': 3})
|
||||||
|
bounded(60, outage)
|
||||||
|
start = time.monotonic()
|
||||||
|
docker('start', nodes[3])
|
||||||
|
admin = admin_gate(start, 'rejoin')
|
||||||
|
gate = canary('rejoin', start, admin)
|
||||||
|
assert not readback('final-readback', gate)
|
||||||
|
result['status'] = 'PASS'
|
||||||
|
except BaseException as e:
|
||||||
|
result['status'] = 'FAIL'
|
||||||
|
result['error'] = f'{type(e).__name__}: {e}'
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
save()
|
||||||
|
for n in nodes:
|
||||||
|
log = docker('logs', n, check=False)
|
||||||
|
(out / (n + '.log')).write_text(log.stdout + log.stderr)
|
||||||
|
docker('rm', '-f', n, check=False)
|
||||||
|
docker('rm', '-f', runid + '-keeper', check=False)
|
||||||
|
for v in volumes:
|
||||||
|
docker('volume', 'rm', v, check=False)
|
||||||
|
docker('network', 'rm', runid, check=False)
|
||||||
|
envfile.unlink(missing_ok=True)
|
||||||
|
print(json.dumps({'version': version, 'status': result['status'], 'evidence': str(out)}), flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('versions', nargs='+', choices=list(IMAGES))
|
||||||
|
parser.add_argument('--output', type=Path, required=True, help='directory for retained evidence')
|
||||||
|
parser.add_argument('--mcli', default=shutil.which('mcli'), help='native mcli executable')
|
||||||
|
parser.add_argument('--current-binary', type=Path, help='Linux binary matching the Docker architecture')
|
||||||
|
parser.add_argument('--source-revision', help='Git revision of --current-binary')
|
||||||
|
parser.add_argument('--current-image', default=IMAGES['current'], help='cached Linux base image for current binary')
|
||||||
|
args = parser.parse_args()
|
||||||
|
if not args.mcli or not Path(args.mcli).is_file():
|
||||||
|
parser.error('--mcli must point to an executable file')
|
||||||
|
if 'current' in args.versions and (not args.current_binary or not args.current_binary.is_file()):
|
||||||
|
parser.error('current requires --current-binary')
|
||||||
|
ROOT = args.output.resolve()
|
||||||
|
ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
MCLI = Path(args.mcli).resolve()
|
||||||
|
CURRENT_BINARY = args.current_binary.resolve() if args.current_binary else None
|
||||||
|
SOURCE_REVISION = args.source_revision
|
||||||
|
IMAGES['current'] = args.current_image
|
||||||
|
for version in args.versions:
|
||||||
|
run(version)
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
# Remaining release correctness work, 2026-09-11
|
||||||
|
|
||||||
|
This records the three work items agreed after the branch/PR consolidation:
|
||||||
|
OIDC #154, Linux restart/readback #116, and the related multi-pool defects
|
||||||
|
#133/#144. The maintained target is SILO with the PGSTY Console, mcli and
|
||||||
|
silo-pkg dependencies in the repository's current `go.mod`.
|
||||||
|
|
||||||
|
## Multi-pool writes and conditional deletion (#133, #144)
|
||||||
|
|
||||||
|
The pools layer now holds its object write lock across PUT and multipart
|
||||||
|
completion, as it already does for metadata updates and DELETE. Multipart
|
||||||
|
completion acquires the object lock before the upload lock. Queued healing and
|
||||||
|
drive healing use this namespace too; healing under a caller-owned lock retains
|
||||||
|
that lock's cancellation context. The destination-pool allocation policy is
|
||||||
|
unchanged.
|
||||||
|
|
||||||
|
Trusted replica writes resolve the addressed version in every pool, including
|
||||||
|
draining and rebalancing pools. Retention, legal hold and tags retain their
|
||||||
|
independent ordering timestamps. A timestamp-only removal survives a stale
|
||||||
|
retransmit. Completion resolves the version persisted in the upload, including
|
||||||
|
the null version, rather than taking a later latest version's metadata.
|
||||||
|
|
||||||
|
Successful replica replacement retires competing copies of that exact version,
|
||||||
|
so an equal-ModTime copy in an earlier pool cannot shadow the reconciled result.
|
||||||
|
Metadata updates evaluate their callback once against the merged version and
|
||||||
|
update all its copies. Replica metadata COPY rechecks ordering under the lock.
|
||||||
|
Cleanup failures propagate; a replacement may already have committed when
|
||||||
|
cleanup fails, and a retry can finish cleanup. Data movement keeps ownership of
|
||||||
|
its source cleanup.
|
||||||
|
|
||||||
|
Conditional DELETE evaluates its precondition against the logical latest or
|
||||||
|
explicitly addressed version. It checks all pools before mutation, removes
|
||||||
|
secondary copies before the authoritative one, and returns cleanup errors.
|
||||||
|
Versioned DELETE without a version ID creates a delete marker and preserves
|
||||||
|
version history. Retention and replication callbacks evaluate the reconciled
|
||||||
|
logical version. The existing all-pool delete helper also propagates errors
|
||||||
|
from non-first pools.
|
||||||
|
|
||||||
|
The deterministic two-pool, 32-drive fixtures cover version selection,
|
||||||
|
duplicate removal, delete failure propagation, PUT/DELETE and completion/DELETE
|
||||||
|
interleavings, independent lock winners, draining/rebalancing owners, null
|
||||||
|
versions, metadata COPY, metadata/healing serialization and cleanup retry.
|
||||||
|
The original branch reproduced the wrong-version lookup, surviving duplicate,
|
||||||
|
suppressed delete error, PUT/DELETE race, and PUT/completion lock-state failures
|
||||||
|
before the fixes were applied.
|
||||||
|
|
||||||
|
Validation completed locally: the full `cmd` suite (255.720 s), all `internal`
|
||||||
|
tests, `go vet ./...`, generated-file checks and the branding/entrypoint checks.
|
||||||
|
Focused race checks cover the pooled interleavings, SSE-C lock regressions,
|
||||||
|
conditional deletion, access-tier movement and TLS defaults. The two-pool and
|
||||||
|
related replica/delete/movement tests also passed as a Linux/arm64 test binary
|
||||||
|
in an isolated container with an 8 GiB `/tmp` tmpfs. Its initial 1 GiB tmpfs
|
||||||
|
was insufficient for the existing single-drive test fixtures' free-space guard.
|
||||||
|
|
||||||
|
## Linux restart/readback (#116)
|
||||||
|
|
||||||
|
The [runner](issue-116/run-linux.py) creates four Linux/arm64 server containers
|
||||||
|
on one Docker Desktop Linux VM, with separate network identities and one drive
|
||||||
|
per node (EC 2+2). A holder container keeps four 256 MiB Linux tmpfs named
|
||||||
|
volumes mounted across full server stops. Docker's ordinary filesystem had
|
||||||
|
only 3.8 GiB free out of 2 TiB and correctly hit the server's free-space limit;
|
||||||
|
the test uses the separate tmpfs filesystems without changing that threshold.
|
||||||
|
This is the four-containers-on-one-Linux-host option explicitly accepted in
|
||||||
|
the [#116 V1 plan](https://github.com/pgsty/silo/issues/116).
|
||||||
|
|
||||||
|
This covers process/container restart and TCP peer reconnection on one Linux
|
||||||
|
host. It does not establish independent-host, host-reboot or physical-media
|
||||||
|
durability. Every resource created by the runner is removed in its cleanup.
|
||||||
|
The [retained summary](issue-116/evidence-20260911.json) records image identities,
|
||||||
|
per-coordinator observations and readback counts.
|
||||||
|
|
||||||
|
| Binary | Full restart to all admin views online | Admin gate to complete 4 PUT / 16 GET round | Timed readbacks | Final readback |
|
||||||
|
|---|---:|---:|---:|---:|
|
||||||
|
| 0806 release | 2.084 s | 14.449 s | 660 / 660 | 240 / 240 |
|
||||||
|
| 0903 release | 2.284 s | 0.191 s | 96 / 96 | 52 / 52 |
|
||||||
|
| Current fix candidate | 2.461 s | 14.489 s | 1356 / 1356 | 904 / 904 |
|
||||||
|
|
||||||
|
Each canary has one hard 60-second deadline covering setup, requests, response
|
||||||
|
body reads and sleeps, with SDK retries disabled. Every acknowledged PUT uses
|
||||||
|
a unique versioned key and immediately records its VersionId, size and SHA-256.
|
||||||
|
The timed checks reread these same objects through all four coordinators at
|
||||||
|
15, 30 and 60 seconds **after the data canary succeeds**. They do not replace
|
||||||
|
early acknowledgements with later writes. The final check also includes writes
|
||||||
|
made during the one-node outage and subsequent rejoin canary.
|
||||||
|
|
||||||
|
All three runs passed the bounded canary, scheduled readbacks, one-node-outage
|
||||||
|
read/write and rejoin readback. The candidate also showed a rejoin window:
|
||||||
|
admin online at 2.490 s, complete data canary 13.977 s later. These are individual
|
||||||
|
observations, not a latency guarantee or a comparison proving one version
|
||||||
|
faster. Admin/health readiness still must be followed by a data-path check.
|
||||||
|
|
||||||
|
To repeat with cached images and a native mcli executable:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \
|
||||||
|
0806 0903 --output /tmp/silo-linux-acceptance --mcli /path/to/mcli
|
||||||
|
|
||||||
|
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -tags kqueue -o /tmp/silo-linux .
|
||||||
|
uv run --with boto3==1.43.92 docs/investigations/issue-116/run-linux.py \
|
||||||
|
current --output /tmp/silo-linux-candidate --mcli /path/to/mcli \
|
||||||
|
--current-binary /tmp/silo-linux --source-revision "$(git rev-parse HEAD)"
|
||||||
|
```
|
||||||
|
|
||||||
|
The runner uses `--pull=never`; preload the two release images, `alpine:3.23`,
|
||||||
|
and the Linux base image selected with `--current-image`. Match the candidate
|
||||||
|
binary architecture to that image. Detailed local evidence, including the
|
||||||
|
acknowledgement ledger and full logs, remains in the requested output directory.
|
||||||
|
|
||||||
|
## OIDC (#154)
|
||||||
|
|
||||||
|
The TLS implementation fix was already merged in `48e1846525cc`: transports
|
||||||
|
honor Go's key-exchange defaults, so `GODEBUG=tlsmlkem=0` can opt out of ML-KEM
|
||||||
|
for an ingress that rejects it. It does not disable certificate validation or
|
||||||
|
force an automatic protocol downgrade. See the existing
|
||||||
|
[Go 1.27 investigation](go127-stack.md) and [issue investigation](issue-154.md).
|
||||||
|
|
||||||
|
A Linux build of main `b32f2d9dd01a` passed fresh isolated fixture checks:
|
||||||
|
|
||||||
|
- ML-KEM-intolerant IdP with `tlsmlkem=0`: discovery, IAM, Console login (204),
|
||||||
|
authenticated bucket listing (200).
|
||||||
|
- Normal TLS 1.3 IdP without that override: the same complete login chain.
|
||||||
|
- Add OIDC through the real administration API with the compatibility setting.
|
||||||
|
- Invalid JWT signature and audience: no session cookie and bucket access 403.
|
||||||
|
- Untrusted CA: discovery fails and cluster readiness remains 503.
|
||||||
|
|
||||||
|
The affected customer's discovery URL and ingress configuration are still
|
||||||
|
unavailable. This establishes the supported local fix and its negative
|
||||||
|
controls, not the root cause or recovery of that hidden deployment. #154 stays
|
||||||
|
open for an affected-environment retest; no release date or published artifact
|
||||||
|
is implied by these checks.
|
||||||
Reference in New Issue
Block a user