commit 2060155b8211dd9c8f798cab8e1ec282124a7631
parent 5b13de07a30dc98da2cdb720a536c1bba9975beb
Author: Joris Hartog <jorishartog@hotmail.com>
Date: Sat, 5 Sep 2026 19:30:41 +0200
Audit live recovery chain evidence
Diffstat:
4 files changed, 303 insertions(+), 4 deletions(-)
diff --git a/PLAN.md b/PLAN.md
@@ -18,13 +18,19 @@ test is nodig, maar sluit een live-soak gate niet automatisch.
## 2. Live recovery-bewijs
-- [ ] Analyseer de chain-database onder `~/.iuna` uitsluitend read-only.
-- [ ] Leg recovery-hoogtes, hashes, voorafgaande stall en eerstvolgende
+- [x] Analyseer de chain-database onder `~/.iuna` uitsluitend read-only.
+- [x] Leg recovery-hoogtes, hashes, voorafgaande stall en eerstvolgende
ticketblock vast zonder walletmateriaal of secrets te kopiëren.
-- [ ] Bepaal welke recovery-gates hiermee objectief gesloten kunnen worden.
-- [ ] Laat gates voor meerdere recovery-candidates open tenzij de live historie
+- [x] Bepaal welke recovery-gates hiermee objectief gesloten kunnen worden.
+- [x] Laat gates voor meerdere recovery-candidates open tenzij de live historie
of de procesniveau-partitietest dit daadwerkelijk bewijst.
+Resultaat: [de live-chain-audit](docs/live-recovery-evidence.md) vond 12
+recovery-blocks in 8 episodes en na iedere episode hervatte een ticketblock.
+De roadmap-gates blijven bewust open omdat de chainhistorie niet bewijst welke
+finalizers tijdens de stalls offline waren en geen concurrerende live
+recovery-candidates bevat.
+
## 3. Sync-bewijs
- [ ] Test een lege node die zonder handwerk vanaf genesis synchroniseert.
diff --git a/ROADMAP.md b/ROADMAP.md
@@ -19,6 +19,13 @@ The current goal is to operate the candidate without unplanned resets, collect l
- [ ] Post-activation network partitions have an implemented objective checkpoint-based recovery rule; live soak evidence is still required to close this gate.
- [ ] Recovery blocks restore liveness when selected finalizers disappear.
- [ ] Multiple recovery candidates converge safely.
+
+The [live recovery audit](docs/live-recovery-evidence.md) records 12 recovery
+blocks across 8 episodes, each followed by resumed ticket production. The
+recovery-liveness and multiple-candidate gates remain open pending correlated
+live node logs and live competing-candidate evidence; the accelerated
+process-level partition test is supporting test evidence, not a substitute for
+that soak evidence.
- [ ] Clock skew and future timestamp cases do not stall the network.
- [ ] Blinded commit/reveal flows survive partitions and delayed gossip.
- [ ] Mempool state remains sane across reorgs.
diff --git a/docs/live-recovery-evidence.md b/docs/live-recovery-evidence.md
@@ -0,0 +1,66 @@
+# Live Recovery Evidence
+
+This note records a read-only audit of the local mainnet-candidate chain
+snapshot. It does not contain wallet data, configuration, recovery phrases, or
+private keys.
+
+## Capture
+
+- Captured at: `2026-09-05T17:24:50Z`
+- Profile: `iuna-mainnet-candidate`
+- Height: `1210`
+- Tip: `441f59b34cac6df6253c7f6b2449369ab7c1c3065d065467022f6b52b067eeb8`
+- Genesis: `3d677cd7ced1c04d3a276cbee7ea38076e34ac65f18a2c9b8286a4872d986a9a`
+- Stable database-copy SHA-256:
+ `fcf26863853ea51e0a865f330a2292fa2bcac6dc1cfaceb33a0880347a0d168e`
+- Verification marker: valid under the current consensus ruleset
+
+The audit command refuses a source database with a WAL file, copies a stable
+source to a temporary database, decodes only that copy, and removes the copy and
+its SQLite companion files afterward:
+
+```sh
+cargo run --locked --bin iuna-chain-audit -- \
+ ~/.iuna/chain.sqlite3 \
+ --output release-evidence/live-chain-audit-2026-09-05.json
+```
+
+## Observed recovery history
+
+`parent gap` is the elapsed time from the preceding block to the recovery
+block. `ticket delay` is measured from that recovery block to the first later
+ticket block. Consecutive recovery rows therefore share the same eventual
+ticket where applicable.
+
+| Recovery height | Recovery hash | Parent gap (ms) | Next ticket height/rank | Next ticket hash | Ticket delay (blocks/ms) |
+| ---: | --- | ---: | ---: | --- | ---: |
+| 116 | `31d341ff29c24727ad9926850cc45401fa5f59378894f3d5b40980bc2e785495` | 5684830 | 117 / 0 | `5eafc133241dbc9235c522dab1659ba1b095cf9cede24e762ce1685209dff858` | 1 / 1457218 |
+| 119 | `beba42e65f49fb5bb369920e420d6cfa1d6c8e2cb4051846fd06d2eb4171dd85` | 3600160 | 120 / 0 | `18b2d2be9b3a9f920bf1731bbdcbe064456c219953bbf7d3225c4fedb4c8ca21` | 1 / 2521415 |
+| 134 | `02b9fd07eb39be179070e637c08621cf0342dfbcada20ca6a5b7ccceaedb9b53` | 3600608 | 135 / 0 | `cb2f106d309f6370ca9c3406c560d752f039c43a9493763b7cd4da09dfe5313c` | 1 / 182550 |
+| 150 | `bf0fb179d209d420af7651483d7ac393eadcfdb5580f4a70fa3f8fa870d777ec` | 34350614 | 153 / 1 | `8fc33d01d0b603632c4aeb325bbdc9b587b499297b514fc6f97ec45492cb203b` | 3 / 8402100 |
+| 151 | `1a6002adbfb2cecc95efe52707825dc62a032c9f1633c0fdcfcd7f0fe07544fd` | 3600714 | 153 / 1 | `8fc33d01d0b603632c4aeb325bbdc9b587b499297b514fc6f97ec45492cb203b` | 2 / 4801386 |
+| 152 | `f521b171aff6023a42e0081b4aa2000fd5cabfcc1549aed86cd2cf11d83b380e` | 3600275 | 153 / 1 | `8fc33d01d0b603632c4aeb325bbdc9b587b499297b514fc6f97ec45492cb203b` | 1 / 1201111 |
+| 189 | `7114c1a74d258c73fbe7f1d168c0ecdf628e9f73da529bb30fb3daa494a18e1e` | 3600028 | 190 / 1 | `49d1c9f7bb2aea87d117b9bd669b83839e215340c58d1b9bc297b99d7c5c3d83` | 1 / 1200000 |
+| 232 | `b39c4ce092ad0e7aa1a01f6377895e1113acc5df7942badc6a6b1363903e5690` | 3600946 | 233 / 0 | `0be7c59ae4aafe1e1ef604dbb7f472402701e477662ef30cdd46111c664fbbb0` | 1 / 1034783 |
+| 525 | `7290046cd2dbd18a7ca97ae1b708b92d4bd8bc0459cc206627b97916e965188e` | 3600273 | 528 / 0 | `a753f41a9fc77cc6226b49f06d51ea0e1b2ae5142b368acf8ac58590a48c7e09` | 3 / 16446423 |
+| 526 | `df42e45217fd9e40d9c81709b92b13d0e45a75daf88d3d94dbd2542ab7da4a98` | 11892983 | 528 / 0 | `a753f41a9fc77cc6226b49f06d51ea0e1b2ae5142b368acf8ac58590a48c7e09` | 2 / 4553440 |
+| 527 | `e423061e1075c70b439fb1378389733c8902dbabce79dacad4f4cd346d8c2e71` | 3600497 | 528 / 0 | `a753f41a9fc77cc6226b49f06d51ea0e1b2ae5142b368acf8ac58590a48c7e09` | 1 / 952943 |
+| 788 | `b46bd39f637cfb213d177081a47673da106740edace3f78ce6f0d0f5a971b0be` | 3959212 | 789 / 0 | `61f3921223b84336e6ad84451865b8a1adbca16484fd44f86dbc5cfd788d991a` | 1 / 1845873 |
+
+The 12 recovery blocks form 8 episodes. Every episode is followed by a normal
+ticket block, including the two episodes containing three consecutive recovery
+blocks. This is direct live-chain evidence that the recovery path has restored
+ticket production after observed stalls.
+
+## Gate decision
+
+This capture supports the recovery-liveness gate, but does not close it by
+itself: block history cannot prove that disappearance of the selected
+finalizers caused each stall. Correlated node logs or a controlled live soak are
+still required for that attribution.
+
+The multiple-recovery-candidate convergence gate also remains open. Consecutive
+recovery blocks are not evidence of competing candidates. The process-level
+3-3 partition test proves deterministic convergence in the accelerated e2e
+environment, while live soak evidence is still required for the roadmap's live
+gate.
diff --git a/src/bin/iuna-chain-audit.rs b/src/bin/iuna-chain-audit.rs
@@ -0,0 +1,220 @@
+use std::{
+ env, fs,
+ io::Read,
+ path::{Path, PathBuf},
+ process,
+ time::{SystemTime, UNIX_EPOCH},
+};
+
+use anyhow::{Context, Result, bail};
+use iuna::{
+ adapters::chain_store::SqliteChainStore,
+ domain::{Block, FinalizerMode},
+};
+use serde::Serialize;
+use sha2::{Digest, Sha256};
+
+#[derive(Serialize)]
+struct TicketResume {
+ height: u64,
+ hash: String,
+ rank: u32,
+ delay_blocks: u64,
+ delay_ms: u64,
+}
+
+#[derive(Serialize)]
+struct RecoveryEvidence {
+ height: u64,
+ hash: String,
+ miner: String,
+ timestamp_ms: u64,
+ gap_from_parent_ms: u64,
+ next_ticket: Option<TicketResume>,
+ immediate_ticket_resume: bool,
+}
+
+#[derive(Serialize)]
+struct AuditReport {
+ format: u8,
+ source: PathBuf,
+ captured_at_ms: u64,
+ database_copy_sha256: String,
+ verified_under_current_consensus_ruleset: bool,
+ profile_id: String,
+ height: u64,
+ tip_hash: String,
+ genesis_hash: String,
+ recovery_count: usize,
+ recoveries: Vec<RecoveryEvidence>,
+}
+
+fn main() -> Result<()> {
+ let mut args = env::args_os().skip(1);
+ let source = PathBuf::from(
+ args.next()
+ .context("usage: iuna-chain-audit CHAIN_DB [--output REPORT.json]")?,
+ );
+ let mut output = None;
+ while let Some(argument) = args.next() {
+ if argument == "--output" {
+ output = Some(PathBuf::from(args.next().context("missing --output path")?));
+ } else {
+ bail!("unknown argument: {}", argument.to_string_lossy());
+ }
+ }
+
+ let report = audit_snapshot(&source)?;
+ let json = serde_json::to_string_pretty(&report)? + "\n";
+ if let Some(output) = output {
+ fs::write(&output, json)
+ .with_context(|| format!("failed to write audit report {}", output.display()))?;
+ println!(
+ "wrote {} recovery events to {}",
+ report.recovery_count,
+ output.display()
+ );
+ } else {
+ print!("{json}");
+ }
+ Ok(())
+}
+
+fn audit_snapshot(source: &Path) -> Result<AuditReport> {
+ if !source.is_file() {
+ bail!("chain database does not exist: {}", source.display());
+ }
+ let companion = ["-wal", "-journal"]
+ .into_iter()
+ .map(|suffix| path_with_suffix(source, suffix))
+ .find(|path| path.exists());
+ if let Some(companion) = companion {
+ bail!(
+ "refusing a database with an active SQLite companion; stop the node or create a SQLite backup first: {}",
+ companion.display()
+ );
+ }
+
+ let before = fs::metadata(source).context("failed to inspect source database")?;
+ let temporary = env::temp_dir().join(format!(
+ "iuna-chain-audit-{}-{}.sqlite3",
+ process::id(),
+ now_ms()?
+ ));
+ fs::copy(source, &temporary).with_context(|| {
+ format!(
+ "failed to copy source database {} to {}",
+ source.display(),
+ temporary.display()
+ )
+ })?;
+ let after = fs::metadata(source).context("failed to re-inspect source database")?;
+ if before.len() != after.len() || before.modified().ok() != after.modified().ok() {
+ cleanup_temporary_database(&temporary);
+ bail!("source database changed while it was copied; retry on a stable snapshot");
+ }
+
+ let result = audit_copy(source, &temporary);
+ cleanup_temporary_database(&temporary);
+ result
+}
+
+fn audit_copy(source: &Path, copy: &Path) -> Result<AuditReport> {
+ let database_copy_sha256 = file_sha256(copy)?;
+ let loaded = SqliteChainStore::open(copy)?
+ .load_with_verification_status()?
+ .context("chain database contains no snapshot")?;
+ let verified_under_current_consensus_ruleset = loaded.revalidation_from_height.is_none();
+ let snapshot = loaded.snapshot;
+ let tip = snapshot.blocks.last().context("chain snapshot is empty")?;
+ let genesis = snapshot.blocks.first().context("chain snapshot is empty")?;
+ let recoveries = snapshot
+ .blocks
+ .iter()
+ .enumerate()
+ .filter(|(_, block)| block.finalizer_mode == FinalizerMode::Recovery)
+ .map(|(index, block)| recovery_evidence(&snapshot.blocks, index, block))
+ .collect::<Vec<_>>();
+
+ Ok(AuditReport {
+ format: 1,
+ source: source.to_path_buf(),
+ captured_at_ms: now_ms()?,
+ database_copy_sha256,
+ verified_under_current_consensus_ruleset,
+ profile_id: snapshot.launch_profile.profile_id.clone(),
+ height: tip.height,
+ tip_hash: tip.hash.clone(),
+ genesis_hash: genesis.hash.clone(),
+ recovery_count: recoveries.len(),
+ recoveries,
+ })
+}
+
+fn recovery_evidence(blocks: &[Block], index: usize, recovery: &Block) -> RecoveryEvidence {
+ let parent_timestamp = index
+ .checked_sub(1)
+ .and_then(|parent| blocks.get(parent))
+ .map(|block| block.timestamp_ms)
+ .unwrap_or(recovery.timestamp_ms);
+ let next_ticket_block = blocks[index.saturating_add(1)..]
+ .iter()
+ .find(|block| block.finalizer_mode == FinalizerMode::Ticket);
+ let next_ticket = next_ticket_block.map(|block| TicketResume {
+ height: block.height,
+ hash: block.hash.clone(),
+ rank: block.finalizer_rank,
+ delay_blocks: block.height.saturating_sub(recovery.height),
+ delay_ms: block.timestamp_ms.saturating_sub(recovery.timestamp_ms),
+ });
+ let immediate_ticket_resume = blocks.get(index.saturating_add(1)).is_some_and(|block| {
+ block.finalizer_mode == FinalizerMode::Ticket && block.prev_hash == recovery.hash
+ });
+
+ RecoveryEvidence {
+ height: recovery.height,
+ hash: recovery.hash.clone(),
+ miner: recovery.miner.clone(),
+ timestamp_ms: recovery.timestamp_ms,
+ gap_from_parent_ms: recovery.timestamp_ms.saturating_sub(parent_timestamp),
+ next_ticket,
+ immediate_ticket_resume,
+ }
+}
+
+fn cleanup_temporary_database(path: &Path) {
+ let _ = fs::remove_file(path);
+ for suffix in ["-wal", "-shm"] {
+ let companion = path_with_suffix(path, suffix);
+ let _ = fs::remove_file(companion);
+ }
+}
+
+fn path_with_suffix(path: &Path, suffix: &str) -> PathBuf {
+ let mut value = path.as_os_str().to_os_string();
+ value.push(suffix);
+ PathBuf::from(value)
+}
+
+fn file_sha256(path: &Path) -> Result<String> {
+ let mut file = fs::File::open(path)?;
+ let mut digest = Sha256::new();
+ let mut buffer = [0_u8; 64 * 1024];
+ loop {
+ let read = file.read(&mut buffer)?;
+ if read == 0 {
+ break;
+ }
+ digest.update(&buffer[..read]);
+ }
+ Ok(format!("{:x}", digest.finalize()))
+}
+
+fn now_ms() -> Result<u64> {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .context("system clock is before Unix epoch")?
+ .as_millis()
+ .try_into()
+ .context("timestamp does not fit in u64")
+}