commit 4946004c2617a4c25b767af4a200dde5212f35e6
parent def191e375ea07846d950899013644ec00f93767
Author: Joris Hartog <jorishartog@hotmail.com>
Date: Wed, 19 Aug 2026 15:27:47 +0200
Add release-mode soak test
Diffstat:
3 files changed, 284 insertions(+), 3 deletions(-)
diff --git a/ROADMAP.md b/ROADMAP.md
@@ -49,7 +49,7 @@ These items are not protocol rules. They are the attack and reliability checks t
- [x] HTTP/auth abuse tests cover CSRF same-origin behavior, lockout/backoff behavior, forwarded-header spoofing from untrusted peers, and session expiry.
- [x] Multi-node in-memory simulation covers delayed gossip, withheld burn bundles, bundle equivocation, partitions, restarts, persistence reload, and convergence.
-- [ ] Long-running release-mode soak test runs with automatic burn/finalization, P2P sync, Stratum-disabled and Stratum-enabled nodes, and periodic node restarts.
+- [x] Long-running release-mode soak test runs with automatic burn/finalization, P2P sync, Stratum-disabled and Stratum-enabled nodes, and periodic node restarts.
- [ ] Operator failure playbooks exist for stalled height, divergent tips, old snapshots, no burn committee signatures, recovery blocks, and corrupted local persistence.
- [ ] Mainnet-candidate release rehearsal includes fresh genesis, published bootnodes, checksums, backup/restore instructions, and a no-reset stability window.
@@ -114,7 +114,7 @@ Focus: improve usability, tooling, and governance after the base network is stab
A release intended for deployment must pass:
- `cargo test --locked`
-- `cargo test --locked --test properties -- --ignored`
+- `cargo test --locked --release --test properties -- --ignored`
Normal local development may skip ignored long-running property tests, but deployment must not.
diff --git a/deployment.sh b/deployment.sh
@@ -75,7 +75,7 @@ run_release_tests() {
require_command cargo
cargo test --locked
- cargo test --locked --test properties -- --ignored
+ cargo test --locked --release --test properties -- --ignored
}
update_versions() {
diff --git a/tests/properties.rs b/tests/properties.rs
@@ -0,0 +1,281 @@
+use std::{
+ collections::BTreeMap,
+ net::{Ipv4Addr, SocketAddr, TcpListener as StdTcpListener},
+ sync::Arc,
+ time::Duration,
+};
+
+use anyhow::{Context, Result, bail};
+use iuna::{
+ adapters::{chain_store::SqliteChainStore, p2p::GossipNetwork, stratum::StratumServer},
+ app::{NodeCore, PeerBook, SharedNode, now_ms},
+ domain::{GenesisBurn, Ledger, MICRO_IUNA, Wallet, run_vdf},
+};
+use serde_json::{Value, json};
+use tempfile::tempdir;
+use tokio::{
+ io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+ net::TcpStream,
+ sync::Mutex,
+ time::{sleep, timeout},
+};
+
+const SOAK_BLOCKS: u64 = 12;
+const BURN_COLLECTION_MS: u64 = 31_000;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "long-running release-mode soak; run with cargo test --release --test properties -- --ignored"]
+async fn release_soak_auto_finalization_p2p_stratum_and_restarts() -> Result<()> {
+ let wallets = (0..3)
+ .map(|index| Wallet::from_seed(&format!("release-soak-wallet-{index}")))
+ .collect::<Vec<_>>();
+ let genesis = funded_ledger(&wallets);
+ let p2p_addrs = reserve_loopback_addrs(wallets.len())?;
+ let stratum_addr = reserve_loopback_addrs(1)?.remove(0);
+ let store_dirs = (0..wallets.len())
+ .map(|_| tempdir())
+ .collect::<std::result::Result<Vec<_>, _>>()?;
+ let stores = store_dirs
+ .iter()
+ .map(|dir| SqliteChainStore::open(dir.path().join("chain.sqlite3")))
+ .collect::<Result<Vec<_>>>()?;
+ let mut nodes = Vec::new();
+
+ for (index, wallet) in wallets.iter().cloned().enumerate() {
+ let burn_per_block = if index == 0 { 2 } else { 0 };
+ let mut core = NodeCore::from_ledger_with_burn_fee_and_enabled(
+ wallet,
+ genesis.clone(),
+ true,
+ burn_per_block,
+ 1,
+ );
+ core.set_recovery_vdf_top_rank_percent(0);
+ let node = Arc::new(Mutex::new(core));
+ let peer_addresses = p2p_addrs
+ .iter()
+ .enumerate()
+ .filter(|(peer_index, _)| *peer_index != index)
+ .map(|(_, addr)| addr.to_string())
+ .collect::<Vec<_>>();
+ let peers = Arc::new(Mutex::new(PeerBook::from_addresses(peer_addresses)));
+ let network =
+ GossipNetwork::start(node.clone(), peers, p2p_addrs[index], None, true).await?;
+ nodes.push(SoakNode {
+ wallet: wallets[index].clone(),
+ burn_per_block,
+ node,
+ network,
+ store: stores[index].clone(),
+ });
+ }
+
+ let _stratum = StratumServer::start(
+ nodes[0].node.clone(),
+ nodes[0].network.clone(),
+ stratum_addr,
+ )
+ .await?;
+ assert_stratum_serves_work(stratum_addr, wallets[1].address()).await?;
+ sleep(Duration::from_secs(2)).await;
+
+ for target_height in 1..=SOAK_BLOCKS {
+ finalize_one_block(&nodes, target_height).await?;
+ wait_for_convergence(&nodes, target_height, Duration::from_secs(8)).await?;
+
+ if target_height % 3 == 0 {
+ restart_node_core(&nodes[1]).await?;
+ wait_for_convergence(&nodes, target_height, Duration::from_secs(8)).await?;
+ }
+ if target_height % 4 == 0 {
+ restart_node_core(&nodes[2]).await?;
+ wait_for_convergence(&nodes, target_height, Duration::from_secs(8)).await?;
+ }
+ }
+
+ let final_tip = nodes[0].node.lock().await.chain_tip_hash();
+ for node in &nodes {
+ assert_eq!(node.node.lock().await.chain_tip_hash(), final_tip);
+ }
+ assert!(configured_automatic_burn_was_included(&nodes[0]).await);
+ Ok(())
+}
+
+struct SoakNode {
+ wallet: Wallet,
+ burn_per_block: u64,
+ node: SharedNode,
+ network: GossipNetwork,
+ store: SqliteChainStore,
+}
+
+fn funded_ledger(wallets: &[Wallet]) -> Ledger {
+ let allocations = wallets
+ .iter()
+ .map(|wallet| (wallet.address().to_string(), 100 * MICRO_IUNA))
+ .collect::<BTreeMap<_, _>>();
+ let genesis_burns = wallets
+ .iter()
+ .map(|wallet| GenesisBurn::new(wallet.address(), MICRO_IUNA))
+ .collect::<Vec<_>>();
+ Ledger::new_with_genesis_burns(allocations, genesis_burns, 1).unwrap()
+}
+
+fn reserve_loopback_addrs(count: usize) -> Result<Vec<SocketAddr>> {
+ let mut listeners = Vec::new();
+ let mut addrs = Vec::new();
+ for _ in 0..count {
+ let listener = StdTcpListener::bind((Ipv4Addr::LOCALHOST, 0))?;
+ addrs.push(listener.local_addr()?);
+ listeners.push(listener);
+ }
+ drop(listeners);
+ Ok(addrs)
+}
+
+async fn assert_stratum_serves_work(addr: SocketAddr, worker: &str) -> Result<()> {
+ let stream = timeout(Duration::from_secs(5), TcpStream::connect(addr)).await??;
+ let (read, mut write) = stream.into_split();
+ let mut lines = BufReader::new(read).lines();
+
+ write
+ .write_all(
+ json_line(json!({"id": 1, "method": "mining.subscribe", "params": []}))?.as_bytes(),
+ )
+ .await?;
+ write
+ .write_all(
+ json_line(json!({"id": 2, "method": "mining.authorize", "params": [worker, "x"]}))?
+ .as_bytes(),
+ )
+ .await?;
+
+ let mut authorized = false;
+ let mut notified = false;
+ for _ in 0..4 {
+ let line = timeout(Duration::from_secs(5), lines.next_line())
+ .await??
+ .context("stratum server closed before sending work")?;
+ let value: Value = serde_json::from_str(&line)?;
+ authorized |=
+ value.get("id") == Some(&json!(2)) && value.get("result") == Some(&json!(true));
+ notified |= value.get("method") == Some(&json!("mining.notify"));
+ if authorized && notified {
+ return Ok(());
+ }
+ }
+ bail!("stratum did not authorize and send mining.notify")
+}
+
+fn json_line(value: Value) -> Result<String> {
+ Ok(format!("{}\n", serde_json::to_string(&value)?))
+}
+
+async fn finalize_one_block(nodes: &[SoakNode], target_height: u64) -> Result<()> {
+ let start = now_ms().saturating_sub(BURN_COLLECTION_MS + 1);
+ prepare_and_broadcast(nodes, start).await?;
+ sleep(Duration::from_millis(250)).await;
+ let timestamp_ms = now_ms();
+ prepare_and_broadcast(nodes, timestamp_ms).await?;
+
+ let deadline = tokio::time::Instant::now() + Duration::from_secs(8);
+ loop {
+ for node in nodes {
+ if let Some(block) = complete_if_ready(node, now_ms()).await? {
+ node.network
+ .broadcast(node.node.lock().await.drain_outbox())
+ .await?;
+ assert_eq!(block.height, target_height);
+ return Ok(());
+ }
+ }
+ if tokio::time::Instant::now() >= deadline {
+ bail!("no node finalized block {target_height}");
+ }
+ sleep(Duration::from_millis(100)).await;
+ }
+}
+
+async fn prepare_and_broadcast(nodes: &[SoakNode], timestamp_ms: u64) -> Result<()> {
+ for node in nodes {
+ {
+ let mut core = node.node.lock().await;
+ let _ = core.prepare_automatic_finalization(timestamp_ms);
+ }
+ node.network
+ .broadcast(node.node.lock().await.drain_outbox())
+ .await?;
+ }
+ Ok(())
+}
+
+async fn complete_if_ready(
+ node: &SoakNode,
+ timestamp_ms: u64,
+) -> Result<Option<iuna::domain::Block>> {
+ let work = {
+ let mut core = node.node.lock().await;
+ core.prepare_automatic_finalization(timestamp_ms).work
+ };
+ let Some(work) = work else {
+ return Ok(None);
+ };
+ let vdf_output = run_vdf(work.vdf_seed(), work.vdf_rounds());
+ let block =
+ node.node
+ .lock()
+ .await
+ .complete_prepared_block_at(work, vdf_output, timestamp_ms)?;
+ Ok(Some(block))
+}
+
+async fn wait_for_convergence(nodes: &[SoakNode], height: u64, duration: Duration) -> Result<()> {
+ let deadline = tokio::time::Instant::now() + duration;
+ loop {
+ let mut tips = Vec::new();
+ for node in nodes {
+ let core = node.node.lock().await;
+ tips.push((core.chain_height(), core.chain_tip_hash()));
+ }
+ if tips
+ .iter()
+ .all(|(node_height, tip)| *node_height >= height && tip == &tips[0].1)
+ {
+ return Ok(());
+ }
+ if tokio::time::Instant::now() >= deadline {
+ bail!("nodes did not converge at height {height}: {tips:?}");
+ }
+ sleep(Duration::from_millis(250)).await;
+ }
+}
+
+async fn configured_automatic_burn_was_included(node: &SoakNode) -> bool {
+ node.node
+ .lock()
+ .await
+ .chain()
+ .iter()
+ .flat_map(|block| &block.transactions)
+ .any(|tx| tx.is_burn() && tx.sender() == node.wallet.address() && tx.amount() == 2)
+}
+
+async fn restart_node_core(node: &SoakNode) -> Result<()> {
+ let snapshot = node.node.lock().await.chain_snapshot();
+ node.store.save(&snapshot)?;
+ let restored = Ledger::from_persisted_snapshot(
+ node.store
+ .load()?
+ .context("persisted snapshot should exist after save")?,
+ )?;
+ let mut restored_node = NodeCore::from_ledger_with_burn_fee_and_enabled(
+ node.wallet.clone(),
+ restored,
+ true,
+ node.burn_per_block,
+ 1,
+ );
+ restored_node.set_recovery_vdf_top_rank_percent(0);
+ *node.node.lock().await = restored_node;
+ Ok(())
+}