commit d3eb035f83f0f263aa63dbb23bc81e9da2ffb81a
parent 4a9bd974c2e7697210ed5987bb8c67542ab4bf7e
Author: Joris Hartog <jorishartog@hotmail.com>
Date: Wed, 19 Aug 2026 16:04:35 +0200
Add operator health metrics
Diffstat:
8 files changed, 265 insertions(+), 7 deletions(-)
diff --git a/ROADMAP.md b/ROADMAP.md
@@ -28,7 +28,7 @@ The current goal is to keep a small real testnet stable while increasing confide
- [ ] Genesis allocation plan is published and reviewed.
- [ ] Security review is complete for consensus validation, transaction validation, P2P input handling, and wallet/key storage.
- [ ] Upgrade and rollback instructions exist.
-- [ ] Basic operational monitoring is available for height, tip hash, peers, last block age, finalizer mode, VDF rounds, mempool, and rejected blocks.
+- [x] Basic operational monitoring is available for height, tip hash, peers, last block age, finalizer mode, VDF rounds, mempool, and rejected blocks.
## Pre-Reset Mainnet-Candidate Test Backlog
@@ -70,8 +70,8 @@ Focus: make failure modes boring and observable.
- [x] Add long-running soak chaos test.
- [x] Keep long-running tests out of normal local test runs.
- [x] Run long-running tests unconditionally during deployment.
-- [ ] Run a multi-day public testnet without manual chain resets.
-- [ ] Add or improve operator-facing health metrics.
+- [x] Run a multi-day public testnet without manual chain resets.
+- [x] Add or improve operator-facing health metrics.
- [x] Document common testnet failure/recovery playbooks.
### M2: Mainnet Candidate
diff --git a/src/adapters/http/api.rs b/src/adapters/http/api.rs
@@ -483,13 +483,37 @@ pub(super) async fn api_network_health(
) -> Json<NetworkHealthResponse> {
let (local, mempool) = {
let node = state.node.lock().await;
+ let status = node.status();
+ let tip = node.chain().last().cloned();
+ let p2p_metrics = state.gossip.metrics();
+ let tip_timestamp_ms = tip
+ .as_ref()
+ .filter(|block| block.height > 0)
+ .map(|block| block.timestamp_ms);
let mempool = MempoolCounts {
plain_transactions: node.pending_transactions().len(),
};
(
NetworkHealthLocalState {
- height: node.chain_height(),
+ height: status.chain.height,
+ tip_hash: status.chain.tip_hash,
+ tip_timestamp_ms,
pending_transactions: mempool.total(),
+ last_finalizer_mode: tip.as_ref().map(|block| match block.finalizer_mode {
+ crate::domain::FinalizerMode::Ticket => "ticket".to_string(),
+ crate::domain::FinalizerMode::Recovery => "recovery".to_string(),
+ }),
+ last_finalizer_rank: tip.as_ref().map(|block| block.finalizer_rank),
+ last_block_finalizer: tip.as_ref().map(|block| block.miner.clone()),
+ current_leader: status.mining.current_leader,
+ wallet_is_current_leader: status.mining.wallet_is_current_leader,
+ last_auto_finalization_status: status.mining.last_auto_finalization_status,
+ vdf_rounds: status.mining.vdf_rounds,
+ vdf_target_block_ms: status.mining.vdf_target_block_ms,
+ rejected_blocks: p2p_metrics.rejected_blocks,
+ rejected_block_batches: p2p_metrics.rejected_block_batches,
+ rejected_snapshots: p2p_metrics.rejected_snapshots,
+ last_chain_payload_error: p2p_metrics.last_chain_payload_error,
},
mempool,
)
diff --git a/src/adapters/http/index_html.rs b/src/adapters/http/index_html.rs
@@ -456,7 +456,7 @@ pub(super) const INDEX_HTML: &str = r#"<!doctype html>
.block-card { flex-basis: 108px; }
}
</style>
- <script defer src="/assets/iuna-ui.js?v=109"></script>
+ <script defer src="/assets/iuna-ui.js?v=110"></script>
<script defer src="/assets/alpine.min.js"></script>
</head>
<body x-data="iunaApp()" x-init="init()" @keydown.window.escape="closeModals()" x-cloak>
@@ -799,16 +799,22 @@ pub(super) const INDEX_HTML: &str = r#"<!doctype html>
<div class="network-health-state" :class="networkHealthClass()">
<div class="network-health-label">Network Health</div>
<div class="network-health-value" x-text="networkHealth.state || '-'"></div>
- <div class="network-health-detail" x-text="networkHealth.last_error || 'No peer errors reported'"></div>
+ <div class="network-health-detail" x-text="networkHealth.last_chain_payload_error || networkHealth.last_error || 'No peer errors reported'"></div>
</div>
<div class="network-health-grid">
<div class="peer-summary-item"><div class="peer-summary-label">Local Height</div><div class="peer-summary-value" x-text="networkHealth.local_height ?? '-'"></div></div>
+ <div class="peer-summary-item"><div class="peer-summary-label">Tip</div><code class="peer-summary-value" x-text="networkTipLabel()"></code></div>
+ <div class="peer-summary-item"><div class="peer-summary-label">Last Block</div><div class="peer-summary-value" x-text="networkLastBlockAgeLabel()"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Best Known</div><div class="peer-summary-value" x-text="networkHealth.best_known_height ?? '-'"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Lag</div><div class="peer-summary-value" x-text="networkLagLabel()"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Stale</div><div class="peer-summary-value" x-text="networkHealth.stale_peers ?? '-'"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Banned</div><div class="peer-summary-value" x-text="networkHealth.banned_peers ?? '-'"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Mempool</div><div class="peer-summary-value" x-text="networkHealth.pending_transactions ?? '-'"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Plain Tx</div><div class="peer-summary-value" x-text="networkHealth.pending_plain_transactions ?? '-'"></div></div>
+ <div class="peer-summary-item"><div class="peer-summary-label">Next Finalizer</div><code class="peer-summary-value" x-text="networkFinalizerLabel()"></code></div>
+ <div class="peer-summary-item"><div class="peer-summary-label">Last Mode</div><div class="peer-summary-value" x-text="networkFinalizerModeLabel()"></div></div>
+ <div class="peer-summary-item"><div class="peer-summary-label">VDF</div><div class="peer-summary-value" x-text="networkVdfLabel()"></div></div>
+ <div class="peer-summary-item"><div class="peer-summary-label">Rejected Chain</div><div class="peer-summary-value" x-text="networkHealth.rejected_chain_payloads ?? '-'"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Time Offset</div><div class="peer-summary-value" x-text="networkTimeOffsetLabel()"></div></div>
<div class="peer-summary-item"><div class="peer-summary-label">Clock Warnings</div><div class="peer-summary-value" x-text="networkHealth.bad_clock_peers ?? '-'"></div></div>
</div>
diff --git a/src/adapters/http/metrics.rs b/src/adapters/http/metrics.rs
@@ -163,6 +163,9 @@ pub(super) fn network_health_at(
now_ms: u64,
) -> NetworkHealthResponse {
let local_height = local.height;
+ let last_block_age_ms = local
+ .tip_timestamp_ms
+ .map(|tip_timestamp_ms| now_ms.saturating_sub(tip_timestamp_ms));
let remote_best_height = peers.iter().filter_map(|peer| peer.last_known_height).max();
let best_known_height = remote_best_height.unwrap_or(local_height).max(local_height);
let healthy_heights = peers
@@ -220,6 +223,10 @@ pub(super) fn network_health_at(
.as_ref()
.map(|error| format!("{}: {error}", peer.address))
});
+ let rejected_chain_payloads = local
+ .rejected_blocks
+ .saturating_add(local.rejected_block_batches)
+ .saturating_add(local.rejected_snapshots);
let state = if peers.is_empty() {
"isolated"
@@ -242,6 +249,8 @@ pub(super) fn network_health_at(
ok: !peers.is_empty() && lag_blocks == 0 && healthy_peers > stale_peers,
state,
local_height,
+ local_tip_hash: local.tip_hash,
+ last_block_age_ms,
best_known_height,
shared_height,
lag_blocks,
@@ -253,6 +262,19 @@ pub(super) fn network_health_at(
banned_peers,
pending_transactions: local.pending_transactions,
pending_plain_transactions: mempool.plain_transactions,
+ last_finalizer_mode: local.last_finalizer_mode,
+ last_finalizer_rank: local.last_finalizer_rank,
+ last_block_finalizer: local.last_block_finalizer,
+ current_leader: local.current_leader,
+ wallet_is_current_leader: local.wallet_is_current_leader,
+ last_auto_finalization_status: local.last_auto_finalization_status,
+ vdf_rounds: local.vdf_rounds,
+ vdf_target_block_ms: local.vdf_target_block_ms,
+ rejected_blocks: local.rejected_blocks,
+ rejected_block_batches: local.rejected_block_batches,
+ rejected_snapshots: local.rejected_snapshots,
+ rejected_chain_payloads,
+ last_chain_payload_error: local.last_chain_payload_error,
network_time_offset_ms,
bad_clock_peers,
last_error,
@@ -278,3 +300,65 @@ fn median_peer_clock_offset(peers: &[PeerInfo], now_ms: u64) -> Option<i64> {
offsets.sort_unstable();
Some(offsets[offsets.len() / 2])
}
+
+#[cfg(test)]
+mod tests {
+ use super::{MempoolCounts, NetworkHealthLocalState, network_health_at};
+
+ #[test]
+ fn network_health_exposes_operator_chain_and_rejection_context() {
+ let local = NetworkHealthLocalState {
+ height: 42,
+ tip_hash: "tip-hash".to_string(),
+ tip_timestamp_ms: Some(1_000),
+ pending_transactions: 3,
+ last_finalizer_mode: Some("ticket".to_string()),
+ last_finalizer_rank: Some(1),
+ last_block_finalizer: Some("last-finalizer".to_string()),
+ current_leader: Some("next-leader".to_string()),
+ wallet_is_current_leader: true,
+ last_auto_finalization_status: Some("waiting for VDF".to_string()),
+ vdf_rounds: 67_000_000,
+ vdf_target_block_ms: 120_000,
+ rejected_blocks: 2,
+ rejected_block_batches: 3,
+ rejected_snapshots: 5,
+ last_chain_payload_error: Some("snapshot: invalid block".to_string()),
+ };
+
+ let health = network_health_at(
+ local,
+ &[],
+ MempoolCounts {
+ plain_transactions: 3,
+ },
+ 2_500,
+ );
+
+ assert_eq!(health.local_height, 42);
+ assert_eq!(health.local_tip_hash, "tip-hash");
+ assert_eq!(health.last_block_age_ms, Some(1_500));
+ assert_eq!(health.last_finalizer_mode.as_deref(), Some("ticket"));
+ assert_eq!(health.last_finalizer_rank, Some(1));
+ assert_eq!(
+ health.last_block_finalizer.as_deref(),
+ Some("last-finalizer")
+ );
+ assert_eq!(health.current_leader.as_deref(), Some("next-leader"));
+ assert!(health.wallet_is_current_leader);
+ assert_eq!(
+ health.last_auto_finalization_status.as_deref(),
+ Some("waiting for VDF")
+ );
+ assert_eq!(health.vdf_rounds, 67_000_000);
+ assert_eq!(health.vdf_target_block_ms, 120_000);
+ assert_eq!(health.rejected_blocks, 2);
+ assert_eq!(health.rejected_block_batches, 3);
+ assert_eq!(health.rejected_snapshots, 5);
+ assert_eq!(health.rejected_chain_payloads, 10);
+ assert_eq!(
+ health.last_chain_payload_error.as_deref(),
+ Some("snapshot: invalid block")
+ );
+ }
+}
diff --git a/src/adapters/http/types.rs b/src/adapters/http/types.rs
@@ -27,6 +27,8 @@ pub(super) struct NetworkHealthResponse {
pub(super) ok: bool,
pub(super) state: String,
pub(super) local_height: u64,
+ pub(super) local_tip_hash: String,
+ pub(super) last_block_age_ms: Option<u64>,
pub(super) best_known_height: u64,
pub(super) shared_height: u64,
pub(super) lag_blocks: u64,
@@ -38,6 +40,19 @@ pub(super) struct NetworkHealthResponse {
pub(super) banned_peers: usize,
pub(super) pending_transactions: usize,
pub(super) pending_plain_transactions: usize,
+ pub(super) last_finalizer_mode: Option<String>,
+ pub(super) last_finalizer_rank: Option<u32>,
+ pub(super) last_block_finalizer: Option<String>,
+ pub(super) current_leader: Option<String>,
+ pub(super) wallet_is_current_leader: bool,
+ pub(super) last_auto_finalization_status: Option<String>,
+ pub(super) vdf_rounds: u64,
+ pub(super) vdf_target_block_ms: u64,
+ pub(super) rejected_blocks: u64,
+ pub(super) rejected_block_batches: u64,
+ pub(super) rejected_snapshots: u64,
+ pub(super) rejected_chain_payloads: u64,
+ pub(super) last_chain_payload_error: Option<String>,
pub(super) network_time_offset_ms: Option<i64>,
pub(super) bad_clock_peers: usize,
pub(super) last_error: Option<String>,
@@ -54,10 +69,24 @@ impl MempoolCounts {
}
}
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Debug)]
pub(super) struct NetworkHealthLocalState {
pub(super) height: u64,
+ pub(super) tip_hash: String,
+ pub(super) tip_timestamp_ms: Option<u64>,
pub(super) pending_transactions: usize,
+ pub(super) last_finalizer_mode: Option<String>,
+ pub(super) last_finalizer_rank: Option<u32>,
+ pub(super) last_block_finalizer: Option<String>,
+ pub(super) current_leader: Option<String>,
+ pub(super) wallet_is_current_leader: bool,
+ pub(super) last_auto_finalization_status: Option<String>,
+ pub(super) vdf_rounds: u64,
+ pub(super) vdf_target_block_ms: u64,
+ pub(super) rejected_blocks: u64,
+ pub(super) rejected_block_batches: u64,
+ pub(super) rejected_snapshots: u64,
+ pub(super) last_chain_payload_error: Option<String>,
}
#[derive(Debug, Deserialize)]
diff --git a/src/adapters/p2p/metrics.rs b/src/adapters/p2p/metrics.rs
@@ -26,6 +26,9 @@ pub(super) struct P2pMetricsCounters {
pub(super) burn_bundle_envelopes_received: AtomicU64,
pub(super) burn_bundles_received: AtomicU64,
pub(super) control_envelopes_received: AtomicU64,
+ pub(super) rejected_blocks: AtomicU64,
+ pub(super) rejected_block_batches: AtomicU64,
+ pub(super) rejected_snapshots: AtomicU64,
pub(super) bytes_received: AtomicU64,
pub(super) parse_errors: AtomicU64,
pub(super) empty_frames: AtomicU64,
@@ -36,6 +39,7 @@ pub(super) struct P2pMetricsCounters {
pub(super) last_session_failure: StdMutex<Option<String>>,
pub(super) last_empty_frame_remote: StdMutex<Option<String>>,
pub(super) last_parse_error: StdMutex<Option<String>>,
+ pub(super) last_chain_payload_error: StdMutex<Option<String>>,
}
#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)]
@@ -59,6 +63,9 @@ pub struct P2pMetrics {
pub burn_bundle_envelopes_received: u64,
pub burn_bundles_received: u64,
pub control_envelopes_received: u64,
+ pub rejected_blocks: u64,
+ pub rejected_block_batches: u64,
+ pub rejected_snapshots: u64,
pub bytes_received: u64,
pub parse_errors: u64,
pub empty_frames: u64,
@@ -69,6 +76,7 @@ pub struct P2pMetrics {
pub last_session_failure: Option<String>,
pub last_empty_frame_remote: Option<String>,
pub last_parse_error: Option<String>,
+ pub last_chain_payload_error: Option<String>,
}
impl P2pMetricsCounters {
@@ -113,6 +121,9 @@ impl P2pMetricsCounters {
.load(Ordering::Relaxed),
burn_bundles_received: self.burn_bundles_received.load(Ordering::Relaxed),
control_envelopes_received: self.control_envelopes_received.load(Ordering::Relaxed),
+ rejected_blocks: self.rejected_blocks.load(Ordering::Relaxed),
+ rejected_block_batches: self.rejected_block_batches.load(Ordering::Relaxed),
+ rejected_snapshots: self.rejected_snapshots.load(Ordering::Relaxed),
bytes_received: self.bytes_received.load(Ordering::Relaxed),
parse_errors: self.parse_errors.load(Ordering::Relaxed),
empty_frames: self.empty_frames.load(Ordering::Relaxed),
@@ -135,6 +146,35 @@ impl P2pMetricsCounters {
.lock()
.ok()
.and_then(|last| last.clone()),
+ last_chain_payload_error: self
+ .last_chain_payload_error
+ .lock()
+ .ok()
+ .and_then(|last| last.clone()),
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::P2pMetricsCounters;
+
+ #[test]
+ fn snapshot_reports_rejected_chain_payloads() {
+ let metrics = P2pMetricsCounters::default();
+
+ P2pMetricsCounters::inc(&metrics.rejected_blocks);
+ P2pMetricsCounters::inc(&metrics.rejected_block_batches);
+ P2pMetricsCounters::inc(&metrics.rejected_snapshots);
+ P2pMetricsCounters::set_last(&metrics.last_chain_payload_error, "block: invalid VDF");
+
+ let snapshot = metrics.snapshot();
+ assert_eq!(snapshot.rejected_blocks, 1);
+ assert_eq!(snapshot.rejected_block_batches, 1);
+ assert_eq!(snapshot.rejected_snapshots, 1);
+ assert_eq!(
+ snapshot.last_chain_payload_error.as_deref(),
+ Some("block: invalid VDF")
+ );
+ }
+}
diff --git a/src/adapters/p2p/process.rs b/src/adapters/p2p/process.rs
@@ -120,6 +120,12 @@ pub(super) async fn process_envelope(
Err(error) => Err(error),
};
let request_snapshot = result.as_ref().err().is_some_and(is_possible_fork_error);
+ record_rejected_chain_payload(
+ network,
+ &network.inner.metrics.rejected_blocks,
+ "block",
+ &result,
+ );
record_inbound_result(network, known_peer, remote_addr, result).await;
if request_snapshot {
write_envelope(writer, &GossipEnvelope::ChainSnapshotRequest).await?;
@@ -141,6 +147,12 @@ pub(super) async fn process_envelope(
Err(error) => Err(error),
};
let request_snapshot = result.as_ref().err().is_some_and(is_possible_fork_error);
+ record_rejected_chain_payload(
+ network,
+ &network.inner.metrics.rejected_block_batches,
+ "block batch",
+ &result,
+ );
record_inbound_result(network, known_peer, remote_addr, result).await;
if request_snapshot {
write_envelope(writer, &GossipEnvelope::ChainSnapshotRequest).await?;
@@ -161,6 +173,12 @@ pub(super) async fn process_envelope(
.map(|_| ()),
Err(error) => Err(error),
};
+ record_rejected_chain_payload(
+ network,
+ &network.inner.metrics.rejected_snapshots,
+ "snapshot",
+ &result,
+ );
record_inbound_result(network, known_peer, remote_addr, result).await;
network.forward_outbox().await;
}
@@ -229,6 +247,22 @@ async fn process_burn_bundles(
network.forward_outbox().await;
}
+fn record_rejected_chain_payload(
+ network: &GossipNetwork,
+ counter: &std::sync::atomic::AtomicU64,
+ kind: &str,
+ result: &Result<()>,
+) {
+ let Err(error) = result else {
+ return;
+ };
+ P2pMetricsCounters::inc(counter);
+ P2pMetricsCounters::set_last(
+ &network.inner.metrics.last_chain_payload_error,
+ format!("{kind}: {error:#}"),
+ );
+}
+
async fn record_inbound_result(
network: &GossipNetwork,
known_peer: &Option<String>,
diff --git a/www/assets/iuna-ui.js b/www/assets/iuna-ui.js
@@ -2940,6 +2940,36 @@ window.iunaApp = function iunaApp() {
return `${lag} behind`;
},
+ networkTipLabel() {
+ return this.short(this.networkHealth.local_tip_hash);
+ },
+
+ networkLastBlockAgeLabel() {
+ const ageMs = this.networkHealth.last_block_age_ms;
+ if (typeof ageMs !== "number" || !Number.isFinite(ageMs)) return "-";
+ return this.durationLabel(ageMs);
+ },
+
+ networkFinalizerLabel() {
+ return this.shortAddressLabel(this.networkHealth.current_leader);
+ },
+
+ networkFinalizerModeLabel() {
+ const mode = this.networkHealth.last_finalizer_mode;
+ if (!mode) return "-";
+ const rank = this.networkHealth.last_finalizer_rank;
+ const label = mode.charAt(0).toUpperCase() + mode.slice(1);
+ return typeof rank === "number" ? `${label} #${rank}` : label;
+ },
+
+ networkVdfLabel() {
+ const rounds = this.networkHealth.vdf_rounds;
+ const targetMs = this.networkHealth.vdf_target_block_ms;
+ if (typeof rounds !== "number") return "-";
+ const target = typeof targetMs === "number" ? ` / ${this.durationLabel(targetMs)}` : "";
+ return `${rounds.toLocaleString()}${target}`;
+ },
+
basicNetworkStatusLabel() {
const state = this.networkHealth.state;
if (!state) return "Network starting";
@@ -3017,6 +3047,17 @@ window.iunaApp = function iunaApp() {
return `${Math.round(ageHours / 24)}d ago`;
},
+ durationLabel(durationMs) {
+ if (typeof durationMs !== "number" || !Number.isFinite(durationMs)) return "-";
+ const seconds = Math.max(0, Math.round(durationMs / 1000));
+ if (seconds < 60) return `${seconds}s`;
+ const minutes = Math.round(seconds / 60);
+ if (minutes < 60) return `${minutes}m`;
+ const hours = Math.round(minutes / 60);
+ if (hours < 48) return `${hours}h`;
+ return `${Math.round(hours / 24)}d`;
+ },
+
peerLastContactLabel(peer) {
return this.relativeTimeLabel(peer.last_contact_ms);
},