From e39c7170a4f755980169f89d5c533c7b1c272743 Mon Sep 17 00:00:00 2001 From: Pablosinyores Date: Wed, 20 May 2026 17:13:19 +0530 Subject: [PATCH] feat(scorer): add reason sub-label to profit_scored_total MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit aether_mempool_profit_scored_total used to be labelled only by decision, so the dashboard could see "10 reverted" but not whether they came from the V2 U256 walker, the f64 absurdity floor, the V3 revm verifier, or organic revm reverts. Add a `reason` sub-label distinguishing those code paths. Five wire labels (pinned by unit test against the constants): - n/a non-reverted decisions, no_path, or any path with no sub-source worth distinguishing - u256_walker V2-only exact-U256 walker reached a verdict (PR #136 path) - absurdity_floor f64 fallback above MAX_PLAUSIBLE_F64_NET_WEI (1 ETH) downgraded to reverted (PR #136 path) - revm_verdict V3-touching revm sim ran to completion with a non-reverting verdict (PR #144 path) - revm_revert V3-touching revm sim explicitly reverted/halted (PR #144 path) The reason is Prometheus-only and NOT persisted to the mempool_profitability table — the migration's CHECK constraint only covers decision, and adding a reason column would force every existing row to back-fill. `NewProfitabilityScore.reason` skips the DB insert path; it only flows into the metric label. revm_verdict_to_decision and f64_fallback_verdict now return a 4-tuple (net, realised, decision, reason). no_path_outcome carries REASON_NA. The aggregating let in score_one destructures (net, realised, decision, reason) and threads reason into ScoreOutcome. Dashboard panels in deploy/docker/grafana/dashboards/mempool.json (panel IDs 19 and 20 from PR #146) updated to sum by (decision, reason) and legend-format {{decision}} / {{reason}}. Title and description updated to reflect the new dimension. Stacks on feat/dedupe-replay-scorer-helpers (PR #148). Verification: - cargo clippy --workspace --all-targets -- -D warnings : clean - cargo test --workspace --lib --bins : 528 passed, 0 failed - new test: reason_constants_are_stable_wire_labels - existing verdict-helper tests updated to assert reason value - python3 -m json.tool mempool.json : parses cleanly --- .../src/bin/aether_profit_scorer.rs | 71 +++++++++++++------ .../grpc-server/src/profitability_writer.rs | 65 +++++++++++++++-- deploy/docker/grafana/dashboards/mempool.json | 16 ++--- 3 files changed, 118 insertions(+), 34 deletions(-) diff --git a/crates/grpc-server/src/bin/aether_profit_scorer.rs b/crates/grpc-server/src/bin/aether_profit_scorer.rs index 47d352f..07f1b7c 100644 --- a/crates/grpc-server/src/bin/aether_profit_scorer.rs +++ b/crates/grpc-server/src/bin/aether_profit_scorer.rs @@ -78,7 +78,8 @@ use aether_grpc_server::historical::{ use aether_grpc_server::profitability_writer::{ profit_writer_from_env, NewProfitabilityScore, PgProfitabilityWriter, ProfitabilitySink, ProfitabilityWriterMetrics, UnscoredConfirmedPrediction, DECISION_NO_PATH, - DECISION_PROFITABLE, DECISION_REVERTED, DECISION_UNPROFITABLE, + DECISION_PROFITABLE, DECISION_REVERTED, DECISION_UNPROFITABLE, REASON_ABSURDITY_FLOOR, + REASON_NA, REASON_REVM_REVERT, REASON_REVM_VERDICT, REASON_U256_WALKER, }; use aether_simulator::calldata::{ build_execute_arb_calldata, build_univ2_swap_calldata, build_univ3_swap_calldata, @@ -366,6 +367,7 @@ async fn score_batch( gas_estimate_wei: score.gas_estimate_wei, net_profit_wei: score.net_profit_wei, decision: score.decision, + reason: score.reason, scoring_engine_git_sha: git_sha.map(str::to_string), }), Err(e) => warn!( @@ -385,6 +387,10 @@ struct ScoreOutcome { gas_estimate_wei: U256, net_profit_wei: i128, decision: &'static str, + /// Prometheus-only sub-label describing which code path produced the + /// decision. Pinned to one of the `REASON_*` constants in + /// `profitability_writer`. Not persisted to Postgres. + reason: &'static str, } async fn score_one( @@ -474,7 +480,7 @@ async fn score_one( // absurdity floor. let v3_touching = is_v3_touching_cycle(best, &graph, token_index, pools, &running_states); - let (net, realized_wei_i128, decision) = if !v3_touching { + let (net, realized_wei_i128, decision, reason) = if !v3_touching { // V2-only: exact U256 getAmountOut walk (unchanged from pre-V3 scorer). let verified_gross = verify_cycle_u256( best, @@ -497,7 +503,7 @@ async fn score_one( } else { DECISION_UNPROFITABLE }; - (exact_net, realised, decision) + (exact_net, realised, decision, REASON_U256_WALKER) } None => f64_fallback_verdict(optimisation.net_profit_wei, gas_wei), } @@ -537,6 +543,7 @@ async fn score_one( gas_estimate_wei, net_profit_wei: net, decision, + reason, }) } @@ -548,6 +555,11 @@ fn no_path_outcome(gas: Option) -> ScoreOutcome { gas_estimate_wei: U256::from(gas_wei), net_profit_wei: -(gas_wei as i128), decision: DECISION_NO_PATH, + // `no_path` has no sub-source worth distinguishing today — the + // upstream causes (pool absent, eth_call empty, optimiser bailed, + // ...) collapse to the same dashboard slice. Add a sub-reason + // when one of those proves operationally interesting. + reason: REASON_NA, } } @@ -840,11 +852,17 @@ struct RevmVerdict { reverted: bool, } -/// Map a `RevmVerdict` into `(net, realised_i128, decision)`. -fn revm_verdict_to_decision(rv: RevmVerdict, gas_cost_wei: u128) -> (i128, i128, &'static str) { +/// Map a `RevmVerdict` into `(net, realised_i128, decision, reason)`. +/// Reverted verdicts carry `REASON_REVM_REVERT`; non-reverted verdicts +/// carry `REASON_REVM_VERDICT` so the dashboard can distinguish revm- +/// executed decisions from f64-fallback or U256-walker decisions. +fn revm_verdict_to_decision( + rv: RevmVerdict, + gas_cost_wei: u128, +) -> (i128, i128, &'static str, &'static str) { if rv.reverted { let gas_i128 = gas_cost_wei as i128; - (-(gas_i128), 0, DECISION_REVERTED) + (-(gas_i128), 0, DECISION_REVERTED, REASON_REVM_REVERT) } else { let gross_i128 = u256_to_i128_saturating(rv.gross_profit_wei); let net = gross_i128.saturating_sub(gas_cost_wei as i128); @@ -854,23 +872,27 @@ fn revm_verdict_to_decision(rv: RevmVerdict, gas_cost_wei: u128) -> (i128, i128, } else { DECISION_UNPROFITABLE }; - (net, realised, decision) + (net, realised, decision, REASON_REVM_VERDICT) } } /// Fallback for cycles that neither the U256 walker nor revm can resolve. /// Applies the absurdity floor: f64 nets above 1 ETH are downgraded to -/// REVERTED (precision-bias artefact). -fn f64_fallback_verdict(f64_net: i128, gas_cost_wei: u128) -> (i128, i128, &'static str) { +/// REVERTED (precision-bias artefact). Reverts via the floor carry +/// `REASON_ABSURDITY_FLOOR`; below-floor verdicts carry `REASON_NA` +/// because the f64-fallback path doesn't subdivide further. +fn f64_fallback_verdict( + f64_net: i128, + gas_cost_wei: u128, +) -> (i128, i128, &'static str, &'static str) { let realised = f64_net.saturating_add(gas_cost_wei as i128).max(0); - let decision = if f64_net > MAX_PLAUSIBLE_F64_NET_WEI { - DECISION_REVERTED + if f64_net > MAX_PLAUSIBLE_F64_NET_WEI { + (f64_net, realised, DECISION_REVERTED, REASON_ABSURDITY_FLOOR) } else if f64_net > 0 { - DECISION_PROFITABLE + (f64_net, realised, DECISION_PROFITABLE, REASON_NA) } else { - DECISION_UNPROFITABLE - }; - (f64_net, realised, decision) + (f64_net, realised, DECISION_UNPROFITABLE, REASON_NA) + } } /// Walk the cycle's hops and return `true` if any hop's pool state is @@ -1484,6 +1506,7 @@ mod tests { fn no_path_outcome_carries_negative_net_when_gas_given() { let out = no_path_outcome(Some(50_000)); assert_eq!(out.decision, DECISION_NO_PATH); + assert_eq!(out.reason, REASON_NA); assert_eq!(out.net_profit_wei, -50_000); assert_eq!(out.gas_estimate_wei, U256::from(50_000u64)); } @@ -1852,8 +1875,9 @@ mod tests { #[test] fn revm_verdict_decision_mapping_reverted() { let rv = RevmVerdict { gross_profit_wei: U256::ZERO, gas_used: 100_000, reverted: true }; - let (net, realised, dec) = revm_verdict_to_decision(rv, 50_000); + let (net, realised, dec, reason) = revm_verdict_to_decision(rv, 50_000); assert_eq!(dec, DECISION_REVERTED); + assert_eq!(reason, REASON_REVM_REVERT); assert!(net < 0); assert_eq!(realised, 0); } @@ -1865,8 +1889,9 @@ mod tests { gas_used: 100_000, reverted: false, }; - let (net, _realised, dec) = revm_verdict_to_decision(rv, 50_000); + let (net, _realised, dec, reason) = revm_verdict_to_decision(rv, 50_000); assert_eq!(dec, DECISION_PROFITABLE); + assert_eq!(reason, REASON_REVM_VERDICT); assert!(net > 0); } @@ -1877,30 +1902,34 @@ mod tests { gas_used: 100_000, reverted: false, }; - let (net, _realised, dec) = revm_verdict_to_decision(rv, 50_000); + let (net, _realised, dec, reason) = revm_verdict_to_decision(rv, 50_000); assert_eq!(dec, DECISION_UNPROFITABLE); + assert_eq!(reason, REASON_REVM_VERDICT); assert!(net <= 0); } #[test] fn f64_fallback_verdict_above_floor_reverted() { let big_net = MAX_PLAUSIBLE_F64_NET_WEI + 1; - let (_net, _realised, dec) = f64_fallback_verdict(big_net, 50_000); + let (_net, _realised, dec, reason) = f64_fallback_verdict(big_net, 50_000); assert_eq!(dec, DECISION_REVERTED); + assert_eq!(reason, REASON_ABSURDITY_FLOOR); } #[test] fn f64_fallback_verdict_below_floor_profitable() { let small_net = 1_000_000i128; - let (_net, _realised, dec) = f64_fallback_verdict(small_net, 50_000); + let (_net, _realised, dec, reason) = f64_fallback_verdict(small_net, 50_000); assert_eq!(dec, DECISION_PROFITABLE); + assert_eq!(reason, REASON_NA); } #[test] fn f64_fallback_verdict_negative_unprofitable() { let neg = -500_000i128; - let (net, _realised, dec) = f64_fallback_verdict(neg, 50_000); + let (net, _realised, dec, reason) = f64_fallback_verdict(neg, 50_000); assert_eq!(dec, DECISION_UNPROFITABLE); + assert_eq!(reason, REASON_NA); assert!(net < 0); } diff --git a/crates/grpc-server/src/profitability_writer.rs b/crates/grpc-server/src/profitability_writer.rs index cab80e0..e106fdd 100644 --- a/crates/grpc-server/src/profitability_writer.rs +++ b/crates/grpc-server/src/profitability_writer.rs @@ -38,6 +38,28 @@ pub const DECISION_UNPROFITABLE: &str = "unprofitable"; pub const DECISION_REVERTED: &str = "reverted"; pub const DECISION_NO_PATH: &str = "no_path"; +/// Prometheus-only sub-label on `aether_mempool_profit_scored_total` that +/// distinguishes WHICH code path produced the decision. NOT persisted to +/// the `mempool_profitability` table (the migration's CHECK constraint +/// only covers `decision`). Cardinality is bounded; every reason here is +/// a `&'static str` so adding a new one requires touching this file. +pub const REASON_NA: &str = "n/a"; +/// V2-only path: the exact-U256 walker (`verify_cycle_u256`, PR #136) +/// reached a verdict. Profitable / unprofitable / reverted variants all +/// share this reason. +pub const REASON_U256_WALKER: &str = "u256_walker"; +/// f64 fallback path: the f64 net exceeded `MAX_PLAUSIBLE_F64_NET_WEI` +/// and was downgraded to `reverted` (PR #136 precision-bias guard). +/// Only ever pairs with `DECISION_REVERTED`. +pub const REASON_ABSURDITY_FLOOR: &str = "absurdity_floor"; +/// V3-touching path: revm sim explicitly reverted/halted (PR #144). Only +/// ever pairs with `DECISION_REVERTED`. Distinct from `revm_verdict` +/// because the cycle ran through revm to completion rather than declining. +pub const REASON_REVM_REVERT: &str = "revm_revert"; +/// V3-touching path: revm sim ran to completion with a non-reverting +/// verdict. Pairs with `DECISION_PROFITABLE` / `DECISION_UNPROFITABLE`. +pub const REASON_REVM_VERDICT: &str = "revm_verdict"; + /// Insert payload for the `mempool_profitability` table. /// /// `realized_profit_eth` is derived from `realized_profit_wei` at write @@ -59,9 +81,18 @@ pub struct NewProfitabilityScore { /// `is_loss` flag set. pub net_profit_wei: i128, pub decision: &'static str, + /// Prometheus-only sub-label: which code path emitted this decision. + /// One of the `REASON_*` constants. Skipped during DB insert (the + /// `mempool_profitability` table has no `reason` column). + #[serde(default = "default_reason")] + pub reason: &'static str, pub scoring_engine_git_sha: Option, } +fn default_reason() -> &'static str { + REASON_NA +} + /// Sink trait. Object-safe so a single `Arc` can /// fan out to multiple scoring tasks (currently only one runs at a time, /// but the trait shape leaves room for a parallel batch scorer). @@ -86,9 +117,9 @@ impl ProfitabilityWriterMetrics { let scored_total = IntCounterVec::new( Opts::new( "aether_mempool_profit_scored_total", - "Confirmed predictions scored by the profitability scorer, by decision", + "Confirmed predictions scored by the profitability scorer, by decision and reason (which code path produced the decision: u256_walker / absurdity_floor / revm_verdict / revm_revert / n/a).", ), - &["decision"], + &["decision", "reason"], ) .expect("aether_mempool_profit_scored_total counter vec"); let drops_total = IntCounter::new( @@ -231,12 +262,13 @@ impl PgProfitabilityWriter { impl ProfitabilitySink for PgProfitabilityWriter { fn insert_score(&self, score: NewProfitabilityScore) { let decision = score.decision; + let reason = score.reason; match self.tx.try_send(score) { Ok(()) => { self.metrics.queue_depth.inc(); self.metrics .scored_total - .with_label_values(&[decision]) + .with_label_values(&[decision, reason]) .inc(); } Err(mpsc::error::TrySendError::Full(_)) => { @@ -438,12 +470,34 @@ mod tests { assert_eq!(DECISION_NO_PATH, "no_path"); } + #[test] + fn reason_constants_are_stable_wire_labels() { + // Pinned because dashboards (Grafana mempool.json from PR #146) + // hard-code these strings in PromQL queries that sum by + // `reason`. Renames break the panels silently. + assert_eq!(REASON_NA, "n/a"); + assert_eq!(REASON_U256_WALKER, "u256_walker"); + assert_eq!(REASON_ABSURDITY_FLOOR, "absurdity_floor"); + assert_eq!(REASON_REVM_REVERT, "revm_revert"); + assert_eq!(REASON_REVM_VERDICT, "revm_verdict"); + } + #[test] fn metrics_register_round_trips() { let registry = Registry::new(); let m = ProfitabilityWriterMetrics::register(®istry); - m.scored_total.with_label_values(&[DECISION_PROFITABLE]).inc(); - m.scored_total.with_label_values(&[DECISION_NO_PATH]).inc(); + m.scored_total + .with_label_values(&[DECISION_PROFITABLE, REASON_U256_WALKER]) + .inc(); + m.scored_total + .with_label_values(&[DECISION_NO_PATH, REASON_NA]) + .inc(); + m.scored_total + .with_label_values(&[DECISION_REVERTED, REASON_ABSURDITY_FLOOR]) + .inc(); + m.scored_total + .with_label_values(&[DECISION_REVERTED, REASON_REVM_REVERT]) + .inc(); m.drops_total.inc(); m.queue_depth.set(2); m.write_latency_ms.with_label_values(&["ok"]).observe(1.0); @@ -477,6 +531,7 @@ mod tests { gas_estimate_wei: U256::from(50_000_000_000_000u64), net_profit_wei: 950_000_000_000_000, decision: DECISION_PROFITABLE, + reason: REASON_U256_WALKER, scoring_engine_git_sha: Some("deadbeef".to_string()), } } diff --git a/deploy/docker/grafana/dashboards/mempool.json b/deploy/docker/grafana/dashboards/mempool.json index ce48efa..11639d2 100644 --- a/deploy/docker/grafana/dashboards/mempool.json +++ b/deploy/docker/grafana/dashboards/mempool.json @@ -506,8 +506,8 @@ { "id": 19, "type": "piechart", - "title": "Decision breakdown", - "description": "Cumulative scored-prediction split across profitable / unprofitable / reverted / no_path. PR #136's absurdity floor and PR #144's revm V3 verifier both feed the `reverted` bucket today; splitting them needs a sub-label on the counter (follow-up PR).", + "title": "Decision breakdown (by reason)", + "description": "Cumulative scored-prediction split across (decision, reason) pairs. Reason distinguishes which code path produced the decision: `u256_walker` (V2 exact walker, PR #136), `absurdity_floor` (f64 fallback above 1 ETH plausibility, PR #136), `revm_verdict` (V3-touching revm sim ran to completion, PR #144), `revm_revert` (V3-touching revm sim explicitly reverted, PR #144), `n/a` (path with no sub-source).", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -515,8 +515,8 @@ "targets": [ { "refId": "A", - "expr": "sum by (decision) (aether_mempool_profit_scored_total{job=\"aether-host-scorer\"})", - "legendFormat": "{{decision}}" + "expr": "sum by (decision, reason) (aether_mempool_profit_scored_total{job=\"aether-host-scorer\"})", + "legendFormat": "{{decision}} / {{reason}}" } ], "options": { @@ -532,8 +532,8 @@ { "id": 20, "type": "timeseries", - "title": "Scored rate by decision", - "description": "Per-decision scoring throughput. Stable non-zero `profitable` or `unprofitable` after the PR #145 engine restart is the V3-verifier end-to-end signal.", + "title": "Scored rate by decision and reason", + "description": "Per-(decision, reason) scoring throughput. Stable non-zero `profitable / revm_verdict` after the PR #145 engine restart is the V3-verifier end-to-end signal. Sustained `reverted / absurdity_floor` indicates f64 precision bias is firing without a U256/revm path to catch it.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -541,8 +541,8 @@ "targets": [ { "refId": "A", - "expr": "sum by (decision) (rate(aether_mempool_profit_scored_total{job=\"aether-host-scorer\"}[5m]))", - "legendFormat": "{{decision}}" + "expr": "sum by (decision, reason) (rate(aether_mempool_profit_scored_total{job=\"aether-host-scorer\"}[5m]))", + "legendFormat": "{{decision}} / {{reason}}" } ], "gridPos": {