arb_bench/report/
markdown.rs

1use std::fmt::Write as _;
2
3use crate::{
4    metrics::RunResult,
5    report::compare::{BootstrapDelta, ComparisonReport, MetricKey, Verdict},
6    runner::abba::AbbaResult,
7};
8
9/// Markdown summary suitable for posting as a PR comment.
10pub fn render_abba(result: &AbbaResult) -> String {
11    let mut s = String::new();
12    let _ = writeln!(s, "## arb-bench: `{}`", result.manifest_name);
13    let _ = writeln!(s, "- Iterations: {}", result.iterations);
14    let _ = writeln!(s, "- Verdict: **{}**", result.verdict.label());
15    if let Verdict::Regression { metric, delta_pct } = &result.verdict {
16        let _ = writeln!(s, "- Regressing metric: `{metric}` (Δ {delta_pct:+.2}%)");
17    }
18    let _ = writeln!(s);
19    let _ = writeln!(
20        s,
21        "| Metric | Baseline | Feature | Δ mean | 95% CI | Verdict |"
22    );
23    let _ = writeln!(s, "|---|---|---|---|---|---|");
24    for (key, d) in &result.deltas {
25        let row = render_metric_row(*key, d);
26        let _ = writeln!(s, "{row}");
27    }
28    s
29}
30
31pub fn render_comparison(report: &ComparisonReport) -> String {
32    let mut s = String::new();
33    let _ = writeln!(s, "## arb-bench compare: `{}`", report.manifest_name);
34    let _ = writeln!(s, "- Verdict: **{}**", report.verdict.label());
35    let _ = writeln!(s);
36    let _ = writeln!(s, "| Metric | Baseline | Feature | Δ mean | 95% CI |");
37    let _ = writeln!(s, "|---|---|---|---|---|");
38    for (key, d) in &report.deltas {
39        let row = render_metric_row(*key, d);
40        let _ = writeln!(s, "{row}");
41    }
42    s
43}
44
45pub fn render_run_summary(r: &RunResult) -> String {
46    let mut s = String::new();
47    let _ = writeln!(s, "### `{}`", r.manifest_name);
48    let _ = writeln!(s, "- Blocks: {}", r.summary.block_count);
49    let _ = writeln!(s, "- Total gas: {}", r.summary.total_gas);
50    let _ = writeln!(s, "- Mean gas/sec: {:.0}", r.summary.gas_per_sec_mean);
51    let _ = writeln!(
52        s,
53        "- p50 / p95 / p99 wall (ms): {:.2} / {:.2} / {:.2}",
54        r.summary.wall_clock_ns_p50 as f64 / 1.0e6,
55        r.summary.wall_clock_ns_p95 as f64 / 1.0e6,
56        r.summary.wall_clock_ns_p99 as f64 / 1.0e6
57    );
58    let _ = writeln!(
59        s,
60        "- Peak RSS: {:.1} MiB",
61        r.summary.peak_rss_bytes as f64 / 1024.0 / 1024.0
62    );
63    if r.summary.monotonic_slowdown_detected {
64        let _ = writeln!(s, "- ⚠️  Monotonic slowdown detected.");
65    }
66    s
67}
68
69fn render_metric_row(key: MetricKey, d: &BootstrapDelta) -> String {
70    let (baseline_fmt, feature_fmt, delta_fmt, ci_fmt, verdict_label) = match key {
71        MetricKey::WallClockNs => {
72            let bl = d.baseline_mean / 1.0e6;
73            let ft = d.feature_mean / 1.0e6;
74            let pct = if d.baseline_mean > 0.0 {
75                d.mean / d.baseline_mean * 100.0
76            } else {
77                0.0
78            };
79            let label = direction_label(pct, false);
80            (
81                format!("{bl:.2} ms"),
82                format!("{ft:.2} ms"),
83                format!("{:+.2} ms ({pct:+.2}%)", d.mean / 1.0e6),
84                format!(
85                    "[{:+.2}, {:+.2}] ms",
86                    d.ci_low_95 / 1.0e6,
87                    d.ci_high_95 / 1.0e6
88                ),
89                label,
90            )
91        }
92        MetricKey::CpuNs => {
93            let bl = d.baseline_mean / 1.0e6;
94            let ft = d.feature_mean / 1.0e6;
95            let pct = if d.baseline_mean > 0.0 {
96                d.mean / d.baseline_mean * 100.0
97            } else {
98                0.0
99            };
100            let label = direction_label(pct, false);
101            (
102                format!("{bl:.2} ms"),
103                format!("{ft:.2} ms"),
104                format!("{:+.2} ms ({pct:+.2}%)", d.mean / 1.0e6),
105                format!(
106                    "[{:+.2}, {:+.2}] ms",
107                    d.ci_low_95 / 1.0e6,
108                    d.ci_high_95 / 1.0e6
109                ),
110                label,
111            )
112        }
113        MetricKey::GasPerSec => {
114            let bl = d.baseline_mean / 1.0e6;
115            let ft = d.feature_mean / 1.0e6;
116            let pct = if d.baseline_mean > 0.0 {
117                d.mean / d.baseline_mean * 100.0
118            } else {
119                0.0
120            };
121            let label = direction_label(pct, true);
122            (
123                format!("{bl:.1} Mgas/s"),
124                format!("{ft:.1} Mgas/s"),
125                format!("{:+.1} Mgas/s ({pct:+.2}%)", d.mean / 1.0e6),
126                format!(
127                    "[{:+.1}, {:+.1}] Mgas/s",
128                    d.ci_low_95 / 1.0e6,
129                    d.ci_high_95 / 1.0e6
130                ),
131                label,
132            )
133        }
134        MetricKey::RssBytes => {
135            let bl = d.baseline_mean / (1024.0 * 1024.0);
136            let ft = d.feature_mean / (1024.0 * 1024.0);
137            let pct = if d.baseline_mean > 0.0 {
138                d.mean / d.baseline_mean * 100.0
139            } else {
140                0.0
141            };
142            let label = direction_label(pct, false);
143            (
144                format!("{bl:.1} MiB"),
145                format!("{ft:.1} MiB"),
146                format!("{:+.1} MiB ({pct:+.2}%)", d.mean / (1024.0 * 1024.0)),
147                format!(
148                    "[{:+.1}, {:+.1}] MiB",
149                    d.ci_low_95 / (1024.0 * 1024.0),
150                    d.ci_high_95 / (1024.0 * 1024.0)
151                ),
152                label,
153            )
154        }
155    };
156    format!(
157        "| {key:?} | {baseline_fmt} | {feature_fmt} | {delta_fmt} | {ci_fmt} | {verdict_label} |"
158    )
159}
160
161fn direction_label(pct: f64, higher_is_better: bool) -> String {
162    let improvement = (pct < 0.0 && !higher_is_better) || (pct > 0.0 && higher_is_better);
163    let regression = (pct > 0.0 && !higher_is_better) || (pct < 0.0 && higher_is_better);
164    if pct.abs() < 0.5 {
165        "≈".into()
166    } else if improvement {
167        "↑ better".into()
168    } else if regression {
169        "↓ worse".into()
170    } else {
171        "≈".into()
172    }
173}
174
175#[cfg(test)]
176mod tests {
177    use super::*;
178    use crate::metrics::SummaryMetrics;
179
180    #[test]
181    fn render_compare_well_formed() {
182        let report = ComparisonReport {
183            manifest_name: "t".into(),
184            baseline_summary: SummaryMetrics::default(),
185            feature_summary: SummaryMetrics::default(),
186            deltas: vec![(
187                MetricKey::WallClockNs,
188                BootstrapDelta {
189                    n: 10,
190                    baseline_mean: 1.0e6,
191                    feature_mean: 9.0e5,
192                    mean: -1.0e5,
193                    ci_low_95: -2.0e5,
194                    ci_high_95: -1.0e4,
195                },
196            )],
197            verdict: Verdict::Improvement,
198        };
199        let s = render_comparison(&report);
200        assert!(s.contains("IMPROVEMENT"));
201        assert!(s.contains("WallClockNs"));
202    }
203}
arb_bench/report/markdown.rs

arb_bench/report/
markdown.rs
