openvm_prof/
summary.rs

1use std::{io::Write, path::PathBuf};
2
3use eyre::Result;
4use itertools::Itertools;
5
6use crate::{
7    aggregate::{
8        AggregateMetrics, EXECUTE_METERED_TIME_LABEL, EXECUTE_PREFLIGHT_INSNS_LABEL,
9        EXECUTE_PREFLIGHT_TIME_LABEL, MAIN_CELLS_USED_LABEL, PROOF_TIME_LABEL,
10        PROVE_EXCL_TRACE_TIME_LABEL, TRACE_GEN_TIME_LABEL,
11    },
12    types::MdTableCell,
13};
14
15#[derive(Clone, Debug)]
16pub struct GithubSummary {
17    pub rows: Vec<SummaryRow>,
18    pub benchmark_results_link: String,
19    pub omit_cells_used: bool,
20}
21
22#[derive(Clone, Debug)]
23pub struct SummaryRow {
24    pub name: String,
25    pub md_filename: String,
26    pub metrics: BenchSummaryMetrics,
27}
28
29#[derive(Clone, Debug)]
30pub struct BenchSummaryMetrics {
31    pub app: SingleSummaryMetrics,
32    pub leaf: Option<SingleSummaryMetrics>,
33    pub internals: Vec<SingleSummaryMetrics>,
34    pub root: Option<SingleSummaryMetrics>,
35    pub halo2_outer: Option<SingleSummaryMetrics>,
36    pub halo2_wrapper: Option<SingleSummaryMetrics>,
37}
38
39#[derive(Clone, Debug)]
40pub struct SingleSummaryMetrics {
41    pub proof_time_ms: MdTableCell,
42    /// Parallel proof time is approximated as the max of proof times within a group
43    pub par_proof_time_ms: MdTableCell,
44    pub cells_used: MdTableCell,
45    pub insns: MdTableCell,
46}
47
48impl GithubSummary {
49    pub fn new(
50        names: &[String],
51        aggregated_metrics: &[(AggregateMetrics, Option<AggregateMetrics>)],
52        md_paths: &[PathBuf],
53        benchmark_results_link: &str,
54    ) -> Self {
55        let omit_cells_used = aggregated_metrics
56            .iter()
57            .all(|(aggregated, _)| !aggregated.has_cells_used_metrics());
58        let rows = aggregated_metrics
59            .iter()
60            .zip_eq(md_paths.iter())
61            .zip_eq(names)
62            .filter_map(|(((aggregated, prev_aggregated), md_path), name)| {
63                let md_filename = md_path
64                    .file_name()
65                    .expect("Path should have a filename")
66                    .to_str()
67                    .expect("Filename should be valid UTF-8");
68                let mut row = match aggregated.get_summary_row(md_filename) {
69                    Some(row) => row,
70                    None => {
71                        eprintln!(
72                            "Warning: skipping benchmark '{}' ({}): no grouped metrics found",
73                            name, md_filename
74                        );
75                        return None;
76                    }
77                };
78                if let Some(prev_aggregated) = prev_aggregated {
79                    // md_filename doesn't matter
80                    if let Some(prev_row) = prev_aggregated.get_summary_row(md_filename) {
81                        if row.name == prev_row.name {
82                            row.metrics.set_diff(&prev_row.metrics);
83                        }
84                    }
85                }
86                row.name = name.clone();
87                Some(row)
88            })
89            .collect();
90
91        Self {
92            rows,
93            benchmark_results_link: benchmark_results_link.to_string(),
94            omit_cells_used,
95        }
96    }
97
98    pub fn write_markdown(&self, writer: &mut impl Write) -> Result<()> {
99        let include_cells_used = !self.omit_cells_used;
100        if include_cells_used {
101            writeln!(writer, "| group | app.proof_time_ms | app.cycles | app.cells_used | leaf.proof_time_ms | leaf.cells_used |")?;
102        } else {
103            writeln!(
104                writer,
105                "| group | app.proof_time_ms | app.cycles | leaf.proof_time_ms |"
106            )?;
107        }
108        write!(writer, "| -- |")?;
109        let trailing_columns = if include_cells_used { 5 } else { 3 };
110        for _ in 0..trailing_columns {
111            write!(writer, " -- |")?;
112        }
113        writeln!(writer)?;
114
115        for row in self.rows.iter() {
116            write!(
117                writer,
118                "| [{}]({}/{}) |",
119                row.name, self.benchmark_results_link, row.md_filename
120            )?;
121            row.metrics
122                .write_partial_md_row(writer, include_cells_used)?;
123            writeln!(writer)?;
124        }
125        writeln!(writer)?;
126        if self.omit_cells_used {
127            writeln!(
128                writer,
129                "Note: cells_used metrics omitted because CUDA tracegen does not expose unpadded trace heights."
130            )?;
131            writeln!(writer)?;
132        }
133
134        Ok(())
135    }
136}
137
138impl BenchSummaryMetrics {
139    pub fn write_partial_md_row(
140        &self,
141        writer: &mut impl Write,
142        include_cells_used: bool,
143    ) -> Result<()> {
144        if include_cells_used {
145            let (leaf_proof_time, leaf_cells_used) = match &self.leaf {
146                Some(leaf) => (leaf.proof_time_ms.to_string(), leaf.cells_used.to_string()),
147                None => ("-".to_string(), "-".to_string()),
148            };
149            self.app.write_partial_md_row(writer)?;
150            write!(writer, "{} | {} |", leaf_proof_time, leaf_cells_used)?;
151        } else {
152            let leaf_proof_time = self
153                .leaf
154                .as_ref()
155                .map(|leaf| leaf.proof_time_ms.to_string())
156                .unwrap_or_else(|| "-".to_string());
157            write!(
158                writer,
159                "{} | {} | {} |",
160                self.app.proof_time_ms, self.app.insns, leaf_proof_time
161            )?;
162        }
163        // Don't print other metrics in summary for now:
164
165        // for internal in &self.internals {
166        //     internal.write_partial_md_row(writer)?;
167        // }
168        // if let Some(root) = &self.root {
169        //     root.write_partial_md_row(writer)?;
170        // }
171
172        Ok(())
173    }
174
175    pub fn set_diff(&mut self, prev: &Self) {
176        self.app.set_diff(&prev.app);
177        if let (Some(leaf), Some(prev_leaf)) = (&mut self.leaf, &prev.leaf) {
178            leaf.set_diff(prev_leaf);
179        }
180        for (internal, prev_internal) in self.internals.iter_mut().zip(prev.internals.iter()) {
181            internal.set_diff(prev_internal);
182        }
183        if let (Some(root), Some(prev_root)) = (&mut self.root, &prev.root) {
184            root.set_diff(prev_root);
185        }
186    }
187}
188
189impl SingleSummaryMetrics {
190    pub fn write_partial_md_row(&self, writer: &mut impl Write) -> Result<()> {
191        write!(
192            writer,
193            "{} | {} | {} |",
194            self.proof_time_ms, self.insns, self.cells_used,
195        )?;
196        Ok(())
197    }
198
199    pub fn set_diff(&mut self, prev: &Self) {
200        self.cells_used.diff = Some(self.cells_used.val - prev.cells_used.val);
201        self.insns.diff = Some(self.insns.val - prev.insns.val);
202        self.proof_time_ms.diff = Some(self.proof_time_ms.val - prev.proof_time_ms.val);
203    }
204}
205
206impl AggregateMetrics {
207    pub fn has_cells_used_metrics(&self) -> bool {
208        self.by_group
209            .values()
210            .any(|stats| stats.contains_key(MAIN_CELLS_USED_LABEL))
211    }
212
213    pub fn get_single_summary(&self, name: &str) -> Option<SingleSummaryMetrics> {
214        let stats = self.by_group.get(name)?;
215        // Any group must have proof_time, but may not have cells_used or cycles (e.g., halo2)
216        let proof_time_ms = if let Some(proof_stats) = stats.get(PROOF_TIME_LABEL) {
217            proof_stats.sum
218        } else {
219            // Note: execute_metered is outside any segment scope, so it should have sum = max = avg
220            let execute_metered = stats
221                .get(EXECUTE_METERED_TIME_LABEL)
222                .map(|s| s.sum.val)
223                .unwrap_or(0.0);
224            let execute_preflight = stats
225                .get(EXECUTE_PREFLIGHT_TIME_LABEL)
226                .map(|s| s.sum.val)
227                .unwrap_or(0.0);
228            // If total_proof_time_ms is not available, compute it from components
229            let trace_gen = stats
230                .get(TRACE_GEN_TIME_LABEL)
231                .map(|s| s.sum.val)
232                .unwrap_or(0.0);
233            let stark_prove = stats
234                .get(PROVE_EXCL_TRACE_TIME_LABEL)
235                .map(|s| s.sum.val)
236                .unwrap_or(0.0);
237            println!("{execute_metered} {execute_preflight} {trace_gen} {stark_prove}");
238            MdTableCell::new(
239                execute_metered + execute_preflight + trace_gen + stark_prove,
240                None,
241            )
242        };
243        println!("{}", self.total_proof_time.val);
244        let par_proof_time_ms = if let Some(proof_stats) = stats.get(PROOF_TIME_LABEL) {
245            proof_stats.max
246        } else {
247            // Use the same computation for max
248            let execute_metered = stats
249                .get(EXECUTE_METERED_TIME_LABEL)
250                .map(|s| s.max.val)
251                .unwrap_or(0.0);
252            let execute_preflight = stats
253                .get(EXECUTE_PREFLIGHT_TIME_LABEL)
254                .map(|s| s.max.val)
255                .unwrap_or(0.0);
256            let trace_gen = stats
257                .get(TRACE_GEN_TIME_LABEL)
258                .map(|s| s.max.val)
259                .unwrap_or(0.0);
260            let stark_prove = stats
261                .get(PROVE_EXCL_TRACE_TIME_LABEL)
262                .map(|s| s.max.val)
263                .unwrap_or(0.0);
264            MdTableCell::new(
265                execute_metered + execute_preflight + trace_gen + stark_prove,
266                None,
267            )
268        };
269        let cells_used = stats
270            .get(MAIN_CELLS_USED_LABEL)
271            .map(|s| s.sum)
272            .unwrap_or_default();
273        let insns = stats
274            .get(EXECUTE_PREFLIGHT_INSNS_LABEL)
275            .map(|s| s.sum)
276            .unwrap_or_default();
277        Some(SingleSummaryMetrics {
278            cells_used,
279            insns,
280            proof_time_ms,
281            par_proof_time_ms,
282        })
283    }
284
285    /// Returns `None` if no group for app is found.
286    pub fn get_summary_row(&self, md_filename: &str) -> Option<SummaryRow> {
287        let app_name = self.name()?;
288        let app = self.get_single_summary(&app_name)?;
289        let leaf = self.get_single_summary("leaf");
290        let mut internals = Vec::new();
291        let mut hgt = 0;
292        while let Some(internal) = self.get_single_summary(&format!("internal.{hgt}")) {
293            internals.push(internal);
294            hgt += 1;
295        }
296        let root = self.get_single_summary("root");
297        let halo2_outer = self.get_single_summary("halo2_outer");
298        let halo2_wrapper = self.get_single_summary("halo2_wrapper");
299        Some(SummaryRow {
300            name: app_name,
301            md_filename: md_filename.to_string(),
302            metrics: BenchSummaryMetrics {
303                app,
304                leaf,
305                internals,
306                root,
307                halo2_outer,
308                halo2_wrapper,
309            },
310        })
311    }
312}
openvm_prof/summary.rs

openvm_prof/
summary.rs
