Skip to main content

fpm_rs/tabular/tables/
benchmark.rs

1use polars::prelude::*;
2
3use crate::{
4    Result,
5    benchmark::{BenchmarkFrameRecord, BenchmarkRecord},
6};
7
8/// Builds one stable summary row per benchmark run.
9pub fn benchmark_runs_dataframe(records: &[BenchmarkRecord]) -> Result<DataFrame> {
10    let length = records.len();
11    let mut run_ids = Vec::with_capacity(length);
12    let mut case_ids = Vec::with_capacity(length);
13    let mut algorithms = Vec::with_capacity(length);
14    let mut algorithm_configurations = Vec::with_capacity(length);
15    let mut dataset_names = Vec::with_capacity(length);
16    let mut dataset_versions = Vec::with_capacity(length);
17    let mut random_seeds = Vec::with_capacity(length);
18    let mut frame_counts = Vec::with_capacity(length);
19    let mut completed_iterations = Vec::with_capacity(length);
20    let mut elapsed_seconds = Vec::with_capacity(length);
21    let mut final_objectives = Vec::with_capacity(length);
22    let mut successes = Vec::with_capacity(length);
23    let mut errors = Vec::with_capacity(length);
24    let mut repetitions = Vec::with_capacity(length);
25    let mut warmups = Vec::with_capacity(length);
26    let mut groups = Vec::with_capacity(length);
27    let mut reference_run_ids = Vec::with_capacity(length);
28    for record in records {
29        run_ids.push(record.run_id.as_str());
30        case_ids.push(record.case_id.as_str());
31        algorithms.push(record.algorithm.as_str());
32        algorithm_configurations.push(record.algorithm_configuration.as_str());
33        dataset_names.push(record.dataset_name.as_str());
34        dataset_versions.push(record.dataset_version.as_deref());
35        random_seeds.push(record.random_seed);
36        frame_counts.push(record.frame_count as u64);
37        completed_iterations.push(record.success.then_some(record.completed_iterations as u64));
38        elapsed_seconds.push(record.elapsed_seconds);
39        final_objectives.push(record.success.then_some(record.final_objective).flatten());
40        successes.push(record.success);
41        errors.push(
42            (!record.success)
43                .then_some(record.error.as_deref())
44                .flatten(),
45        );
46        repetitions.push(
47            record
48                .metadata
49                .get("repetition")
50                .and_then(|value| value.parse::<u64>().ok()),
51        );
52        warmups.push(
53            record
54                .metadata
55                .get("warmup")
56                .and_then(|value| value.parse::<bool>().ok()),
57        );
58        groups.push(record.metadata.get("benchmark_group").map(String::as_str));
59        reference_run_ids.push(record.metadata.get("reference_run_id").map(String::as_str));
60    }
61    Ok(df!(
62        "run_id" => run_ids,
63        "case_id" => case_ids,
64        "algorithm" => algorithms,
65        "algorithm_configuration" => algorithm_configurations,
66        "dataset_name" => dataset_names,
67        "dataset_version" => dataset_versions,
68        "random_seed" => random_seeds,
69        "frame_count" => frame_counts,
70        "crop_row" => records.iter().map(|r| r.spatial_crop.map(|c| c[0] as u64)).collect::<Vec<_>>(),
71        "crop_column" => records.iter().map(|r| r.spatial_crop.map(|c| c[1] as u64)).collect::<Vec<_>>(),
72        "crop_height" => records.iter().map(|r| r.spatial_crop.map(|c| c[2] as u64)).collect::<Vec<_>>(),
73        "crop_width" => records.iter().map(|r| r.spatial_crop.map(|c| c[3] as u64)).collect::<Vec<_>>(),
74        "completed_iterations" => completed_iterations,
75        "elapsed_seconds" => elapsed_seconds,
76        "final_objective" => final_objectives,
77        "success" => successes,
78        "error" => errors,
79        "repetition" => repetitions,
80        "warmup" => warmups,
81        "benchmark_group" => groups,
82        "reference_run_id" => reference_run_ids,
83    )?)
84}
85
86/// Builds one row per recorded acquisition-frame benchmark metric.
87pub fn benchmark_frames_dataframe(records: &[BenchmarkRecord]) -> Result<DataFrame> {
88    let length = records.iter().map(|record| record.frames.len()).sum();
89    let mut run_ids = Vec::with_capacity(length);
90    let mut frame_indices = Vec::with_capacity(length);
91    let mut original_frame_indices = Vec::with_capacity(length);
92    let mut original_illumination_indices = Vec::with_capacity(length);
93    let mut normalized_l2 = Vec::with_capacity(length);
94    for record in records {
95        for BenchmarkFrameRecord {
96            frame_index,
97            original_frame_index,
98            original_illumination_index,
99            normalized_l2: residual,
100        } in &record.frames
101        {
102            run_ids.push(record.run_id.as_str());
103            frame_indices.push(*frame_index as u64);
104            original_frame_indices.push(*original_frame_index as u64);
105            original_illumination_indices
106                .push(original_illumination_index.map(|value| value as u64));
107            normalized_l2.push(*residual);
108        }
109    }
110    Ok(df!(
111        "run_id" => run_ids,
112        "frame_index" => frame_indices,
113        "original_frame_index" => original_frame_indices,
114        "original_illumination_index" => original_illumination_indices,
115        "normalized_l2" => normalized_l2,
116    )?)
117}
118
119/// Builds one result-bundle artifact row per run with a supplied relative path.
120pub fn benchmark_artifacts_dataframe(
121    records: &[BenchmarkRecord],
122    relative_result_paths: &std::collections::BTreeMap<String, String>,
123) -> Result<DataFrame> {
124    let length = records.len();
125    let mut run_ids = Vec::with_capacity(length);
126    let mut roles = Vec::with_capacity(length);
127    let mut relative_paths = Vec::with_capacity(length);
128    for record in records {
129        if let Some(path) = relative_result_paths.get(&record.run_id) {
130            run_ids.push(record.run_id.as_str());
131            roles.push("result_bundle");
132            relative_paths.push(path.as_str());
133        }
134    }
135    Ok(df!(
136        "run_id" => run_ids,
137        "role" => roles,
138        "relative_path" => relative_paths,
139    )?)
140}
141
142/// Builds long-form non-promoted key/value metadata rows for benchmark runs.
143pub fn benchmark_metadata_dataframe(records: &[BenchmarkRecord]) -> Result<DataFrame> {
144    const PROMOTED: &[&str] = &[
145        "repetition",
146        "warmup",
147        "benchmark_group",
148        "reference_run_id",
149    ];
150    let length = records
151        .iter()
152        .map(|record| record.metadata.len())
153        .sum::<usize>();
154    let mut run_ids = Vec::with_capacity(length);
155    let mut keys = Vec::with_capacity(length);
156    let mut values = Vec::with_capacity(length);
157    for record in records {
158        for (key, value) in &record.metadata {
159            if PROMOTED.contains(&key.as_str()) {
160                continue;
161            }
162            run_ids.push(record.run_id.as_str());
163            keys.push(key.as_str());
164            values.push(value.as_str());
165        }
166    }
167    Ok(df!(
168        "run_id" => run_ids,
169        "key" => keys,
170        "value" => values,
171    )?)
172}