fpm_rs/tabular/tables/
benchmark.rs1use polars::prelude::*;
2
3use crate::{
4 Result,
5 benchmark::{BenchmarkFrameRecord, BenchmarkRecord},
6};
7
8pub fn benchmark_runs_dataframe(records: &[BenchmarkRecord]) -> Result<DataFrame> {
10 let length = records.len();
11 let mut run_ids = Vec::with_capacity(length);
12 let mut case_ids = Vec::with_capacity(length);
13 let mut algorithms = Vec::with_capacity(length);
14 let mut algorithm_configurations = Vec::with_capacity(length);
15 let mut dataset_names = Vec::with_capacity(length);
16 let mut dataset_versions = Vec::with_capacity(length);
17 let mut random_seeds = Vec::with_capacity(length);
18 let mut frame_counts = Vec::with_capacity(length);
19 let mut completed_iterations = Vec::with_capacity(length);
20 let mut elapsed_seconds = Vec::with_capacity(length);
21 let mut final_objectives = Vec::with_capacity(length);
22 let mut successes = Vec::with_capacity(length);
23 let mut errors = Vec::with_capacity(length);
24 let mut repetitions = Vec::with_capacity(length);
25 let mut warmups = Vec::with_capacity(length);
26 let mut groups = Vec::with_capacity(length);
27 let mut reference_run_ids = Vec::with_capacity(length);
28 for record in records {
29 run_ids.push(record.run_id.as_str());
30 case_ids.push(record.case_id.as_str());
31 algorithms.push(record.algorithm.as_str());
32 algorithm_configurations.push(record.algorithm_configuration.as_str());
33 dataset_names.push(record.dataset_name.as_str());
34 dataset_versions.push(record.dataset_version.as_deref());
35 random_seeds.push(record.random_seed);
36 frame_counts.push(record.frame_count as u64);
37 completed_iterations.push(record.success.then_some(record.completed_iterations as u64));
38 elapsed_seconds.push(record.elapsed_seconds);
39 final_objectives.push(record.success.then_some(record.final_objective).flatten());
40 successes.push(record.success);
41 errors.push(
42 (!record.success)
43 .then_some(record.error.as_deref())
44 .flatten(),
45 );
46 repetitions.push(
47 record
48 .metadata
49 .get("repetition")
50 .and_then(|value| value.parse::<u64>().ok()),
51 );
52 warmups.push(
53 record
54 .metadata
55 .get("warmup")
56 .and_then(|value| value.parse::<bool>().ok()),
57 );
58 groups.push(record.metadata.get("benchmark_group").map(String::as_str));
59 reference_run_ids.push(record.metadata.get("reference_run_id").map(String::as_str));
60 }
61 Ok(df!(
62 "run_id" => run_ids,
63 "case_id" => case_ids,
64 "algorithm" => algorithms,
65 "algorithm_configuration" => algorithm_configurations,
66 "dataset_name" => dataset_names,
67 "dataset_version" => dataset_versions,
68 "random_seed" => random_seeds,
69 "frame_count" => frame_counts,
70 "crop_row" => records.iter().map(|r| r.spatial_crop.map(|c| c[0] as u64)).collect::<Vec<_>>(),
71 "crop_column" => records.iter().map(|r| r.spatial_crop.map(|c| c[1] as u64)).collect::<Vec<_>>(),
72 "crop_height" => records.iter().map(|r| r.spatial_crop.map(|c| c[2] as u64)).collect::<Vec<_>>(),
73 "crop_width" => records.iter().map(|r| r.spatial_crop.map(|c| c[3] as u64)).collect::<Vec<_>>(),
74 "completed_iterations" => completed_iterations,
75 "elapsed_seconds" => elapsed_seconds,
76 "final_objective" => final_objectives,
77 "success" => successes,
78 "error" => errors,
79 "repetition" => repetitions,
80 "warmup" => warmups,
81 "benchmark_group" => groups,
82 "reference_run_id" => reference_run_ids,
83 )?)
84}
85
86pub fn benchmark_frames_dataframe(records: &[BenchmarkRecord]) -> Result<DataFrame> {
88 let length = records.iter().map(|record| record.frames.len()).sum();
89 let mut run_ids = Vec::with_capacity(length);
90 let mut frame_indices = Vec::with_capacity(length);
91 let mut original_frame_indices = Vec::with_capacity(length);
92 let mut original_illumination_indices = Vec::with_capacity(length);
93 let mut normalized_l2 = Vec::with_capacity(length);
94 for record in records {
95 for BenchmarkFrameRecord {
96 frame_index,
97 original_frame_index,
98 original_illumination_index,
99 normalized_l2: residual,
100 } in &record.frames
101 {
102 run_ids.push(record.run_id.as_str());
103 frame_indices.push(*frame_index as u64);
104 original_frame_indices.push(*original_frame_index as u64);
105 original_illumination_indices
106 .push(original_illumination_index.map(|value| value as u64));
107 normalized_l2.push(*residual);
108 }
109 }
110 Ok(df!(
111 "run_id" => run_ids,
112 "frame_index" => frame_indices,
113 "original_frame_index" => original_frame_indices,
114 "original_illumination_index" => original_illumination_indices,
115 "normalized_l2" => normalized_l2,
116 )?)
117}
118
119pub fn benchmark_artifacts_dataframe(
121 records: &[BenchmarkRecord],
122 relative_result_paths: &std::collections::BTreeMap<String, String>,
123) -> Result<DataFrame> {
124 let length = records.len();
125 let mut run_ids = Vec::with_capacity(length);
126 let mut roles = Vec::with_capacity(length);
127 let mut relative_paths = Vec::with_capacity(length);
128 for record in records {
129 if let Some(path) = relative_result_paths.get(&record.run_id) {
130 run_ids.push(record.run_id.as_str());
131 roles.push("result_bundle");
132 relative_paths.push(path.as_str());
133 }
134 }
135 Ok(df!(
136 "run_id" => run_ids,
137 "role" => roles,
138 "relative_path" => relative_paths,
139 )?)
140}
141
142pub fn benchmark_metadata_dataframe(records: &[BenchmarkRecord]) -> Result<DataFrame> {
144 const PROMOTED: &[&str] = &[
145 "repetition",
146 "warmup",
147 "benchmark_group",
148 "reference_run_id",
149 ];
150 let length = records
151 .iter()
152 .map(|record| record.metadata.len())
153 .sum::<usize>();
154 let mut run_ids = Vec::with_capacity(length);
155 let mut keys = Vec::with_capacity(length);
156 let mut values = Vec::with_capacity(length);
157 for record in records {
158 for (key, value) in &record.metadata {
159 if PROMOTED.contains(&key.as_str()) {
160 continue;
161 }
162 run_ids.push(record.run_id.as_str());
163 keys.push(key.as_str());
164 values.push(value.as_str());
165 }
166 }
167 Ok(df!(
168 "run_id" => run_ids,
169 "key" => keys,
170 "value" => values,
171 )?)
172}