1#[cfg(feature = "parquet")]
8use std::fs;
9use std::{
10 collections::BTreeMap,
11 fs::File,
12 io::BufWriter,
13 path::{Path, PathBuf},
14 time::Instant,
15};
16
17use ndarray::ArrayView2;
18use serde::{Deserialize, Serialize};
19use uuid::Uuid;
20
21use crate::{
22 Complex64, Result,
23 algorithms::ReconstructionAlgorithm,
24 datasets::DatasetSubset,
25 evaluation::{evaluate_frame_intensity, evaluate_reconstruction_with_problem},
26 measurements::{FrameMetadata, MeasurementRead},
27 model::ImagePlaneModel,
28 reconstruction::{ReconstructionProblem, ReconstructionResult},
29};
30
31pub const BENCHMARK_RECORD_FORMAT_VERSION: u32 = 1;
33pub const SMOKE_BENCHMARK_PROFILE: &str = "smoke";
35pub const CPU_BENCHMARK_PROFILE: &str = "cpu";
37
38#[derive(Clone, Copy, Debug, Eq, PartialEq)]
44pub struct BenchmarkProfile {
45 pub name: &'static str,
47 pub description: &'static str,
49 pub expected_runtime: &'static str,
51 pub output_directory: &'static str,
53 pub algorithms: &'static [&'static str],
55}
56
57impl BenchmarkProfile {
58 pub fn output_path(&self) -> PathBuf {
60 PathBuf::from(self.output_directory)
61 }
62}
63
64pub const BENCHMARK_PROFILES: &[BenchmarkProfile] = &[
66 BenchmarkProfile {
67 name: SMOKE_BENCHMARK_PROFILE,
68 description: "offline synthetic sanity profile for all implemented CPU algorithms",
69 expected_runtime: "under 1 minute on a typical laptop CPU",
70 output_directory: "target/benchmark-results/smoke",
71 algorithms: &[
72 "AlternatingProjection",
73 "AdaptiveAlternatingProjection",
74 "Fpie",
75 "Mpie",
76 "Epry",
77 "Admm",
78 "GradientDescent",
79 ],
80 },
81 BenchmarkProfile {
82 name: CPU_BENCHMARK_PROFILE,
83 description: "offline synthetic CPU comparison with longer iteration counts",
84 expected_runtime: "1-5 minutes on a typical laptop CPU",
85 output_directory: "target/benchmark-results/cpu",
86 algorithms: &[
87 "AlternatingProjection",
88 "AdaptiveAlternatingProjection",
89 "Fpie",
90 "Mpie",
91 "Epry",
92 "Admm",
93 "GradientDescent",
94 ],
95 },
96];
97
98pub fn benchmark_profile(name: &str) -> Option<&'static BenchmarkProfile> {
100 BENCHMARK_PROFILES
101 .iter()
102 .find(|profile| profile.name == name)
103}
104
105pub fn annotate_benchmark_profile(record: &mut BenchmarkRecord, profile: &BenchmarkProfile) {
107 record
108 .metadata
109 .insert("benchmark_profile".into(), profile.name.into());
110 record.metadata.insert(
111 "benchmark_profile_expected_runtime".into(),
112 profile.expected_runtime.into(),
113 );
114 record.metadata.insert(
115 "benchmark_profile_output_directory".into(),
116 profile.output_directory.into(),
117 );
118}
119
120#[derive(Clone, Debug, Serialize, Deserialize)]
122#[serde(deny_unknown_fields)]
123pub struct BenchmarkRecord {
124 pub format_version: u32,
126 pub case_id: String,
128 pub run_id: String,
130 pub dataset_name: String,
132 pub dataset_version: Option<String>,
134 pub preset_name: Option<String>,
136 pub crate_version: String,
138 pub random_seed: Option<u64>,
140 pub spatial_crop: Option<[usize; 4]>,
142 pub algorithm: String,
144 pub algorithm_configuration: String,
146 pub success: bool,
148 pub error: Option<String>,
150 pub frame_count: usize,
152 pub image_shape: [usize; 2],
154 pub reconstruction_shape: [usize; 2],
156 pub completed_iterations: usize,
158 pub elapsed_seconds: f64,
160 pub initial_objective: Option<f64>,
162 pub final_objective: Option<f64>,
164 pub final_to_initial_objective_ratio: Option<f64>,
166 pub amplitude_rmse: Option<f64>,
168 pub phase_rmse: Option<f64>,
170 pub complex_field_relative_error: Option<f64>,
172 pub fourier_domain_relative_error: Option<f64>,
174 pub pupil_amplitude_rmse: Option<f64>,
176 pub pupil_phase_rmse: Option<f64>,
178 pub illumination_position_rmse: Option<f64>,
180 pub per_frame_residual_mean: Option<f64>,
182 pub per_frame_residual_max: Option<f64>,
184 pub frames: Vec<BenchmarkFrameRecord>,
186 pub output_paths: Vec<PathBuf>,
188 pub metadata: BTreeMap<String, String>,
190}
191
192#[derive(Clone, Debug, Serialize, Deserialize)]
194#[serde(deny_unknown_fields)]
195pub struct BenchmarkFrameRecord {
196 pub frame_index: usize,
198 pub original_frame_index: usize,
200 pub original_illumination_index: Option<usize>,
205 pub normalized_l2: Option<f64>,
207}
208
209impl BenchmarkRecord {
210 pub fn from_subset_result(
217 case_id: impl Into<String>,
218 dataset_name: impl Into<String>,
219 algorithm_configuration: impl Into<String>,
220 result: &ReconstructionResult,
221 subset: &DatasetSubset,
222 ) -> Result<Self> {
223 let mut record = Self::from_result(case_id, dataset_name, algorithm_configuration, result);
224 if record.frame_count != subset.measurements().frame_count()
225 || record.image_shape != [subset.spatial_crop().height, subset.spatial_crop().width]
226 || record.reconstruction_shape
227 != [
228 subset.configuration().reconstruction_shape.0,
229 subset.configuration().reconstruction_shape.1,
230 ]
231 {
232 return Err(crate::Error::InvalidParameter {
233 name: "dataset subset",
234 reason: "result frame count and shapes must match the supplied subset".into(),
235 });
236 }
237 annotate_subset(&mut record, subset);
238 Ok(record)
239 }
240
241 pub fn from_result(
246 case_id: impl Into<String>,
247 dataset_name: impl Into<String>,
248 algorithm_configuration: impl Into<String>,
249 result: &ReconstructionResult,
250 ) -> Self {
251 let image_shape = result.recovered_pupil.shape();
252 let reconstruction_shape = result.object.dim();
253 let frame_count = result
254 .metadata
255 .get("frame_count")
256 .and_then(|value| value.parse().ok())
257 .or_else(|| result.recovered_frame_gains.as_ref().map(Vec::len))
258 .unwrap_or(0);
259 let initial_objective = result
260 .trace
261 .iterations
262 .first()
263 .map(|record| record.objective);
264 let final_objective = result.trace.final_objective();
265 Self {
266 format_version: BENCHMARK_RECORD_FORMAT_VERSION,
267 case_id: case_id.into(),
268 run_id: Uuid::new_v4().to_string(),
269 dataset_name: dataset_name.into(),
270 dataset_version: result.metadata.get("dataset_version").cloned(),
271 preset_name: result.metadata.get("preset_name").cloned(),
272 crate_version: env!("CARGO_PKG_VERSION").into(),
273 random_seed: result
274 .metadata
275 .get("random_seed")
276 .and_then(|value| value.parse().ok()),
277 spatial_crop: None,
278 algorithm: result.runtime.algorithm.clone(),
279 algorithm_configuration: algorithm_configuration.into(),
280 success: true,
281 error: None,
282 frame_count,
283 image_shape: [image_shape.0, image_shape.1],
284 reconstruction_shape: [reconstruction_shape.0, reconstruction_shape.1],
285 completed_iterations: result.runtime.completed_iterations,
286 elapsed_seconds: result.runtime.elapsed_seconds,
287 initial_objective,
288 final_objective,
289 final_to_initial_objective_ratio: initial_objective.and_then(|initial| {
290 final_objective
291 .filter(|_| initial.abs() > f64::EPSILON)
292 .map(|final_value| final_value / initial)
293 }),
294 amplitude_rmse: None,
295 phase_rmse: None,
296 complex_field_relative_error: None,
297 fourier_domain_relative_error: None,
298 pupil_amplitude_rmse: None,
299 pupil_phase_rmse: None,
300 illumination_position_rmse: None,
301 per_frame_residual_mean: None,
302 per_frame_residual_max: None,
303 frames: (0..frame_count)
304 .map(|frame_index| BenchmarkFrameRecord {
305 frame_index,
306 original_frame_index: frame_index,
307 original_illumination_index: None,
308 normalized_l2: None,
309 })
310 .collect(),
311 output_paths: Vec::new(),
312 metadata: BTreeMap::new(),
313 }
314 }
315}
316
317pub fn run_benchmark_subset_case<A: ReconstructionAlgorithm>(
342 dataset_name: impl Into<String>,
343 algorithm_configuration: impl Into<String>,
344 algorithm: A,
345 subset: &DatasetSubset,
346) -> Result<(BenchmarkRecord, Option<ReconstructionResult>)> {
347 let problem = subset.reconstruction_problem()?;
348 let (mut record, mut result) = run_benchmark_case(
349 dataset_name,
350 algorithm_configuration,
351 algorithm,
352 &problem,
353 subset.ground_truth_object().map(|value| value.view()),
354 Some(&subset.configuration().compiled_models.true_model),
355 subset.valid_object_mask().map(|value| value.view()),
356 );
357 annotate_subset(&mut record, subset);
358 record.case_id = format!("{:016x}", case_hash(&record));
359 if let Some(result) = &mut result {
360 result.metadata.extend(record.metadata.clone());
361 result
362 .metadata
363 .insert("case_id".into(), record.case_id.clone());
364 result
365 .metadata
366 .insert("dataset_name".into(), record.dataset_name.clone());
367 result.metadata.insert(
368 "algorithm_configuration".into(),
369 record.algorithm_configuration.clone(),
370 );
371 if let Some(version) = &record.dataset_version {
372 result
373 .metadata
374 .insert("dataset_version".into(), version.clone());
375 }
376 }
377 Ok((record, result))
378}
379
380fn annotate_subset(record: &mut BenchmarkRecord, subset: &DatasetSubset) {
381 let crop = subset.spatial_crop();
382 record.spatial_crop = Some([crop.row, crop.column, crop.height, crop.width]);
383 for (frame, metadata) in record
384 .frames
385 .iter_mut()
386 .zip(subset.measurements().frame_metadata())
387 {
388 frame.original_frame_index = metadata.original_frame_index.unwrap_or(frame.frame_index);
389 frame.original_illumination_index = original_illumination_index(metadata);
390 }
391 record.metadata.extend(subset.provenance().clone());
392 if let Some(units) = subset.measurement_units() {
393 record
394 .metadata
395 .insert("measurement_units".into(), units.into());
396 }
397 record.dataset_version = subset
398 .provenance()
399 .get("dataset_version")
400 .cloned()
401 .or_else(|| record.dataset_version.clone());
402}
403
404fn original_illumination_index(metadata: &FrameMetadata) -> Option<usize> {
405 if metadata.original_frame_index.is_some() {
408 metadata.original_illumination_index
409 } else {
410 metadata
411 .original_illumination_index
412 .or(metadata.illumination_index)
413 }
414}
415
416pub fn run_benchmark_case<A, M>(
420 dataset_name: impl Into<String>,
421 algorithm_configuration: impl Into<String>,
422 algorithm: A,
423 problem: &ReconstructionProblem<M>,
424 ground_truth: Option<ArrayView2<'_, Complex64>>,
425 true_model: Option<&ImagePlaneModel>,
426 valid_object_mask: Option<ArrayView2<'_, u8>>,
427) -> (BenchmarkRecord, Option<ReconstructionResult>)
428where
429 A: ReconstructionAlgorithm,
430 M: MeasurementRead,
431{
432 let algorithm_name = short_type_name::<A>().to_owned();
433 let image_shape = problem.measurements.image_shape();
434 let reconstruction_shape = problem.model.reconstruction_shape;
435 let mut record = BenchmarkRecord {
436 format_version: BENCHMARK_RECORD_FORMAT_VERSION,
437 case_id: String::new(),
438 run_id: Uuid::new_v4().to_string(),
439 dataset_name: dataset_name.into(),
440 dataset_version: None,
441 preset_name: None,
442 crate_version: env!("CARGO_PKG_VERSION").into(),
443 random_seed: None,
444 spatial_crop: None,
445 algorithm: algorithm_name,
446 algorithm_configuration: algorithm_configuration.into(),
447 success: false,
448 error: None,
449 frame_count: problem.measurements.frame_count(),
450 image_shape: [image_shape.0, image_shape.1],
451 reconstruction_shape: [reconstruction_shape.0, reconstruction_shape.1],
452 completed_iterations: 0,
453 elapsed_seconds: 0.0,
454 initial_objective: None,
455 final_objective: None,
456 final_to_initial_objective_ratio: None,
457 amplitude_rmse: None,
458 phase_rmse: None,
459 complex_field_relative_error: None,
460 fourier_domain_relative_error: None,
461 pupil_amplitude_rmse: None,
462 pupil_phase_rmse: None,
463 illumination_position_rmse: None,
464 per_frame_residual_mean: None,
465 per_frame_residual_max: None,
466 frames: problem
467 .measurements
468 .frame_metadata()
469 .iter()
470 .enumerate()
471 .map(|(index, metadata)| BenchmarkFrameRecord {
472 frame_index: index,
473 original_frame_index: metadata.original_frame_index.unwrap_or(index),
474 original_illumination_index: original_illumination_index(metadata),
475 normalized_l2: None,
476 })
477 .collect(),
478 output_paths: Vec::new(),
479 metadata: BTreeMap::new(),
480 };
481 record.case_id = format!("{:016x}", case_hash(&record));
482
483 let started = Instant::now();
484 let result = match algorithm.run(problem) {
485 Ok(result) => result,
486 Err(error) => {
487 record.elapsed_seconds = started.elapsed().as_secs_f64();
488 record.error = Some(error.to_string());
489 return (record, None);
490 }
491 };
492 record.elapsed_seconds = started.elapsed().as_secs_f64();
493 record.algorithm = result.runtime.algorithm.clone();
494 record.completed_iterations = result.runtime.completed_iterations;
495 record.initial_objective = result.trace.iterations.first().map(|entry| entry.objective);
496 record.final_objective = result.trace.final_objective();
497 record.final_to_initial_objective_ratio = record.initial_objective.and_then(|initial| {
498 record
499 .final_objective
500 .filter(|_| initial.abs() > f64::EPSILON)
501 .map(|final_objective| final_objective / initial)
502 });
503
504 let residuals: Vec<f64> = if let Some(truth) = ground_truth {
505 let metrics = evaluate_reconstruction_with_problem(
506 &result,
507 problem,
508 truth,
509 true_model,
510 valid_object_mask,
511 );
512 match metrics {
513 Ok(metrics) => {
514 record.amplitude_rmse = Some(metrics.object.amplitude_rmse);
515 record.phase_rmse = Some(metrics.object.phase_rmse);
516 record.complex_field_relative_error = Some(metrics.object.complex_nrmse);
517 record.fourier_domain_relative_error = Some(metrics.object.fourier_nrmse);
518 record.pupil_amplitude_rmse =
519 metrics.pupil.as_ref().map(|value| value.amplitude_rmse);
520 record.pupil_phase_rmse = metrics.pupil.as_ref().map(|value| value.phase_rmse);
521 record.illumination_position_rmse = metrics
522 .illumination
523 .as_ref()
524 .map(|value| value.position_rmse);
525 metrics
526 .intensity
527 .map(|value| {
528 value
529 .per_frame
530 .into_iter()
531 .map(|frame| frame.normalized_l2)
532 .collect()
533 })
534 .unwrap_or_default()
535 }
536 Err(error) => {
537 record.error = Some(format!("benchmark metric calculation failed: {error}"));
538 return (record, Some(result));
539 }
540 }
541 } else {
542 match evaluate_frame_intensity(&result, problem) {
543 Ok(metrics) => metrics
544 .per_frame
545 .into_iter()
546 .map(|frame| frame.normalized_l2)
547 .collect(),
548 Err(error) => {
549 record.error = Some(format!("benchmark residual calculation failed: {error}"));
550 return (record, Some(result));
551 }
552 }
553 };
554 if !residuals.is_empty() {
555 record.per_frame_residual_mean =
556 Some(residuals.iter().sum::<f64>() / residuals.len() as f64);
557 record.per_frame_residual_max = residuals.iter().copied().reduce(f64::max);
558 }
559 for (frame, residual) in record.frames.iter_mut().zip(residuals) {
560 frame.normalized_l2 = Some(residual);
561 }
562 let mut result = result;
563 result
564 .metadata
565 .insert("case_id".into(), record.case_id.clone());
566 result
567 .metadata
568 .insert("dataset_name".into(), record.dataset_name.clone());
569 if let Some(version) = &record.dataset_version {
570 result
571 .metadata
572 .insert("dataset_version".into(), version.clone());
573 }
574 result.metadata.insert(
575 "algorithm_configuration".into(),
576 record.algorithm_configuration.clone(),
577 );
578 result
579 .metadata
580 .insert("frame_count".into(), record.frame_count.to_string());
581 if let Some(seed) = record.random_seed {
582 result
583 .metadata
584 .insert("random_seed".into(), seed.to_string());
585 }
586 record.success = true;
587 (record, Some(result))
588}
589
590#[cfg(feature = "parquet")]
593pub fn save_benchmark_outputs(
594 record: &mut BenchmarkRecord,
595 result: &ReconstructionResult,
596 directory: impl AsRef<Path>,
597) -> Result<()> {
598 let directory = directory.as_ref();
599 fs::create_dir_all(directory)?;
600 let stem = format!(
601 "{}-{}-{:016x}",
602 safe_stem(&record.dataset_name),
603 safe_stem(&record.algorithm),
604 case_hash(record),
605 );
606 let outputs = [
607 directory.join(format!("{stem}-amplitude.png")),
608 directory.join(format!("{stem}-phase.png")),
609 directory.join(format!("{stem}-result")),
610 directory.join(format!("{stem}-trace.csv")),
611 ];
612 result.save_amplitude(&outputs[0])?;
613 result.save_phase(&outputs[1])?;
614 result.write_bundle(
615 &outputs[2],
616 crate::reconstruction::BundleExportOptions {
617 run_id: Some(record.run_id.clone()),
618 label: None,
619 include_previews: true,
620 },
621 )?;
622 result.save_trace_csv(&outputs[3])?;
623 record.output_paths.extend(outputs);
624 Ok(())
625}
626
627#[cfg(not(feature = "parquet"))]
628pub fn save_benchmark_outputs(
639 _record: &mut BenchmarkRecord,
640 _result: &ReconstructionResult,
641 _directory: impl AsRef<Path>,
642) -> Result<()> {
643 Err(crate::Error::Unsupported(
644 "benchmark result bundles require the `parquet` feature".into(),
645 ))
646}
647
648pub fn write_benchmark_json(records: &[BenchmarkRecord], path: impl AsRef<Path>) -> Result<()> {
650 #[derive(Serialize)]
651 struct Report<'a> {
652 format_version: u32,
653 records: &'a [BenchmarkRecord],
654 }
655
656 let writer = BufWriter::new(File::create(path)?);
657 serde_json::to_writer_pretty(
658 writer,
659 &Report {
660 format_version: BENCHMARK_RECORD_FORMAT_VERSION,
661 records,
662 },
663 )?;
664 Ok(())
665}
666
667pub fn write_benchmark_csv(records: &[BenchmarkRecord], path: impl AsRef<Path>) -> Result<()> {
669 let mut writer = csv::Writer::from_path(path)?;
670 writer.write_record([
671 "format_version",
672 "case_id",
673 "run_id",
674 "dataset_name",
675 "dataset_version",
676 "preset_name",
677 "crate_version",
678 "random_seed",
679 "spatial_crop",
680 "algorithm",
681 "algorithm_configuration",
682 "success",
683 "error",
684 "frame_count",
685 "image_height",
686 "image_width",
687 "reconstruction_height",
688 "reconstruction_width",
689 "completed_iterations",
690 "elapsed_seconds",
691 "initial_objective",
692 "final_objective",
693 "final_to_initial_objective_ratio",
694 "amplitude_rmse",
695 "phase_rmse",
696 "complex_field_relative_error",
697 "fourier_domain_relative_error",
698 "pupil_amplitude_rmse",
699 "pupil_phase_rmse",
700 "illumination_position_rmse",
701 "per_frame_residual_mean",
702 "per_frame_residual_max",
703 "frames_json",
704 "output_paths",
705 "metadata_json",
706 ])?;
707 for record in records {
708 writer.write_record([
709 record.format_version.to_string(),
710 record.case_id.clone(),
711 record.run_id.clone(),
712 record.dataset_name.clone(),
713 record.dataset_version.clone().unwrap_or_default(),
714 record.preset_name.clone().unwrap_or_default(),
715 record.crate_version.clone(),
716 record
717 .random_seed
718 .map_or_else(String::new, |seed| seed.to_string()),
719 record.spatial_crop.map_or_else(String::new, |crop| {
720 crop.iter()
721 .map(usize::to_string)
722 .collect::<Vec<_>>()
723 .join(";")
724 }),
725 record.algorithm.clone(),
726 record.algorithm_configuration.clone(),
727 record.success.to_string(),
728 record.error.clone().unwrap_or_default(),
729 record.frame_count.to_string(),
730 record.image_shape[0].to_string(),
731 record.image_shape[1].to_string(),
732 record.reconstruction_shape[0].to_string(),
733 record.reconstruction_shape[1].to_string(),
734 record.completed_iterations.to_string(),
735 record.elapsed_seconds.to_string(),
736 optional_number(record.initial_objective),
737 optional_number(record.final_objective),
738 optional_number(record.final_to_initial_objective_ratio),
739 optional_number(record.amplitude_rmse),
740 optional_number(record.phase_rmse),
741 optional_number(record.complex_field_relative_error),
742 optional_number(record.fourier_domain_relative_error),
743 optional_number(record.pupil_amplitude_rmse),
744 optional_number(record.pupil_phase_rmse),
745 optional_number(record.illumination_position_rmse),
746 optional_number(record.per_frame_residual_mean),
747 optional_number(record.per_frame_residual_max),
748 serde_json::to_string(&record.frames)?,
749 record
750 .output_paths
751 .iter()
752 .map(|path| path.display().to_string())
753 .collect::<Vec<_>>()
754 .join(";"),
755 serde_json::to_string(&record.metadata)?,
756 ])?;
757 }
758 writer.flush()?;
759 Ok(())
760}
761
762fn short_type_name<T>() -> &'static str {
763 std::any::type_name::<T>()
764 .rsplit("::")
765 .next()
766 .unwrap_or("reconstruction algorithm")
767}
768
769fn optional_number(value: Option<f64>) -> String {
770 value.map_or_else(String::new, |value| value.to_string())
771}
772
773#[cfg(feature = "parquet")]
774fn safe_stem(value: &str) -> String {
775 let stem: String = value
776 .chars()
777 .map(|character| {
778 if character.is_ascii_alphanumeric() || matches!(character, '-' | '_') {
779 character
780 } else {
781 '_'
782 }
783 })
784 .collect();
785 if stem.is_empty() {
786 "benchmark".into()
787 } else {
788 stem
789 }
790}
791
792fn case_hash(record: &BenchmarkRecord) -> u64 {
793 let mut hash = 0xcbf29ce484222325_u64;
796 let mut update = |bytes: &[u8]| {
797 for &byte in bytes {
798 hash ^= u64::from(byte);
799 hash = hash.wrapping_mul(0x100000001b3);
800 }
801 hash ^= 0xff;
802 hash = hash.wrapping_mul(0x100000001b3);
803 };
804 update(record.dataset_name.as_bytes());
805 update(
806 record
807 .dataset_version
808 .as_deref()
809 .unwrap_or_default()
810 .as_bytes(),
811 );
812 update(record.preset_name.as_deref().unwrap_or_default().as_bytes());
813 update(record.algorithm.as_bytes());
814 update(record.algorithm_configuration.as_bytes());
815 for frame in &record.frames {
816 update(&frame.original_frame_index.to_le_bytes());
817 update(
818 &frame
819 .original_illumination_index
820 .unwrap_or(usize::MAX)
821 .to_le_bytes(),
822 );
823 }
824 if let Some(crop) = record.spatial_crop {
825 for value in crop {
826 update(&value.to_le_bytes());
827 }
828 }
829 hash
830}