Skip to main content

fpm_rs/reconstruction/
spectral_checkpoint.rs

1//! Typed iteration-boundary persistence for spectral AP and shared-OPD descent.
2
3use super::{
4    OpticalPathDifferenceResult, ReconstructionTrace, SpectralFrameSchedule,
5    SpectralReconstructionProblem,
6};
7use crate::{
8    Error, Result, array_serde::Array2Data, measurements::MeasurementRead,
9    model::SpectralImagePlaneModel,
10};
11use ndarray::{Array2, ArrayView2};
12use num_complex::Complex64;
13use serde::{Deserialize, Serialize};
14use sha2::{Digest, Sha256};
15use std::{
16    fs::{self, File},
17    io::{BufReader, BufWriter, Write},
18    path::{Path, PathBuf},
19};
20
21/// Version of the separately tagged spectral checkpoint format.
22/// Ordinary reconstruction checkpoint versions are independent.
23pub const SPECTRAL_CHECKPOINT_FORMAT_VERSION: u32 = 1;
24const KIND: &str = "fpm_rs.spectral_checkpoint";
25
26/// Optional periodic checkpoint output at accepted iteration boundaries.
27/// A checkpoint is also retained in supported solver results when no directory is set.
28#[derive(Clone, Debug)]
29pub struct SpectralCheckpointOptions {
30    /// Optional output directory, created as needed. Existing checkpoint files with
31    /// the same solver kind and iteration number are atomically replaced.
32    pub directory: Option<PathBuf>,
33    /// Positive interval in complete passes or accepted OPD updates; default one.
34    pub every: usize,
35}
36impl Default for SpectralCheckpointOptions {
37    fn default() -> Self {
38        Self {
39            directory: None,
40            every: 1,
41        }
42    }
43}
44impl SpectralCheckpointOptions {
45    /// Rejects a zero interval or an empty directory path.
46    pub fn validate(&self) -> Result<()> {
47        if self.every == 0
48            || self
49                .directory
50                .as_ref()
51                .is_some_and(|p| p.as_os_str().is_empty())
52        {
53            return Err(invalid(
54                "checkpoint_options",
55                "interval must be positive and directory nonempty",
56            ));
57        }
58        Ok(())
59    }
60    pub(crate) fn write(
61        &self,
62        checkpoint: &SpectralReconstructionCheckpoint,
63        final_state: bool,
64    ) -> Result<()> {
65        if let Some(directory) = &self.directory
66            && (final_state || checkpoint.completed_iterations.is_multiple_of(self.every))
67        {
68            fs::create_dir_all(directory)?;
69            let prefix = match checkpoint.state {
70                StoredSolver::Spectral { .. } => "spectral",
71                StoredSolver::JointOpd { .. } => "joint_opd",
72            };
73            checkpoint.save(directory.join(format!(
74                "{prefix}_checkpoint_{:05}.json",
75                checkpoint.completed_iterations
76            )))?;
77        }
78        Ok(())
79    }
80}
81
82/// Validated snapshot of a compiled spectral problem and one solver's numerical state.
83///
84/// Spectral state stores centered spectra directly. Joint state stores amplitudes,
85/// OPD bounds and the fixed gauge, and retains automatic initialization records.
86/// Resume checks the exact ordered compiled model and a streaming SHA-256 of
87/// detector values, masks and weights. A changed total iteration target is allowed;
88/// stepping options and explicitly selected schedules must match.
89/// Scientific arrays and objectives resume exactly; elapsed wall times accumulate.
90/// The format rejects ordinary checkpoints and other spectral solver kinds.
91///
92/// # Example
93/// ```
94/// use fpm_rs::{Result, algorithms::SpectralAlternatingProjection,
95///     measurements::MeasurementRead, reconstruction::{SpectralRunner,
96///     SpectralReconstructionCheckpoint, SpectralReconstructionProblem,
97///     SpectralReconstructionResult}};
98/// fn resume<M: MeasurementRead>(problem: &SpectralReconstructionProblem<M>,
99///     checkpoint: SpectralReconstructionCheckpoint) -> Result<SpectralReconstructionResult> {
100///     SpectralRunner::new(SpectralAlternatingProjection::default().iterations(50))
101///         .with_checkpoint(checkpoint).run(problem)
102/// }
103/// ```
104/// The example requires saved default stepping options and a target of at least
105/// the completed pass count. The saved schedule is inherited.
106
107#[derive(Clone, Debug, Serialize, Deserialize)]
108#[serde(deny_unknown_fields)]
109pub struct SpectralReconstructionCheckpoint {
110    kind: String,
111    format_version: u32,
112    pub(crate) model: SpectralImagePlaneModel,
113    model_fingerprint: String,
114    measurement_fingerprint: String,
115    pub(crate) completed_iterations: usize,
116    pub(crate) elapsed_seconds: f64,
117    pub(crate) trace: ReconstructionTrace,
118    pub(crate) state: StoredSolver,
119}
120
121#[derive(Clone, Debug, Serialize, Deserialize)]
122#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
123pub(crate) enum StoredSolver {
124    Spectral {
125        algorithm: String,
126        configuration: String,
127        spectra: Vec<Array2Data<Complex64>>,
128        schedule: SpectralFrameSchedule,
129    },
130    JointOpd {
131        configuration: String,
132        opd: Array2Data<f64>,
133        amplitudes: Vec<Array2Data<f64>>,
134        opd_range_m: (f64, f64),
135        gauge: crate::algorithms::Gauge,
136        initialization_opd: Option<Box<StoredOpd>>,
137        initialization_trace: Option<ReconstructionTrace>,
138    },
139}
140
141/// JSON representation of OPD diagnostics. Null is reserved for invalid NaN pixels.
142#[derive(Clone, Debug, Serialize, Deserialize)]
143#[serde(deny_unknown_fields)]
144pub(crate) struct StoredOpd {
145    opd_m: Array2Data<Option<f64>>,
146    valid_mask: Array2Data<u8>,
147    phase_residual_rad: Array2Data<Option<f64>>,
148    fringe_orders: Vec<Array2Data<i64>>,
149    phase_offsets_rad: Vec<f64>,
150    wavelengths_vacuum_m: Vec<f64>,
151    wavelength_ladder_m: Vec<f64>,
152}
153impl StoredOpd {
154    pub(crate) fn from_result(result: &OpticalPathDifferenceResult) -> Self {
155        let optional = |a: &Array2<f64>| Array2Data {
156            height: a.nrows(),
157            width: a.ncols(),
158            data: a
159                .iter()
160                .map(|&v| if v.is_nan() { None } else { Some(v) })
161                .collect(),
162        };
163        Self {
164            opd_m: optional(&result.opd_m),
165            valid_mask: Array2Data::from_view(result.valid_mask.view()),
166            phase_residual_rad: optional(&result.phase_residual_rad),
167            fringe_orders: result
168                .fringe_orders
169                .iter()
170                .map(|a| Array2Data::from_view(a.view()))
171                .collect(),
172            phase_offsets_rad: result.phase_offsets_rad.clone(),
173            wavelengths_vacuum_m: result.wavelengths_vacuum_m.clone(),
174            wavelength_ladder_m: result.wavelength_ladder_m.clone(),
175        }
176    }
177    pub(crate) fn into_result(self) -> Result<OpticalPathDifferenceResult> {
178        let result = OpticalPathDifferenceResult {
179            opd_m: self.opd_m.into_array()?.mapv(|v| v.unwrap_or(f64::NAN)),
180            valid_mask: self.valid_mask.into_array()?,
181            phase_residual_rad: self
182                .phase_residual_rad
183                .into_array()?
184                .mapv(|v| v.unwrap_or(f64::NAN)),
185            fringe_orders: self
186                .fringe_orders
187                .into_iter()
188                .map(Array2Data::into_array)
189                .collect::<Result<_>>()?,
190            phase_offsets_rad: self.phase_offsets_rad,
191            wavelengths_vacuum_m: self.wavelengths_vacuum_m,
192            wavelength_ladder_m: self.wavelength_ladder_m,
193        };
194        validate_opd(&result)?;
195        Ok(result)
196    }
197}
198
199pub(crate) fn validate_opd(result: &OpticalPathDifferenceResult) -> Result<()> {
200    let shape = result.opd_m.dim();
201    crate::array_layout::checked_len_2d(shape)?;
202    let count = result.wavelengths_vacuum_m.len();
203    if count < 2
204        || result.valid_mask.dim() != shape
205        || result.phase_residual_rad.dim() != shape
206        || result.fringe_orders.len() != count
207        || result.phase_offsets_rad.len() != count
208        || result.fringe_orders.iter().any(|a| a.dim() != shape)
209    {
210        return Err(Error::InvalidShape(
211            "OPD persistence arrays and channel records disagree".into(),
212        ));
213    }
214    let mut wavelengths = result.wavelengths_vacuum_m.clone();
215    wavelengths.sort_by(f64::total_cmp);
216    if wavelengths.iter().any(|v| !v.is_finite() || *v <= 0.0)
217        || wavelengths.windows(2).any(|w| w[0] == w[1])
218        || result.phase_offsets_rad.iter().any(|v| !v.is_finite())
219        || result.wavelength_ladder_m.is_empty()
220        || result
221            .wavelength_ladder_m
222            .iter()
223            .any(|v| !v.is_finite() || *v <= 0.0)
224        || result.wavelength_ladder_m.windows(2).any(|w| w[0] <= w[1])
225    {
226        return Err(invalid(
227            "opd_metadata",
228            "wavelengths, ladder, and phase offsets are invalid",
229        ));
230    }
231    for ((&opd, &valid), &residual) in result
232        .opd_m
233        .iter()
234        .zip(&result.valid_mask)
235        .zip(&result.phase_residual_rad)
236    {
237        if valid > 1
238            || (valid == 1 && (!opd.is_finite() || !residual.is_finite() || residual < 0.0))
239            || (valid == 0 && (!opd.is_nan() || !residual.is_nan()))
240        {
241            return Err(invalid(
242                "opd_arrays",
243                "valid pixels must be finite and invalid pixels must be NaN",
244            ));
245        }
246    }
247    Ok(())
248}
249
250impl SpectralReconstructionCheckpoint {
251    pub(crate) fn capture(
252        model: SpectralImagePlaneModel,
253        fingerprints: &(String, String),
254        completed_iterations: usize,
255        elapsed_seconds: f64,
256        trace: ReconstructionTrace,
257        state: StoredSolver,
258    ) -> Self {
259        Self {
260            kind: KIND.into(),
261            format_version: SPECTRAL_CHECKPOINT_FORMAT_VERSION,
262            model,
263            model_fingerprint: fingerprints.0.clone(),
264            measurement_fingerprint: fingerprints.1.clone(),
265            completed_iterations,
266            elapsed_seconds,
267            trace,
268            state,
269        }
270    }
271    /// Returns the separately versioned format number.
272    pub fn format_version(&self) -> u32 {
273        self.format_version
274    }
275    /// Returns completed passes or accepted joint updates, including those before resume.
276    pub fn completed_iterations(&self) -> usize {
277        self.completed_iterations
278    }
279    /// Returns the solver name; it determines which resume entry point may consume this snapshot.
280    pub fn algorithm(&self) -> &str {
281        match &self.state {
282            StoredSolver::Spectral { algorithm, .. } => {
283                algorithm.rsplit("::").next().unwrap_or(algorithm)
284            }
285            StoredSolver::JointOpd { .. } => "MultiWavelengthGradientDescent",
286        }
287    }
288    /// Borrows the exact ordered compiled model, including fixed pupils and detector calibration.
289    pub fn model(&self) -> &SpectralImagePlaneModel {
290        &self.model
291    }
292    /// Borrows the accumulated objective trace; timing includes earlier runs.
293    pub fn trace(&self) -> &ReconstructionTrace {
294        &self.trace
295    }
296    /// Borrows the authoritative joint OPD map in metres, or returns None for spectral AP.
297    pub fn opd_m(&self) -> Option<ArrayView2<'_, f64>> {
298        match &self.state {
299            StoredSolver::JointOpd { opd, .. } => Some(
300                ArrayView2::from_shape((opd.height, opd.width), &opd.data)
301                    .expect("validated checkpoint OPD"),
302            ),
303            _ => None,
304        }
305    }
306    /// Validates and atomically saves JSON, preserving binary64 values through float-roundtrip parsing.
307    /// A failed write leaves an existing destination intact.
308    pub fn save(&self, path: impl AsRef<Path>) -> Result<()> {
309        self.validate()?;
310        let path = path.as_ref();
311        let parent = path
312            .parent()
313            .filter(|p| !p.as_os_str().is_empty())
314            .unwrap_or_else(|| Path::new("."));
315        let temporary = parent.join(format!(".spectral-checkpoint-{}.tmp", uuid::Uuid::new_v4()));
316        let result = (|| -> Result<()> {
317            let mut writer = BufWriter::new(File::create(&temporary)?);
318            serde_json::to_writer(&mut writer, self)?;
319            writer.flush()?;
320            writer.get_ref().sync_all()?;
321            fs::rename(&temporary, path)?;
322            Ok(())
323        })();
324        if result.is_err() {
325            let _ = fs::remove_file(&temporary);
326        }
327        result
328    }
329    /// Loads a spectral checkpoint, rejecting unknown fields, wrong kind/version, and invalid state.
330    pub fn load(path: impl AsRef<Path>) -> Result<Self> {
331        let checkpoint: Self = serde_json::from_reader(BufReader::new(File::open(path)?))?;
332        checkpoint.validate()?;
333        Ok(checkpoint)
334    }
335    /// Loads and verifies the model plus a streaming fingerprint of measurements, masks, and weights.
336    pub fn load_for_problem<M: MeasurementRead>(
337        path: impl AsRef<Path>,
338        problem: &SpectralReconstructionProblem<M>,
339    ) -> Result<Self> {
340        let checkpoint = Self::load(path)?;
341        checkpoint.validate_for_problem(problem)?;
342        Ok(checkpoint)
343    }
344    /// Checks metadata, array layouts/shapes, finite values, gauge, and contiguous trace numbering.
345    pub fn validate(&self) -> Result<()> {
346        if self.kind != KIND || self.format_version != SPECTRAL_CHECKPOINT_FORMAT_VERSION {
347            return Err(invalid(
348                "spectral_checkpoint",
349                "unsupported checkpoint kind or version",
350            ));
351        }
352        self.model.validate()?;
353        if self.model_fingerprint != model_fingerprint(&self.model)?
354            || !is_hash(&self.measurement_fingerprint)
355        {
356            return Err(invalid(
357                "spectral_checkpoint",
358                "model or measurement fingerprint is invalid",
359            ));
360        }
361        if !self.elapsed_seconds.is_finite() || self.elapsed_seconds < 0.0 {
362            return Err(invalid("elapsed_seconds", "must be finite and nonnegative"));
363        }
364        let shape = self.model.reconstruction_shape();
365        let joint = matches!(self.state, StoredSolver::JointOpd { .. });
366        let expected = self
367            .completed_iterations
368            .checked_add(usize::from(joint))
369            .ok_or_else(|| invalid("completed_iterations", "trace length overflow"))?;
370        if self.trace.iterations.len() != expected {
371            return Err(invalid(
372                "spectral_trace",
373                "trace length disagrees with completed iterations",
374            ));
375        }
376        let mut previous_time = 0.0;
377        for (index, row) in self.trace.iterations.iter().enumerate() {
378            if row.iteration != index + usize::from(!joint)
379                || !row.objective.is_finite()
380                || !row.elapsed_seconds.is_finite()
381                || row.elapsed_seconds < previous_time
382                || row.elapsed_seconds > self.elapsed_seconds
383            {
384                return Err(invalid(
385                    "spectral_trace",
386                    "trace numbering, objectives, or times are invalid",
387                ));
388            }
389            previous_time = row.elapsed_seconds;
390        }
391        if self
392            .trace
393            .algorithm_metrics
394            .iter()
395            .any(|m| !m.value.is_finite() || m.iteration > self.completed_iterations)
396        {
397            return Err(invalid("spectral_trace", "algorithm metrics are invalid"));
398        }
399        match &self.state {
400            StoredSolver::Spectral {
401                algorithm,
402                configuration,
403                spectra,
404                schedule,
405            } => {
406                if algorithm.trim().is_empty()
407                    || serde_json::from_str::<serde_json::Value>(configuration).is_err()
408                    || spectra.len() != self.model.object_count()
409                {
410                    return Err(invalid(
411                        "spectral_state",
412                        "invalid algorithm, options, or spectrum count",
413                    ));
414                }
415                schedule.order(self.model.frame_count(), self.completed_iterations)?;
416                for a in spectra {
417                    validate_array(a, shape, |v| v.re.is_finite() && v.im.is_finite())?;
418                }
419            }
420            StoredSolver::JointOpd {
421                configuration,
422                opd,
423                amplitudes,
424                opd_range_m,
425                gauge,
426                initialization_opd,
427                initialization_trace,
428            } => {
429                let options: crate::algorithms::MultiWavelengthGradientDescent =
430                    serde_json::from_str(configuration)?;
431                options.validate()?;
432                if self.model.object_coupling() != crate::model::ObjectCoupling::Independent
433                    || self.model.channels().len() < 2
434                    || amplitudes.len() != self.model.channels().len()
435                {
436                    return Err(invalid(
437                        "joint_state",
438                        "requires independent channel amplitudes and at least two channels",
439                    ));
440                }
441                super::SyntheticWavelengthUnwrapper::new(*opd_range_m)?;
442                validate_array(opd, shape, |v| {
443                    v.is_finite() && *v >= opd_range_m.0 && *v < opd_range_m.1
444                })?;
445                for a in amplitudes {
446                    validate_array(a, shape, |v| v.is_finite() && *v >= options.epsilon.sqrt())?;
447                }
448                gauge.validate(opd, *opd_range_m)?;
449                if initialization_opd.is_some() != initialization_trace.is_some() {
450                    return Err(invalid(
451                        "initialization",
452                        "OPD diagnostics and AP trace must be present together",
453                    ));
454                }
455                if let Some(stored) = initialization_opd {
456                    let result = stored.clone().into_result()?;
457                    if result.opd_m.dim() != shape
458                        || result.valid_mask.iter().any(|&v| v != 1)
459                        || result.wavelengths_vacuum_m
460                            != self
461                                .model
462                                .channels()
463                                .iter()
464                                .map(|c| c.model.sampling().wavelength.unwrap())
465                                .collect::<Vec<_>>()
466                    {
467                        return Err(invalid(
468                            "initialization_opd",
469                            "initialization must be valid on the common grid and channel order",
470                        ));
471                    }
472                }
473                if let Some(trace) = initialization_trace
474                    && (trace.iterations.len() != options.initialization_iterations
475                        || trace.iterations.iter().enumerate().any(|(i, r)| {
476                            r.iteration != i + 1
477                                || !r.objective.is_finite()
478                                || !r.elapsed_seconds.is_finite()
479                                || r.elapsed_seconds < 0.0
480                        }))
481                {
482                    return Err(invalid(
483                        "initialization_trace",
484                        "initialization trace is invalid",
485                    ));
486                }
487            }
488        }
489        Ok(())
490    }
491    /// Rejects changed ordered kernels, acquisition, detector data, masks, or weights.
492    /// Fingerprinting streams one detector frame at a time and performs no reconstruction.
493    pub fn validate_for_problem<M: MeasurementRead>(
494        &self,
495        problem: &SpectralReconstructionProblem<M>,
496    ) -> Result<()> {
497        self.validate()?;
498        let current = fingerprints(problem)?;
499        if current.0 != self.model_fingerprint || current.1 != self.measurement_fingerprint {
500            return Err(invalid(
501                "spectral_checkpoint",
502                "compiled spectral model or detector measurements changed",
503            ));
504        }
505        Ok(())
506    }
507    pub(crate) fn fingerprints(&self) -> (String, String) {
508        (
509            self.model_fingerprint.clone(),
510            self.measurement_fingerprint.clone(),
511        )
512    }
513}
514
515pub(crate) fn fingerprints<M: MeasurementRead>(
516    problem: &SpectralReconstructionProblem<M>,
517) -> Result<(String, String)> {
518    problem.validate()?;
519    let mut hash = Sha256::new();
520    hash.update(b"fpm_rs.spectral_measurements.v1");
521    hash.update((problem.measurements.frame_count() as u64).to_le_bytes());
522    let shape = problem.measurements.image_shape();
523    hash.update((shape.0 as u64).to_le_bytes());
524    hash.update((shape.1 as u64).to_le_bytes());
525    for frame in 0..problem.measurements.frame_count() {
526        hash.update(problem.measurements.frame_weight(frame)?.to_le_bytes());
527        let values = problem.measurements.frame(frame)?;
528        if values.len() != shape.0 * shape.1 || values.iter().any(|v| !v.is_finite()) {
529            return Err(Error::InvalidMeasurements(format!(
530                "frame {frame} must have the declared shape and finite detector values"
531            )));
532        }
533        for &value in values.iter() {
534            hash.update(value.to_le_bytes());
535        }
536        match problem.measurements.frame_mask(frame)? {
537            None => hash.update([0]),
538            Some(mask) => {
539                hash.update([1]);
540                hash.update(mask);
541            }
542        }
543    }
544    Ok((
545        model_fingerprint(&problem.model)?,
546        format!("{:x}", hash.finalize()),
547    ))
548}
549fn model_fingerprint(model: &SpectralImagePlaneModel) -> Result<String> {
550    Ok(format!("{:x}", Sha256::digest(serde_json::to_vec(model)?)))
551}
552fn is_hash(value: &str) -> bool {
553    value.len() == 64
554        && value
555            .bytes()
556            .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
557}
558fn validate_array<T>(
559    a: &Array2Data<T>,
560    shape: (usize, usize),
561    finite: impl Fn(&T) -> bool,
562) -> Result<()> {
563    let len = crate::array_layout::checked_len_2d(shape)?;
564    if (a.height, a.width) != shape || a.data.len() != len || a.data.iter().any(|v| !finite(v)) {
565        return Err(invalid(
566            "spectral_state_arrays",
567            "arrays must have the common shape and valid finite values",
568        ));
569    }
570    Ok(())
571}
572fn invalid(name: &'static str, reason: impl Into<String>) -> Error {
573    Error::InvalidParameter {
574        name,
575        reason: reason.into(),
576    }
577}