Skip to main content

fpm_rs/tabular/parquet/
spectral.rs

1//! Separately tagged spectral result bundles with lossless per-channel arrays.
2use super::{
3    BundleArtifact, BundleExportOptions, BundleVerificationResult,
4    manifest::ManifestArtifact,
5    npy,
6    write::{sha256, unique_paths, validate_run_id, write_json_atomic},
7};
8use crate::{
9    Error, Result,
10    algorithms::MultiWavelengthSolverResult,
11    model::{ObjectCoupling, Pupil},
12    reconstruction::{
13        MultiWavelengthReconstructionResult, OpticalPathDifferenceResult, ReconstructionTrace,
14        RuntimeInfo, SpectralChannelResult, SpectralReconstructionCheckpoint,
15        SpectralReconstructionResult,
16        spectral_checkpoint::{StoredOpd, StoredSolver},
17    },
18};
19use ndarray::Array2;
20use polars::prelude::{DataFrame, KeyValueMetadata, ParquetReader, ParquetWriter, SerReader};
21use serde::{Deserialize, Serialize};
22use std::{
23    collections::{BTreeMap, BTreeSet},
24    fs::{self, File},
25    io::BufReader,
26    path::{Component, Path, PathBuf},
27    sync::{Arc, Mutex},
28};
29
30/// Version of the separately tagged spectral result-bundle manifest.
31pub const SPECTRAL_BUNDLE_FORMAT_VERSION: u32 = 1;
32const KIND: &str = "fpm_rs.spectral_result_bundle";
33#[derive(Clone, Debug, Serialize, Deserialize)]
34#[serde(deny_unknown_fields)]
35struct Channel {
36    channel_id: String,
37    wavelength_vacuum_m: f64,
38}
39#[derive(Clone, Debug, Serialize, Deserialize)]
40#[serde(deny_unknown_fields)]
41struct Manifest {
42    kind: String,
43    bundle_format_version: u32,
44    run_id: String,
45    label: Option<String>,
46    crate_version: String,
47    channels: Vec<Channel>,
48    object_coupling: ObjectCoupling,
49    image_shape: (usize, usize),
50    reconstruction_shape: (usize, usize),
51    runtime: RuntimeInfo,
52    trace: ReconstructionTrace,
53    has_joint_opd: bool,
54    has_unwrapped_opd: bool,
55    artifacts: Vec<ManifestArtifact>,
56}
57/// Reopened spectral bundle. Manifest metadata is eager; arrays/checkpoint are verified
58/// when requested. `result` caches an immutable owned result shared through `Arc`.
59/// Ordinary bundles must use `ResultBundle`. Failed exports keep an `.inprogress`
60/// directory and never publish a complete manifest at the requested destination.
61///
62/// # Example
63/// ```no_run
64/// use fpm_rs::{Result, reconstruction::SpectralReconstructionResult,
65///     tabular::parquet::{BundleExportOptions, read_spectral_bundle}};
66/// fn persist(result: &SpectralReconstructionResult) -> Result<()> {
67///     let bundle = result.write_bundle("spectral-result", BundleExportOptions::default())?;
68///     let reopened = read_spectral_bundle(bundle.path())?;
69///     reopened.verify()?;
70///     assert_eq!(reopened.result()?.channels.len(), result.channels.len());
71///     Ok(())
72/// }
73/// ```
74
75#[derive(Debug)]
76pub struct SpectralResultBundle {
77    root: PathBuf,
78    manifest: Manifest,
79    artifacts: BTreeMap<String, BundleArtifact>,
80    cache: Mutex<Option<Arc<SpectralReconstructionResult>>>,
81}
82/// Opens a spectral manifest without loading scientific arrays; rejects ordinary bundles.
83pub fn read_spectral_bundle(path: impl AsRef<Path>) -> Result<SpectralResultBundle> {
84    SpectralResultBundle::read(path)
85}
86impl SpectralResultBundle {
87    /// Validates kind/version, unique safe artifact paths, shapes, metadata and required roles.
88    pub fn read(path: impl AsRef<Path>) -> Result<Self> {
89        let root = path.as_ref().canonicalize()?;
90        let manifest: Manifest = serde_json::from_reader(BufReader::new(File::open(safe_file(
91            &root,
92            Path::new("manifest.json"),
93        )?)?))?;
94        if manifest.kind != KIND || manifest.bundle_format_version != SPECTRAL_BUNDLE_FORMAT_VERSION
95        {
96            return Err(invalid("unsupported spectral bundle kind/version"));
97        }
98        validate_run_id(&manifest.run_id)?;
99        crate::array_layout::checked_len_2d(manifest.image_shape)?;
100        crate::array_layout::checked_len_2d(manifest.reconstruction_shape)?;
101        if manifest.channels.is_empty()
102            || manifest.image_shape.0 == 0
103            || manifest.image_shape.1 == 0
104            || manifest.reconstruction_shape.0 == 0
105            || manifest.reconstruction_shape.1 == 0
106        {
107            return Err(invalid("empty channel list or grid"));
108        }
109        let mut ids = BTreeSet::new();
110        let mut wavelengths = Vec::new();
111        for c in &manifest.channels {
112            if c.channel_id.trim().is_empty()
113                || !ids.insert(&c.channel_id)
114                || !c.wavelength_vacuum_m.is_finite()
115                || c.wavelength_vacuum_m <= 0.0
116                || wavelengths.contains(&c.wavelength_vacuum_m)
117            {
118                return Err(invalid("invalid ordered channel metadata"));
119            }
120            wavelengths.push(c.wavelength_vacuum_m);
121        }
122        let expected = expected_roles(&manifest);
123        let mut artifacts = BTreeMap::new();
124        let mut paths = BTreeSet::new();
125        for a in &manifest.artifacts {
126            let path = safe_file(&root, &a.relative_path)?;
127            let descriptor = expected
128                .get(&a.role)
129                .ok_or_else(|| invalid("unknown artifact role"))?;
130            if artifacts.contains_key(&a.role)
131                || !paths.insert(path.clone())
132                || a.dtype != descriptor.0
133                || a.shape != descriptor.1
134                || a.media_type != descriptor.2
135                || a.sha256.len() != 64
136                || !a
137                    .sha256
138                    .bytes()
139                    .all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase())
140            {
141                return Err(invalid(
142                    "duplicate artifact or invalid dtype/shape/hash/media type",
143                ));
144            }
145            artifacts.insert(
146                a.role.clone(),
147                BundleArtifact {
148                    role: a.role.clone(),
149                    path,
150                    media_type: a.media_type.clone(),
151                    byte_size: a.byte_size,
152                    sha256: a.sha256.clone(),
153                    dtype: a.dtype.clone(),
154                    shape: a.shape.clone(),
155                },
156            );
157        }
158        if artifacts.len() != expected.len() {
159            return Err(invalid("required spectral artifacts missing"));
160        }
161        Ok(Self {
162            root,
163            manifest,
164            artifacts,
165            cache: Mutex::new(None),
166        })
167    }
168    /// Borrows the canonical published bundle directory.
169    pub fn path(&self) -> &Path {
170        &self.root
171    }
172    /// Borrows the run identity preserved in manifest and tables.
173    pub fn run_id(&self) -> &str {
174        &self.manifest.run_id
175    }
176    /// Borrows an optional caller label.
177    pub fn label(&self) -> Option<&str> {
178        self.manifest.label.as_deref()
179    }
180    /// Borrows verified manifest descriptors, sorted by semantic role.
181    pub fn artifacts(&self) -> &BTreeMap<String, BundleArtifact> {
182        &self.artifacts
183    }
184    fn artifact(&self, role: &str) -> Result<&BundleArtifact> {
185        let a = self
186            .artifacts
187            .get(role)
188            .ok_or_else(|| invalid("missing artifact"))?;
189        // Recheck containment on every access; a replaced symlink cannot escape the root.
190        safe_file(
191            &self.root,
192            a.path
193                .strip_prefix(&self.root)
194                .map_err(|_| invalid("artifact outside root"))?,
195        )?;
196        if fs::metadata(&a.path)?.len() != a.byte_size || sha256(&a.path)? != a.sha256 {
197            return Err(invalid(format!("artifact size/hash mismatch: {role}")));
198        }
199        Ok(a)
200    }
201    /// Loads the lossless resumable state, checking hash and exact manifest metadata.
202    pub fn checkpoint(&self) -> Result<SpectralReconstructionCheckpoint> {
203        let checkpoint =
204            SpectralReconstructionCheckpoint::load(&self.artifact("state.checkpoint")?.path)?;
205        let m = checkpoint.model();
206        if m.image_shape() != self.manifest.image_shape
207            || m.reconstruction_shape() != self.manifest.reconstruction_shape
208            || m.object_coupling() != self.manifest.object_coupling
209            || m.channels().len() != self.manifest.channels.len()
210            || m.channels()
211                .iter()
212                .zip(&self.manifest.channels)
213                .any(|(a, b)| {
214                    a.channel_id != b.channel_id
215                        || a.model.sampling().wavelength != Some(b.wavelength_vacuum_m)
216                })
217            || checkpoint.completed_iterations() != self.manifest.runtime.completed_iterations
218            || checkpoint.algorithm() != self.manifest.runtime.algorithm
219            || serde_json::to_value(checkpoint.trace())?
220                != serde_json::to_value(&self.manifest.trace)?
221            || checkpoint.elapsed_seconds != self.manifest.runtime.elapsed_seconds
222            || matches!(checkpoint.state, StoredSolver::JointOpd { .. })
223                != self.manifest.has_joint_opd
224        {
225            return Err(invalid("checkpoint and manifest disagree"));
226        }
227        Ok(checkpoint)
228    }
229    /// Loads per-channel NPY fields, amplitudes, phases, spectra and fixed pupils.
230    /// Returned arrays preserve saved binary64 values and are cached together.
231    pub fn result(&self) -> Result<Arc<SpectralReconstructionResult>> {
232        let mut cache = self
233            .cache
234            .lock()
235            .map_err(|_| invalid("result cache poisoned"))?;
236        if let Some(result) = &*cache {
237            return Ok(result.clone());
238        }
239        let checkpoint = self.checkpoint()?;
240        let mut channels = Vec::new();
241        for (i, c) in self.manifest.channels.iter().enumerate() {
242            let role = |name: &str| format!("channels.{i}.{name}");
243            let complex = |name: &str, shape| {
244                npy::read_complex2(&self.artifact(&role(name))?.path, &role(name), shape)
245            };
246            let real = |name: &str| -> Result<Array2<f64>> {
247                Ok(Array2::from_shape_vec(
248                    self.manifest.reconstruction_shape,
249                    npy::read_f64(
250                        &self.artifact(&role(name))?.path,
251                        &role(name),
252                        &[
253                            self.manifest.reconstruction_shape.0,
254                            self.manifest.reconstruction_shape.1,
255                        ],
256                    )?,
257                )?)
258            };
259            let object = complex("object", self.manifest.reconstruction_shape)?;
260            let amplitude = real("amplitude")?;
261            let phase = real("phase")?;
262            let object_spectrum = complex("object_spectrum", self.manifest.reconstruction_shape)?;
263            let pupil = Pupil::new(
264                complex("pupil", self.manifest.image_shape)?,
265                npy::read_u8_2(
266                    &self.artifact(&role("pupil_support"))?.path,
267                    &role("pupil_support"),
268                    self.manifest.image_shape,
269                )?,
270            )?;
271            channels.push(SpectralChannelResult {
272                channel_id: c.channel_id.clone(),
273                wavelength_vacuum_m: c.wavelength_vacuum_m,
274                object,
275                amplitude,
276                phase,
277                object_spectrum,
278                pupil,
279            });
280        }
281        let result = Arc::new(SpectralReconstructionResult {
282            channels,
283            object_coupling: self.manifest.object_coupling,
284            trace: self.manifest.trace.clone(),
285            runtime: self.manifest.runtime.clone(),
286            checkpoint: if self.manifest.has_joint_opd {
287                None
288            } else {
289                Some(checkpoint.clone())
290            },
291        });
292        validate_result(&result, &checkpoint)?;
293        *cache = Some(result.clone());
294        Ok(result)
295    }
296    /// Restores the authoritative joint OPD result and initialization records, or None for AP.
297    pub fn joint_result(&self) -> Result<Option<MultiWavelengthSolverResult>> {
298        if !self.manifest.has_joint_opd {
299            return Ok(None);
300        }
301        let checkpoint = self.checkpoint()?;
302        let opd_m = Array2::from_shape_vec(
303            self.manifest.reconstruction_shape,
304            npy::read_f64(
305                &self.artifact("arrays.opd_m")?.path,
306                "arrays.opd_m",
307                &[
308                    self.manifest.reconstruction_shape.0,
309                    self.manifest.reconstruction_shape.1,
310                ],
311            )?,
312        )?;
313        let StoredSolver::JointOpd {
314            opd,
315            initialization_opd,
316            initialization_trace,
317            ..
318        } = &checkpoint.state
319        else {
320            return Err(invalid("joint state missing"));
321        };
322        if opd.data != opd_m.iter().copied().collect::<Vec<_>>() {
323            return Err(invalid("OPD array differs from checkpoint"));
324        }
325        Ok(Some(MultiWavelengthSolverResult {
326            spectral: (*self.result()?).clone(),
327            opd_m,
328            initialization_opd: initialization_opd
329                .clone()
330                .map(|o| (*o).into_result())
331                .transpose()?,
332            initialization_trace: initialization_trace.clone(),
333            checkpoint,
334        }))
335    }
336    /// Restores optional synthetic-wavelength diagnostic arrays, including invalid NaN pixels.
337    pub fn unwrapped_opd(&self) -> Result<Option<OpticalPathDifferenceResult>> {
338        if !self.manifest.has_unwrapped_opd {
339            return Ok(None);
340        }
341        let stored: StoredOpd = serde_json::from_reader(BufReader::new(File::open(
342            &self.artifact("state.unwrapped_opd")?.path,
343        )?))?;
344        let opd = stored.into_result()?;
345        if opd.opd_m.dim() != self.manifest.reconstruction_shape
346            || opd.wavelengths_vacuum_m
347                != self
348                    .manifest
349                    .channels
350                    .iter()
351                    .map(|c| c.wavelength_vacuum_m)
352                    .collect::<Vec<_>>()
353        {
354            return Err(invalid("unwrapped OPD and channels disagree"));
355        }
356        Ok(Some(opd))
357    }
358    /// Checks every hash, numerical state and table against authoritative manifest records.
359    pub fn verify(&self) -> Result<BundleVerificationResult> {
360        let mut bytes = 0_u64;
361        for role in self.artifacts.keys() {
362            bytes = bytes
363                .checked_add(self.artifact(role)?.byte_size)
364                .ok_or_else(|| invalid("artifact byte count overflow"))?;
365        }
366        self.result()?;
367        self.joint_result()?;
368        self.unwrapped_opd()?;
369        for (role, expected) in [
370            (
371                "tables.history",
372                crate::tabular::history_dataframe(self.run_id(), &self.manifest.trace)?,
373            ),
374            (
375                "tables.algorithm_metrics",
376                crate::tabular::algorithm_metrics_dataframe(self.run_id(), &self.manifest.trace)?,
377            ),
378        ] {
379            let actual = ParquetReader::new(File::open(&self.artifact(role)?.path)?).finish()?;
380            if !actual.equals_missing(&expected) {
381                return Err(invalid(format!("table disagrees with trace: {role}")));
382            }
383        }
384        let channels = channel_table(self.run_id(), &self.manifest.channels)?;
385        if !ParquetReader::new(File::open(&self.artifact("tables.channels")?.path)?)
386            .finish()?
387            .equals_missing(&channels)
388        {
389            return Err(invalid("channel table disagrees with manifest"));
390        }
391        Ok(BundleVerificationResult {
392            artifact_count: self.artifacts.len(),
393            total_bytes: bytes,
394        })
395    }
396    /// Releases the cached result; subsequent reads verify and reload its arrays.
397    pub fn clear_cache(&self) {
398        if let Ok(mut cache) = self.cache.lock() {
399            *cache = None;
400        }
401    }
402}
403impl SpectralReconstructionResult {
404    /// Writes a separate spectral bundle from a supported solver's final checkpoint.
405    /// Scientific arrays use lossless NPY, traces/channel metadata use Parquet.
406    /// Existing destinations get a numbered sibling; publication is manifest-last.
407    /// Preview generation is not supported for this format and must be disabled.
408    pub fn write_bundle(
409        &self,
410        path: impl AsRef<Path>,
411        options: BundleExportOptions,
412    ) -> Result<SpectralResultBundle> {
413        let checkpoint=self.checkpoint.as_ref().ok_or_else(||invalid("spectral result has no resumable checkpoint; write the enclosing joint result instead"))?;
414        write(self, checkpoint, None, None, path.as_ref(), options)
415    }
416}
417impl MultiWavelengthSolverResult {
418    /// Writes channel fields plus authoritative joint OPD, gauge, bounds and initialization records.
419    pub fn write_bundle(
420        &self,
421        path: impl AsRef<Path>,
422        options: BundleExportOptions,
423    ) -> Result<SpectralResultBundle> {
424        write(
425            &self.spectral,
426            &self.checkpoint,
427            Some(&self.opd_m),
428            None,
429            path.as_ref(),
430            options,
431        )
432    }
433}
434impl MultiWavelengthReconstructionResult {
435    /// Writes spectral AP fields and synthetic-wavelength diagnostics, preserving invalid NaNs.
436    pub fn write_bundle(
437        &self,
438        path: impl AsRef<Path>,
439        options: BundleExportOptions,
440    ) -> Result<SpectralResultBundle> {
441        let checkpoint = self
442            .spectral
443            .checkpoint
444            .as_ref()
445            .ok_or_else(|| invalid("spectral result has no checkpoint"))?;
446        write(
447            &self.spectral,
448            checkpoint,
449            None,
450            Some(&self.opd),
451            path.as_ref(),
452            options,
453        )
454    }
455}
456fn validate_result(
457    result: &SpectralReconstructionResult,
458    checkpoint: &SpectralReconstructionCheckpoint,
459) -> Result<()> {
460    checkpoint.validate()?;
461    let model = checkpoint.model();
462    let backend =
463        crate::backend::CpuBackend::new(model.image_shape(), model.reconstruction_shape())?;
464    if result.channels.len() != model.channels().len()
465        || result.object_coupling != model.object_coupling()
466        || result.runtime.completed_iterations != checkpoint.completed_iterations()
467        || result.runtime.algorithm != checkpoint.algorithm()
468        || result.runtime.elapsed_seconds != checkpoint.elapsed_seconds
469        || serde_json::to_value(&result.trace)? != serde_json::to_value(checkpoint.trace())?
470    {
471        return Err(invalid("result metadata differs from checkpoint"));
472    }
473    for (i, (channel, kernel)) in result.channels.iter().zip(model.channels()).enumerate() {
474        if channel.channel_id != kernel.channel_id
475            || Some(channel.wavelength_vacuum_m) != kernel.model.sampling().wavelength
476            || channel.pupil != *kernel.model.pupil()
477            || channel.object.dim() != model.reconstruction_shape()
478            || channel.object_spectrum.dim() != model.reconstruction_shape()
479            || channel.amplitude.dim() != model.reconstruction_shape()
480            || channel.phase.dim() != model.reconstruction_shape()
481            || !channel.object.is_standard_layout()
482            || !channel.object_spectrum.is_standard_layout()
483            || !channel.amplitude.is_standard_layout()
484            || !channel.phase.is_standard_layout()
485            || channel
486                .object_spectrum
487                .iter()
488                .any(|v| !v.re.is_finite() || !v.im.is_finite())
489        {
490            return Err(invalid(
491                "channel metadata, pupil or array differs from model",
492            ));
493        }
494        for ((v, &a), &p) in channel
495            .object
496            .iter()
497            .zip(&channel.amplitude)
498            .zip(&channel.phase)
499        {
500            if !v.re.is_finite()
501                || !v.im.is_finite()
502                || !a.is_finite()
503                || a < 0.0
504                || !p.is_finite()
505                || (v.norm() - a).abs() > 64.0 * f64::EPSILON * a.max(1.0)
506                || (v.arg() - p).abs() > 64.0 * f64::EPSILON
507            {
508                return Err(invalid("invalid object/amplitude/phase values"));
509            }
510        }
511        let shape = model.reconstruction_shape();
512        let spectrum: Vec<_> = channel.object_spectrum.iter().copied().collect();
513        let mut object = vec![crate::Complex64::default(); spectrum.len()];
514        crate::model::ifftshift_copy(&spectrum, &mut object, shape);
515        let mut column = vec![crate::Complex64::default(); shape.0.max(model.image_shape().0)];
516        crate::backend::Backend::fft2(
517            &backend,
518            &mut object,
519            shape,
520            crate::backend::FftDirection::Inverse,
521            &mut column,
522        )?;
523        let tolerance = 64.0 * f64::EPSILON * (object.len() as f64).sqrt();
524        if object
525            .iter()
526            .zip(&channel.object)
527            .any(|(a, b)| (*a - *b).norm() > tolerance * a.norm().max(b.norm()).max(1.0))
528        {
529            return Err(invalid("object and centered spectrum disagree"));
530        }
531        match &checkpoint.state {
532            StoredSolver::Spectral { spectra, .. } => {
533                if spectra[model.object_index(i)?].data
534                    != channel.object_spectrum.iter().copied().collect::<Vec<_>>()
535                {
536                    return Err(invalid("spectrum differs from checkpoint"));
537                }
538            }
539            StoredSolver::JointOpd {
540                opd, amplitudes, ..
541            } => {
542                if amplitudes[i].data != channel.amplitude.iter().copied().collect::<Vec<_>>() {
543                    return Err(invalid("amplitude differs from joint checkpoint"));
544                }
545                if channel
546                    .object
547                    .iter()
548                    .zip(&opd.data)
549                    .zip(&amplitudes[i].data)
550                    .any(|((field, &opd), &amplitude)| {
551                        let expected = crate::Complex64::from_polar(
552                            amplitude,
553                            std::f64::consts::TAU * opd / channel.wavelength_vacuum_m,
554                        );
555                        (*field - expected).norm() > tolerance * amplitude.max(1.0)
556                    })
557                {
558                    return Err(invalid("channel field differs from authoritative OPD"));
559                }
560            }
561        }
562    }
563    Ok(())
564}
565fn write(
566    result: &SpectralReconstructionResult,
567    checkpoint: &SpectralReconstructionCheckpoint,
568    opd: Option<&Array2<f64>>,
569    unwrapped: Option<&OpticalPathDifferenceResult>,
570    path: &Path,
571    options: BundleExportOptions,
572) -> Result<SpectralResultBundle> {
573    validate_result(result, checkpoint)?;
574    if options.include_previews {
575        return Err(invalid(
576            "spectral bundles currently require include_previews=false",
577        ));
578    }
579    let joint = matches!(checkpoint.state, StoredSolver::JointOpd { .. });
580    if joint != opd.is_some() {
581        return Err(invalid("joint OPD array required for joint checkpoint"));
582    }
583    if let Some(opd) = opd
584        && (!opd.is_standard_layout()
585            || checkpoint.opd_m().is_none_or(|stored| stored != opd.view()))
586    {
587        return Err(invalid("OPD array differs from checkpoint"));
588    }
589    if let Some(unwrapped) = unwrapped {
590        crate::reconstruction::spectral_checkpoint::validate_opd(unwrapped)?;
591        if unwrapped.opd_m.dim() != checkpoint.model().reconstruction_shape()
592            || unwrapped.wavelengths_vacuum_m
593                != result
594                    .channels
595                    .iter()
596                    .map(|c| c.wavelength_vacuum_m)
597                    .collect::<Vec<_>>()
598        {
599            return Err(invalid("unwrapped OPD metadata differs from channels"));
600        }
601    }
602    let run_id = options
603        .run_id
604        .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
605    validate_run_id(&run_id)?;
606    let (destination, workspace) = unique_paths(path)?;
607    fs::create_dir_all(destination.parent().unwrap_or(Path::new(".")))?;
608    fs::create_dir(&workspace)?;
609    let mut manifest = Manifest {
610        kind: KIND.into(),
611        bundle_format_version: SPECTRAL_BUNDLE_FORMAT_VERSION,
612        run_id,
613        label: options.label,
614        crate_version: env!("CARGO_PKG_VERSION").into(),
615        channels: result
616            .channels
617            .iter()
618            .map(|c| Channel {
619                channel_id: c.channel_id.clone(),
620                wavelength_vacuum_m: c.wavelength_vacuum_m,
621            })
622            .collect(),
623        object_coupling: result.object_coupling,
624        image_shape: checkpoint.model().image_shape(),
625        reconstruction_shape: checkpoint.model().reconstruction_shape(),
626        runtime: result.runtime.clone(),
627        trace: result.trace.clone(),
628        has_joint_opd: joint,
629        has_unwrapped_opd: unwrapped.is_some(),
630        artifacts: Vec::new(),
631    };
632    let descriptors = expected_roles(&manifest);
633    for (role, (dtype, shape, media)) in descriptors {
634        let relative_path = if role.starts_with("channels.") || role == "arrays.opd_m" {
635            PathBuf::from(format!("arrays/{role}.npy"))
636        } else if role.starts_with("tables.") {
637            PathBuf::from(format!("tables/{role}.parquet"))
638        } else {
639            PathBuf::from(format!("state/{role}.json"))
640        };
641        let file = workspace.join(&relative_path);
642        fs::create_dir_all(file.parent().unwrap())?;
643        if role == "state.checkpoint" {
644            checkpoint.save(&file)?;
645        } else if role == "state.unwrapped_opd" {
646            write_json_atomic(&file, &StoredOpd::from_result(unwrapped.unwrap()))?;
647        } else if role == "arrays.opd_m" {
648            npy::write_f64_2(&file, opd.unwrap().view())?;
649        } else if role.starts_with("tables.") {
650            let mut table = match role.as_str() {
651                "tables.channels" => channel_table(&manifest.run_id, &manifest.channels)?,
652                "tables.history" => {
653                    crate::tabular::history_dataframe(&manifest.run_id, &manifest.trace)?
654                }
655                _ => {
656                    crate::tabular::algorithm_metrics_dataframe(&manifest.run_id, &manifest.trace)?
657                }
658            };
659            let mut output = File::create(&file)?;
660            {
661                ParquetWriter::new(&mut output)
662                    .with_key_value_metadata(Some(KeyValueMetadata::from_static(vec![
663                        ("fpm.bundle_kind".into(), KIND.into()),
664                        (
665                            "fpm.bundle_format_version".into(),
666                            SPECTRAL_BUNDLE_FORMAT_VERSION.to_string(),
667                        ),
668                        ("fpm.run_id".into(), manifest.run_id.clone()),
669                        ("fpm.table_role".into(), role.clone()),
670                    ])))
671                    .finish(&mut table)?;
672            }
673            output.sync_all()?;
674        } else {
675            let parts: Vec<_> = role.split('.').collect();
676            let c = &result.channels[parts[1]
677                .parse::<usize>()
678                .map_err(|_| invalid("bad channel index"))?];
679            match parts[2] {
680                "object" => npy::write_complex2(&file, c.object.view())?,
681                "object_spectrum" => npy::write_complex2(&file, c.object_spectrum.view())?,
682                "amplitude" => npy::write_f64_2(&file, c.amplitude.view())?,
683                "phase" => npy::write_f64_2(&file, c.phase.view())?,
684                "pupil" => npy::write_complex2(&file, c.pupil.values())?,
685                "pupil_support" => npy::write_u8_2(&file, c.pupil.support())?,
686                _ => return Err(invalid("unknown channel artifact")),
687            };
688        }
689        manifest.artifacts.push(ManifestArtifact {
690            role,
691            relative_path,
692            media_type: media.into(),
693            byte_size: fs::metadata(&file)?.len(),
694            sha256: sha256(&file)?,
695            dtype,
696            shape,
697        });
698    }
699    write_json_atomic(&workspace.join("manifest.json"), &manifest)?;
700    fs::rename(&workspace, &destination)?;
701    SpectralResultBundle::read(destination)
702}
703type Descriptor = (Option<String>, Option<Vec<u64>>, &'static str);
704fn expected_roles(m: &Manifest) -> BTreeMap<String, Descriptor> {
705    let mut roles = BTreeMap::new();
706    for i in 0..m.channels.len() {
707        for (name, dtype, shape) in [
708            ("object", "<c16", m.reconstruction_shape),
709            ("object_spectrum", "<c16", m.reconstruction_shape),
710            ("amplitude", "<f8", m.reconstruction_shape),
711            ("phase", "<f8", m.reconstruction_shape),
712            ("pupil", "<c16", m.image_shape),
713            ("pupil_support", "|u1", m.image_shape),
714        ] {
715            roles.insert(
716                format!("channels.{i}.{name}"),
717                (
718                    Some(dtype.into()),
719                    Some(vec![shape.0 as u64, shape.1 as u64]),
720                    "application/x-npy",
721                ),
722            );
723        }
724    }
725    for role in [
726        "tables.channels",
727        "tables.history",
728        "tables.algorithm_metrics",
729    ] {
730        roles.insert(role.into(), (None, None, "application/vnd.apache.parquet"));
731    }
732    roles.insert("state.checkpoint".into(), (None, None, "application/json"));
733    if m.has_joint_opd {
734        roles.insert(
735            "arrays.opd_m".into(),
736            (
737                Some("<f8".into()),
738                Some(vec![
739                    m.reconstruction_shape.0 as u64,
740                    m.reconstruction_shape.1 as u64,
741                ]),
742                "application/x-npy",
743            ),
744        );
745    }
746    if m.has_unwrapped_opd {
747        roles.insert(
748            "state.unwrapped_opd".into(),
749            (None, None, "application/json"),
750        );
751    }
752    roles
753}
754fn channel_table(run_id: &str, channels: &[Channel]) -> Result<DataFrame> {
755    use polars::prelude::{Column, NamedFrom, Series};
756    Ok(DataFrame::new(
757        channels.len(),
758        vec![
759            Column::from(Series::new("run_id".into(), vec![run_id; channels.len()])),
760            Column::from(Series::new(
761                "channel_index".into(),
762                (0..channels.len() as u64).collect::<Vec<_>>(),
763            )),
764            Column::from(Series::new(
765                "channel_id".into(),
766                channels
767                    .iter()
768                    .map(|c| c.channel_id.as_str())
769                    .collect::<Vec<_>>(),
770            )),
771            Column::from(Series::new(
772                "wavelength_vacuum_m".into(),
773                channels
774                    .iter()
775                    .map(|c| c.wavelength_vacuum_m)
776                    .collect::<Vec<_>>(),
777            )),
778        ],
779    )?)
780}
781fn safe_file(root: &Path, path: &Path) -> Result<PathBuf> {
782    if path.as_os_str().is_empty()
783        || path
784            .components()
785            .any(|c| !matches!(c, Component::Normal(_)))
786    {
787        return Err(invalid("artifact path must be safely relative"));
788    }
789    let absolute = root.join(path).canonicalize()?;
790    if !absolute.starts_with(root) {
791        return Err(invalid("artifact symlink escapes bundle root"));
792    }
793    if !absolute.is_file() {
794        return Err(invalid("artifact is not a regular file"));
795    }
796    Ok(absolute)
797}
798fn invalid(reason: impl Into<String>) -> Error {
799    Error::InvalidParameter {
800        name: "spectral_bundle",
801        reason: reason.into(),
802    }
803}