Skip to main content

fpm_rs/datasets/
spectral.rs

1//! Explicit version-two narrowband datasets; source-specific conversion stays external.
2use super::DatasetLoader;
3use crate::{
4    Complex64, Error, Result,
5    experiment::SpectralAcquisitionPlan,
6    measurements::{ImageSet, MeasurementSpec, MeasurementStack},
7    model::{ImagePlaneModel, ObjectCoupling, SpectralImagePlaneModel, SpectralModelChannel},
8    reconstruction::SpectralReconstructionProblem,
9};
10use ndarray::Array2;
11use serde::{Deserialize, Serialize};
12use std::{
13    collections::BTreeMap,
14    fs::File,
15    io::BufReader,
16    path::{Path, PathBuf},
17};
18
19/// Version of the explicit spectral dataset profile; ordinary bundles remain version one.
20pub const SPECTRAL_DATASET_FORMAT_VERSION: u32 = 2;
21/// Converter-supplied channel record in stable solver order.
22#[derive(Clone, Debug, Serialize, Deserialize)]
23#[serde(deny_unknown_fields)]
24pub struct SpectralDatasetChannel {
25    /// Nonempty unique channel ID.
26    pub channel_id: String,
27    /// Explicit positive distinct vacuum wavelength in metres; must match the kernel.
28    pub wavelength_vacuum_m: f64,
29    /// Safe relative path to JSON-serialized compiled `ImagePlaneModel`.
30    pub compiled_model: PathBuf,
31    /// Required nonempty description of spectral response, weights and calibration origins.
32    pub response_provenance: String,
33    /// Optional common-grid complex truth using height/width/data JSON array encoding.
34    #[serde(default)]
35    pub ground_truth_object: Option<PathBuf>,
36    /// Optional common-grid binary mask using height/width/data JSON array encoding.
37    #[serde(default)]
38    pub valid_object_mask: Option<PathBuf>,
39}
40/// `dataset.json` for the version-two spectral profile, never inferred from RGB images.
41#[derive(Clone, Debug, Serialize, Deserialize)]
42#[serde(deny_unknown_fields)]
43pub struct SpectralDatasetManifest {
44    /// Must equal [`SPECTRAL_DATASET_FORMAT_VERSION`].
45    pub format_version: u32,
46    /// Safe relative path to the ordinary grayscale measurement manifest.
47    pub measurement_manifest: PathBuf,
48    /// Stable ordered wavelength records with externally compiled kernels.
49    pub channels: Vec<SpectralDatasetChannel>,
50    /// Explicit complex field reuse rule.
51    pub object_coupling: ObjectCoupling,
52    /// Sparse physical exposure order, including gains/background and channel-local references.
53    pub acquisition: SpectralAcquisitionPlan,
54    /// Converter provenance metadata.
55    #[serde(default)]
56    pub provenance: BTreeMap<String, String>,
57    /// Optional semantic detector units, such as camera counts.
58    #[serde(default)]
59    pub measurement_units: Option<String>,
60}
61/// Resident spectral measurements, compiled model, converter metadata and optional per-channel truth.
62#[derive(Clone, Debug)]
63pub struct SpectralDataset {
64    source_path: PathBuf,
65    measurements: MeasurementStack,
66    model: SpectralImagePlaneModel,
67    manifest: SpectralDatasetManifest,
68    truth: Vec<Option<Array2<Complex64>>>,
69    masks: Vec<Option<Array2<u8>>>,
70}
71impl SpectralDataset {
72    /// Borrows the local bundle directory.
73    pub fn source_path(&self) -> &Path {
74        &self.source_path
75    }
76    /// Borrows resident scalar detector data and masks/weights.
77    pub fn measurements(&self) -> &MeasurementStack {
78        &self.measurements
79    }
80    /// Borrows the ordered compiled spectral model.
81    pub fn model(&self) -> &SpectralImagePlaneModel {
82        &self.model
83    }
84    /// Borrows converter metadata and response provenance in channel order.
85    pub fn manifest(&self) -> &SpectralDatasetManifest {
86        &self.manifest
87    }
88    /// Borrows optional per-channel common-grid ground truths.
89    pub fn ground_truth_objects(&self) -> &[Option<Array2<Complex64>>] {
90        &self.truth
91    }
92    /// Borrows optional per-channel binary truth-validity masks.
93    pub fn valid_object_masks(&self) -> &[Option<Array2<u8>>] {
94        &self.masks
95    }
96    /// Copies resident data/model into an independently owned validated spectral problem.
97    pub fn reconstruction_problem(
98        &self,
99    ) -> Result<SpectralReconstructionProblem<MeasurementStack>> {
100        SpectralReconstructionProblem::new(self.measurements.clone(), self.model.clone())
101    }
102}
103impl DatasetLoader {
104    /// Loads a version-two spectral profile entirely offline.
105    /// Requires explicit wavelength/channel metadata and response provenance; validates
106    /// common grids, sparse references, scalar frames, truth/masks and safe contained paths.
107    /// Ordinary version-one data must use `load` instead. No registration or RGB demixing occurs.
108    ///
109    /// # Example
110    /// ```no_run
111    /// use fpm_rs::{Result, datasets::DatasetLoader,
112    ///     algorithms::{SpectralAlternatingProjection, SpectralReconstructionAlgorithm}};
113    /// fn main() -> Result<()> {
114    ///     let dataset = DatasetLoader::new("converted-spectral")?.load_spectral()?;
115    ///     let result = SpectralAlternatingProjection::default().run(&dataset.reconstruction_problem()?)?;
116    ///     assert_eq!(result.channels.len(), dataset.model().channels().len());
117    ///     Ok(())
118    /// }
119    /// ```
120    pub fn load_spectral(&self) -> Result<SpectralDataset> {
121        let root = self.root();
122        let manifest_path = contained(root, Path::new("dataset.json"))?;
123        let manifest: SpectralDatasetManifest =
124            serde_json::from_reader(BufReader::new(File::open(manifest_path)?))?;
125        if manifest.format_version != SPECTRAL_DATASET_FORMAT_VERSION {
126            return Err(Error::Dataset(
127                "load_spectral requires dataset format_version 2; use load for version 1".into(),
128            ));
129        }
130        if manifest
131            .measurement_units
132            .as_ref()
133            .is_some_and(|v| v.trim().is_empty())
134            || manifest
135                .provenance
136                .iter()
137                .any(|(k, v)| k.trim().is_empty() || v.trim().is_empty())
138        {
139            return Err(Error::Dataset(
140                "units and provenance entries must be nonempty".into(),
141            ));
142        }
143        let measurement_path = contained(root, &manifest.measurement_manifest)?;
144        let spec = MeasurementSpec::load(&measurement_path)?;
145        let base = measurement_path.parent().unwrap_or(root);
146        validate_measurement_files(base, &spec)?;
147        let measurements = MeasurementStack::from_manifest_definition(spec, base)?;
148        let mut channels = Vec::new();
149        let mut truth = Vec::new();
150        let mut masks = Vec::new();
151        for channel in &manifest.channels {
152            if channel.response_provenance.trim().is_empty() {
153                return Err(Error::Dataset(format!(
154                    "channel '{}' requires response_provenance",
155                    channel.channel_id
156                )));
157            }
158            let model: ImagePlaneModel = serde_json::from_reader(BufReader::new(File::open(
159                contained(root, &channel.compiled_model)?,
160            )?))?;
161            if model.sampling().wavelength != Some(channel.wavelength_vacuum_m) {
162                return Err(Error::Dataset(format!(
163                    "channel '{}' wavelength differs from compiled kernel",
164                    channel.channel_id
165                )));
166            }
167            if channel.valid_object_mask.is_some() && channel.ground_truth_object.is_none() {
168                return Err(Error::Dataset(
169                    "spectral valid_object_mask requires ground_truth_object".into(),
170                ));
171            }
172            let object = channel
173                .ground_truth_object
174                .as_ref()
175                .map(|p| super::loader::load_json_array(contained(root, p)?, "spectral truth"))
176                .transpose()?;
177            let mask = channel
178                .valid_object_mask
179                .as_ref()
180                .map(|p| super::loader::load_json_array(contained(root, p)?, "spectral truth mask"))
181                .transpose()?;
182            if object.as_ref().is_some_and(|a: &Array2<Complex64>| {
183                a.dim() != model.reconstruction_shape()
184                    || a.iter().any(|v| !v.re.is_finite() || !v.im.is_finite())
185            }) || mask.as_ref().is_some_and(|a: &Array2<u8>| {
186                a.dim() != model.reconstruction_shape() || a.iter().any(|&v| v > 1)
187            }) {
188                return Err(Error::Dataset(format!(
189                    "channel '{}' truth or mask has invalid shape/values",
190                    channel.channel_id
191                )));
192            }
193            channels.push(SpectralModelChannel {
194                channel_id: channel.channel_id.clone(),
195                model,
196            });
197            truth.push(object);
198            masks.push(mask);
199        }
200        let model = SpectralImagePlaneModel::from_compiled_channels(
201            channels,
202            manifest.acquisition.clone(),
203            manifest.object_coupling,
204        )?;
205        SpectralReconstructionProblem::new(&measurements, model.clone())?;
206        Ok(SpectralDataset {
207            source_path: root.to_owned(),
208            measurements,
209            model,
210            manifest,
211            truth,
212            masks,
213        })
214    }
215    /// Reads the profile version without interpreting the rest of the manifest.
216    pub fn format_version(&self) -> Result<u32> {
217        let value: serde_json::Value =
218            serde_json::from_reader(BufReader::new(File::open(self.manifest_path())?))?;
219        value
220            .get("format_version")
221            .and_then(|v| v.as_u64())
222            .and_then(|v| u32::try_from(v).ok())
223            .ok_or_else(|| Error::Dataset("dataset.json requires an integer format_version".into()))
224    }
225    pub(super) fn validate_profile(&self, expected: u32) -> Result<()> {
226        if self.format_version()? != expected {
227            return Err(Error::Dataset(
228                "registry format_version differs from dataset.json".into(),
229            ));
230        }
231        match expected {
232            1 => {
233                self.load()?;
234            }
235            2 => {
236                self.load_spectral()?;
237            }
238            _ => return Err(Error::Dataset("unsupported dataset profile".into())),
239        };
240        Ok(())
241    }
242}
243fn contained(root: &Path, path: &Path) -> Result<PathBuf> {
244    super::loader::validate_contained_path("spectral dataset artifact", path)?;
245    let resolved = root.join(path).canonicalize()?;
246    if !resolved.starts_with(root.canonicalize()?) {
247        return Err(Error::Dataset(format!(
248            "spectral artifact escapes its manifest directory: {}",
249            path.display()
250        )));
251    }
252    Ok(resolved)
253}
254fn validate_measurement_files(base: &Path, spec: &MeasurementSpec) -> Result<()> {
255    for frame in &spec.frames {
256        contained(base, &frame.path)?;
257    }
258    for path in [&spec.dark_frame, &spec.flat_field].into_iter().flatten() {
259        contained(base, path)?;
260    }
261    for set in [&spec.background, &spec.mask].into_iter().flatten() {
262        match set {
263            ImageSet::Single(p) => {
264                contained(base, p)?;
265            }
266            ImageSet::PerFrame(paths) => {
267                for p in paths {
268                    contained(base, p)?;
269                }
270            }
271        }
272    }
273    Ok(())
274}