Skip to main content

fpm_rs/datasets/
loader.rs

1use std::{
2    collections::BTreeMap,
3    fs::File,
4    io::BufReader,
5    path::{Path, PathBuf},
6};
7
8use ndarray::Array2;
9use serde::{Deserialize, Serialize};
10
11use crate::{
12    Complex64, Result,
13    array_serde::Array2Data,
14    configuration::SimulationConfiguration,
15    error::Error,
16    measurements::{ImageSet, MeasurementSpec, MeasurementStack},
17    reconstruction::ReconstructionProblem,
18};
19
20/// Version of the ordinary configuration-based dataset profile.
21/// Spectral bundles use the separate version-two profile.
22pub const DATASET_FORMAT_VERSION: u32 = 1;
23
24/// The `dataset.json` entry point defined by `dataset_spec.md`.
25#[derive(Clone, Debug, Serialize, Deserialize)]
26#[serde(deny_unknown_fields)]
27pub struct DatasetManifest {
28    /// Dataset-bundle schema version; must equal [`DATASET_FORMAT_VERSION`].
29    pub format_version: u32,
30    /// Relative path to a [`crate::measurements::MeasurementSpec`] JSON file.
31    pub measurement_manifest: PathBuf,
32    /// Relative path to a [`crate::configuration::SimulationConfiguration`] JSON file.
33    pub configuration: PathBuf,
34    /// Optional JSON-serialized `Array2<Complex64>` in reconstruction space.
35    #[serde(default)]
36    pub ground_truth_object: Option<PathBuf>,
37    /// Optional JSON-serialized binary `Array2<u8>` in reconstruction space.
38    #[serde(default)]
39    pub valid_object_mask: Option<PathBuf>,
40    #[serde(default)]
41    /// User-visible provenance key/value metadata.
42    pub provenance: BTreeMap<String, String>,
43    #[serde(default)]
44    /// Optional semantic units for stored measurements, such as camera counts.
45    pub measurement_units: Option<String>,
46}
47
48/// A validated dataset loaded from a bundle conforming to `dataset_spec.md`.
49#[derive(Clone, Debug)]
50pub struct Dataset {
51    source_path: Option<PathBuf>,
52    measurements: MeasurementStack,
53    configuration: SimulationConfiguration,
54    ground_truth_object: Option<Array2<Complex64>>,
55    valid_object_mask: Option<Array2<u8>>,
56    provenance: BTreeMap<String, String>,
57    measurement_units: Option<String>,
58}
59
60impl Dataset {
61    /// Constructs a dataset from already-loaded values.
62    pub fn new(
63        measurements: MeasurementStack,
64        configuration: SimulationConfiguration,
65    ) -> Result<Self> {
66        measurements.validate()?;
67        configuration.validate()?;
68        ReconstructionProblem::new(
69            measurements.clone(),
70            configuration.compiled_models.reconstruction_model.clone(),
71        )?;
72        Ok(Self {
73            source_path: None,
74            measurements,
75            configuration,
76            ground_truth_object: None,
77            valid_object_mask: None,
78            provenance: BTreeMap::new(),
79            measurement_units: None,
80        })
81    }
82
83    /// Borrows the resident measured-intensity stack.
84    pub fn measurements(&self) -> &MeasurementStack {
85        &self.measurements
86    }
87
88    /// Root directory of the loaded bundle, or `None` for programmatic data.
89    pub fn source_path(&self) -> Option<&Path> {
90        self.source_path.as_deref()
91    }
92
93    /// Borrows the validated true/assumed experiment configuration.
94    pub fn configuration(&self) -> &SimulationConfiguration {
95        &self.configuration
96    }
97
98    /// Borrows optional high-resolution complex ground truth shaped `(height, width)`.
99    pub fn ground_truth_object(&self) -> Option<&Array2<Complex64>> {
100        self.ground_truth_object.as_ref()
101    }
102
103    /// Borrows an optional same-shaped binary validity mask for object metrics.
104    pub fn valid_object_mask(&self) -> Option<&Array2<u8>> {
105        self.valid_object_mask.as_ref()
106    }
107
108    /// Borrows provenance metadata from the dataset manifest.
109    pub fn provenance(&self) -> &BTreeMap<String, String> {
110        &self.provenance
111    }
112
113    /// Borrows optional semantic measurement units.
114    pub fn measurement_units(&self) -> Option<&str> {
115        self.measurement_units.as_deref()
116    }
117
118    /// Clones measurements and the assumed model into a validated reconstruction problem.
119    pub fn reconstruction_problem(&self) -> Result<ReconstructionProblem<MeasurementStack>> {
120        ReconstructionProblem::new(
121            self.measurements.clone(),
122            self.configuration
123                .compiled_models
124                .reconstruction_model
125                .clone(),
126        )
127    }
128
129    /// Starts a deterministic frame-selection and spatial-cropping builder.
130    pub fn subset(&self) -> super::DatasetSubsetBuilder<'_> {
131        super::DatasetSubsetBuilder::new(self)
132    }
133
134    fn with_metadata(
135        mut self,
136        ground_truth_object: Option<Array2<Complex64>>,
137        valid_object_mask: Option<Array2<u8>>,
138        provenance: BTreeMap<String, String>,
139        measurement_units: Option<String>,
140    ) -> Result<Self> {
141        validate_dataset_metadata(
142            &self.configuration,
143            ground_truth_object.as_ref(),
144            valid_object_mask.as_ref(),
145            &provenance,
146            measurement_units.as_deref(),
147        )?;
148        self.ground_truth_object = ground_truth_object;
149        self.valid_object_mask = valid_object_mask;
150        self.provenance = provenance;
151        self.measurement_units = measurement_units;
152        Ok(self)
153    }
154}
155
156/// Loads an already-converted dataset bundle from a local directory.
157#[derive(Clone, Debug)]
158pub struct DatasetLoader {
159    root: PathBuf,
160}
161
162impl DatasetLoader {
163    /// Opens an existing local bundle root without performing network access.
164    pub fn new(root: impl Into<PathBuf>) -> Result<Self> {
165        let root = root.into();
166        if !root.is_dir() {
167            return Err(Error::Dataset(format!(
168                "dataset path does not exist or is not a directory: {}",
169                root.display()
170            )));
171        }
172        Ok(Self { root })
173    }
174
175    /// Returns the local dataset-bundle root.
176    pub fn root(&self) -> &Path {
177        &self.root
178    }
179
180    /// Returns `root/dataset.json` without checking whether the file exists.
181    pub fn manifest_path(&self) -> PathBuf {
182        self.root.join("dataset.json")
183    }
184
185    /// Loads and validates the manifest, measurements, configuration, optional ground truth,
186    /// mask, provenance, and measurement units entirely from local files.
187    pub fn load(&self) -> Result<Dataset> {
188        let manifest_path = self.manifest_path();
189        if self.format_version()? == 2 {
190            return Err(Error::Dataset(
191                "ordinary load expected 1; spectral version-2 data requires load_spectral".into(),
192            ));
193        }
194        let manifest: DatasetManifest =
195            serde_json::from_reader(BufReader::new(File::open(&manifest_path)?))?;
196        if manifest.format_version != DATASET_FORMAT_VERSION {
197            return Err(Error::Dataset(format!(
198                "unsupported dataset format version {} in {}; expected {}",
199                manifest.format_version,
200                manifest_path.display(),
201                DATASET_FORMAT_VERSION
202            )));
203        }
204        validate_dataset_manifest_paths(&manifest)?;
205        let measurement_manifest_path = self.root.join(&manifest.measurement_manifest);
206        let measurement_spec = MeasurementSpec::load(&measurement_manifest_path)?;
207        validate_measurement_paths(&measurement_spec)?;
208        let measurement_base = measurement_manifest_path
209            .parent()
210            .unwrap_or_else(|| Path::new("."));
211        let measurements =
212            MeasurementStack::from_manifest_definition(measurement_spec, measurement_base)?;
213        let configuration = SimulationConfiguration::load(self.root.join(&manifest.configuration))?;
214        let ground_truth_object = manifest
215            .ground_truth_object
216            .as_ref()
217            .map(|path| load_json_array(self.root.join(path), "ground-truth object"))
218            .transpose()?;
219        let valid_object_mask = manifest
220            .valid_object_mask
221            .as_ref()
222            .map(|path| load_json_array(self.root.join(path), "valid-object mask"))
223            .transpose()?;
224        let mut dataset = Dataset::new(measurements, configuration)?;
225        dataset.source_path = Some(self.root.clone());
226        dataset.with_metadata(
227            ground_truth_object,
228            valid_object_mask,
229            manifest.provenance,
230            manifest.measurement_units,
231        )
232    }
233}
234
235fn validate_dataset_manifest_paths(manifest: &DatasetManifest) -> Result<()> {
236    validate_contained_path("measurement manifest", &manifest.measurement_manifest)?;
237    validate_contained_path("configuration", &manifest.configuration)?;
238    if let Some(path) = &manifest.ground_truth_object {
239        validate_contained_path("ground-truth object", path)?;
240    }
241    if let Some(path) = &manifest.valid_object_mask {
242        validate_contained_path("valid-object mask", path)?;
243    }
244    Ok(())
245}
246
247fn validate_measurement_paths(spec: &MeasurementSpec) -> Result<()> {
248    for frame in &spec.frames {
249        validate_contained_path("measurement frame", &frame.path)?;
250    }
251    if let Some(path) = &spec.dark_frame {
252        validate_contained_path("dark frame", path)?;
253    }
254    if let Some(path) = &spec.flat_field {
255        validate_contained_path("flat field", path)?;
256    }
257    if let Some(images) = &spec.background {
258        validate_image_set_paths("background", images)?;
259    }
260    if let Some(images) = &spec.mask {
261        validate_image_set_paths("mask", images)?;
262    }
263    Ok(())
264}
265
266fn validate_image_set_paths(label: &str, images: &ImageSet) -> Result<()> {
267    match images {
268        ImageSet::Single(path) => validate_contained_path(label, path),
269        ImageSet::PerFrame(paths) => paths
270            .iter()
271            .try_for_each(|path| validate_contained_path(label, path)),
272    }
273}
274
275pub(super) fn validate_contained_path(label: &str, path: &Path) -> Result<()> {
276    if path.as_os_str().is_empty()
277        || path
278            .components()
279            .any(|component| !matches!(component, std::path::Component::Normal(_)))
280    {
281        return Err(Error::Dataset(format!(
282            "{label} must be a safe relative path contained by its manifest directory: {}",
283            path.display()
284        )));
285    }
286    Ok(())
287}
288
289pub(super) fn load_json_array<T>(path: PathBuf, label: &str) -> Result<Array2<T>>
290where
291    T: for<'de> Deserialize<'de>,
292{
293    let file = File::open(&path).map_err(|error| {
294        Error::Dataset(format!(
295            "failed to open {label} at {}: {error}",
296            path.display()
297        ))
298    })?;
299    let representation: Array2Data<T> =
300        serde_json::from_reader(BufReader::new(file)).map_err(|error| {
301            Error::Dataset(format!(
302                "failed to parse {label} at {}: {error}",
303                path.display()
304            ))
305        })?;
306    representation
307        .into_array()
308        .map_err(|error| Error::Dataset(format!("invalid {label} at {}: {error}", path.display())))
309}
310
311fn validate_dataset_metadata(
312    configuration: &SimulationConfiguration,
313    ground_truth_object: Option<&Array2<Complex64>>,
314    valid_object_mask: Option<&Array2<u8>>,
315    provenance: &BTreeMap<String, String>,
316    measurement_units: Option<&str>,
317) -> Result<()> {
318    if let Some(ground_truth) = ground_truth_object {
319        if ground_truth.dim() != configuration.reconstruction_shape {
320            return Err(Error::Dataset(format!(
321                "ground-truth shape {:?} differs from reconstruction shape {:?}",
322                ground_truth.dim(),
323                configuration.reconstruction_shape
324            )));
325        }
326        if ground_truth
327            .iter()
328            .any(|value| !value.re.is_finite() || !value.im.is_finite())
329        {
330            return Err(Error::Dataset(
331                "ground-truth object contains non-finite values".into(),
332            ));
333        }
334    }
335    if let Some(mask) = valid_object_mask {
336        if ground_truth_object.is_none() {
337            return Err(Error::Dataset(
338                "a valid-object mask requires a ground-truth object".into(),
339            ));
340        }
341        if mask.dim() != configuration.reconstruction_shape {
342            return Err(Error::Dataset(format!(
343                "valid-object mask shape {:?} differs from reconstruction shape {:?}",
344                mask.dim(),
345                configuration.reconstruction_shape
346            )));
347        }
348        if mask.iter().any(|&value| value > 1) || !mask.iter().any(|&value| value == 1) {
349            return Err(Error::Dataset(
350                "valid-object mask must contain only zero and one and select at least one pixel"
351                    .into(),
352            ));
353        }
354    }
355    if provenance
356        .iter()
357        .any(|(key, value)| key.trim().is_empty() || value.trim().is_empty())
358    {
359        return Err(Error::Dataset(
360            "dataset provenance keys and values must not be empty".into(),
361        ));
362    }
363    if measurement_units.is_some_and(|units| units.trim().is_empty()) {
364        return Err(Error::Dataset(
365            "measurement units must not be empty when provided".into(),
366        ));
367    }
368    Ok(())
369}