1use std::{
2 collections::BTreeMap,
3 fs::File,
4 io::BufReader,
5 path::{Path, PathBuf},
6};
7
8use ndarray::Array2;
9use serde::{Deserialize, Serialize};
10
11use crate::{
12 Complex64, Result,
13 array_serde::Array2Data,
14 configuration::SimulationConfiguration,
15 error::Error,
16 measurements::{ImageSet, MeasurementSpec, MeasurementStack},
17 reconstruction::ReconstructionProblem,
18};
19
20pub const DATASET_FORMAT_VERSION: u32 = 1;
23
24#[derive(Clone, Debug, Serialize, Deserialize)]
26#[serde(deny_unknown_fields)]
27pub struct DatasetManifest {
28 pub format_version: u32,
30 pub measurement_manifest: PathBuf,
32 pub configuration: PathBuf,
34 #[serde(default)]
36 pub ground_truth_object: Option<PathBuf>,
37 #[serde(default)]
39 pub valid_object_mask: Option<PathBuf>,
40 #[serde(default)]
41 pub provenance: BTreeMap<String, String>,
43 #[serde(default)]
44 pub measurement_units: Option<String>,
46}
47
48#[derive(Clone, Debug)]
50pub struct Dataset {
51 source_path: Option<PathBuf>,
52 measurements: MeasurementStack,
53 configuration: SimulationConfiguration,
54 ground_truth_object: Option<Array2<Complex64>>,
55 valid_object_mask: Option<Array2<u8>>,
56 provenance: BTreeMap<String, String>,
57 measurement_units: Option<String>,
58}
59
60impl Dataset {
61 pub fn new(
63 measurements: MeasurementStack,
64 configuration: SimulationConfiguration,
65 ) -> Result<Self> {
66 measurements.validate()?;
67 configuration.validate()?;
68 ReconstructionProblem::new(
69 measurements.clone(),
70 configuration.compiled_models.reconstruction_model.clone(),
71 )?;
72 Ok(Self {
73 source_path: None,
74 measurements,
75 configuration,
76 ground_truth_object: None,
77 valid_object_mask: None,
78 provenance: BTreeMap::new(),
79 measurement_units: None,
80 })
81 }
82
83 pub fn measurements(&self) -> &MeasurementStack {
85 &self.measurements
86 }
87
88 pub fn source_path(&self) -> Option<&Path> {
90 self.source_path.as_deref()
91 }
92
93 pub fn configuration(&self) -> &SimulationConfiguration {
95 &self.configuration
96 }
97
98 pub fn ground_truth_object(&self) -> Option<&Array2<Complex64>> {
100 self.ground_truth_object.as_ref()
101 }
102
103 pub fn valid_object_mask(&self) -> Option<&Array2<u8>> {
105 self.valid_object_mask.as_ref()
106 }
107
108 pub fn provenance(&self) -> &BTreeMap<String, String> {
110 &self.provenance
111 }
112
113 pub fn measurement_units(&self) -> Option<&str> {
115 self.measurement_units.as_deref()
116 }
117
118 pub fn reconstruction_problem(&self) -> Result<ReconstructionProblem<MeasurementStack>> {
120 ReconstructionProblem::new(
121 self.measurements.clone(),
122 self.configuration
123 .compiled_models
124 .reconstruction_model
125 .clone(),
126 )
127 }
128
129 pub fn subset(&self) -> super::DatasetSubsetBuilder<'_> {
131 super::DatasetSubsetBuilder::new(self)
132 }
133
134 fn with_metadata(
135 mut self,
136 ground_truth_object: Option<Array2<Complex64>>,
137 valid_object_mask: Option<Array2<u8>>,
138 provenance: BTreeMap<String, String>,
139 measurement_units: Option<String>,
140 ) -> Result<Self> {
141 validate_dataset_metadata(
142 &self.configuration,
143 ground_truth_object.as_ref(),
144 valid_object_mask.as_ref(),
145 &provenance,
146 measurement_units.as_deref(),
147 )?;
148 self.ground_truth_object = ground_truth_object;
149 self.valid_object_mask = valid_object_mask;
150 self.provenance = provenance;
151 self.measurement_units = measurement_units;
152 Ok(self)
153 }
154}
155
156#[derive(Clone, Debug)]
158pub struct DatasetLoader {
159 root: PathBuf,
160}
161
162impl DatasetLoader {
163 pub fn new(root: impl Into<PathBuf>) -> Result<Self> {
165 let root = root.into();
166 if !root.is_dir() {
167 return Err(Error::Dataset(format!(
168 "dataset path does not exist or is not a directory: {}",
169 root.display()
170 )));
171 }
172 Ok(Self { root })
173 }
174
175 pub fn root(&self) -> &Path {
177 &self.root
178 }
179
180 pub fn manifest_path(&self) -> PathBuf {
182 self.root.join("dataset.json")
183 }
184
185 pub fn load(&self) -> Result<Dataset> {
188 let manifest_path = self.manifest_path();
189 if self.format_version()? == 2 {
190 return Err(Error::Dataset(
191 "ordinary load expected 1; spectral version-2 data requires load_spectral".into(),
192 ));
193 }
194 let manifest: DatasetManifest =
195 serde_json::from_reader(BufReader::new(File::open(&manifest_path)?))?;
196 if manifest.format_version != DATASET_FORMAT_VERSION {
197 return Err(Error::Dataset(format!(
198 "unsupported dataset format version {} in {}; expected {}",
199 manifest.format_version,
200 manifest_path.display(),
201 DATASET_FORMAT_VERSION
202 )));
203 }
204 validate_dataset_manifest_paths(&manifest)?;
205 let measurement_manifest_path = self.root.join(&manifest.measurement_manifest);
206 let measurement_spec = MeasurementSpec::load(&measurement_manifest_path)?;
207 validate_measurement_paths(&measurement_spec)?;
208 let measurement_base = measurement_manifest_path
209 .parent()
210 .unwrap_or_else(|| Path::new("."));
211 let measurements =
212 MeasurementStack::from_manifest_definition(measurement_spec, measurement_base)?;
213 let configuration = SimulationConfiguration::load(self.root.join(&manifest.configuration))?;
214 let ground_truth_object = manifest
215 .ground_truth_object
216 .as_ref()
217 .map(|path| load_json_array(self.root.join(path), "ground-truth object"))
218 .transpose()?;
219 let valid_object_mask = manifest
220 .valid_object_mask
221 .as_ref()
222 .map(|path| load_json_array(self.root.join(path), "valid-object mask"))
223 .transpose()?;
224 let mut dataset = Dataset::new(measurements, configuration)?;
225 dataset.source_path = Some(self.root.clone());
226 dataset.with_metadata(
227 ground_truth_object,
228 valid_object_mask,
229 manifest.provenance,
230 manifest.measurement_units,
231 )
232 }
233}
234
235fn validate_dataset_manifest_paths(manifest: &DatasetManifest) -> Result<()> {
236 validate_contained_path("measurement manifest", &manifest.measurement_manifest)?;
237 validate_contained_path("configuration", &manifest.configuration)?;
238 if let Some(path) = &manifest.ground_truth_object {
239 validate_contained_path("ground-truth object", path)?;
240 }
241 if let Some(path) = &manifest.valid_object_mask {
242 validate_contained_path("valid-object mask", path)?;
243 }
244 Ok(())
245}
246
247fn validate_measurement_paths(spec: &MeasurementSpec) -> Result<()> {
248 for frame in &spec.frames {
249 validate_contained_path("measurement frame", &frame.path)?;
250 }
251 if let Some(path) = &spec.dark_frame {
252 validate_contained_path("dark frame", path)?;
253 }
254 if let Some(path) = &spec.flat_field {
255 validate_contained_path("flat field", path)?;
256 }
257 if let Some(images) = &spec.background {
258 validate_image_set_paths("background", images)?;
259 }
260 if let Some(images) = &spec.mask {
261 validate_image_set_paths("mask", images)?;
262 }
263 Ok(())
264}
265
266fn validate_image_set_paths(label: &str, images: &ImageSet) -> Result<()> {
267 match images {
268 ImageSet::Single(path) => validate_contained_path(label, path),
269 ImageSet::PerFrame(paths) => paths
270 .iter()
271 .try_for_each(|path| validate_contained_path(label, path)),
272 }
273}
274
275pub(super) fn validate_contained_path(label: &str, path: &Path) -> Result<()> {
276 if path.as_os_str().is_empty()
277 || path
278 .components()
279 .any(|component| !matches!(component, std::path::Component::Normal(_)))
280 {
281 return Err(Error::Dataset(format!(
282 "{label} must be a safe relative path contained by its manifest directory: {}",
283 path.display()
284 )));
285 }
286 Ok(())
287}
288
289pub(super) fn load_json_array<T>(path: PathBuf, label: &str) -> Result<Array2<T>>
290where
291 T: for<'de> Deserialize<'de>,
292{
293 let file = File::open(&path).map_err(|error| {
294 Error::Dataset(format!(
295 "failed to open {label} at {}: {error}",
296 path.display()
297 ))
298 })?;
299 let representation: Array2Data<T> =
300 serde_json::from_reader(BufReader::new(file)).map_err(|error| {
301 Error::Dataset(format!(
302 "failed to parse {label} at {}: {error}",
303 path.display()
304 ))
305 })?;
306 representation
307 .into_array()
308 .map_err(|error| Error::Dataset(format!("invalid {label} at {}: {error}", path.display())))
309}
310
311fn validate_dataset_metadata(
312 configuration: &SimulationConfiguration,
313 ground_truth_object: Option<&Array2<Complex64>>,
314 valid_object_mask: Option<&Array2<u8>>,
315 provenance: &BTreeMap<String, String>,
316 measurement_units: Option<&str>,
317) -> Result<()> {
318 if let Some(ground_truth) = ground_truth_object {
319 if ground_truth.dim() != configuration.reconstruction_shape {
320 return Err(Error::Dataset(format!(
321 "ground-truth shape {:?} differs from reconstruction shape {:?}",
322 ground_truth.dim(),
323 configuration.reconstruction_shape
324 )));
325 }
326 if ground_truth
327 .iter()
328 .any(|value| !value.re.is_finite() || !value.im.is_finite())
329 {
330 return Err(Error::Dataset(
331 "ground-truth object contains non-finite values".into(),
332 ));
333 }
334 }
335 if let Some(mask) = valid_object_mask {
336 if ground_truth_object.is_none() {
337 return Err(Error::Dataset(
338 "a valid-object mask requires a ground-truth object".into(),
339 ));
340 }
341 if mask.dim() != configuration.reconstruction_shape {
342 return Err(Error::Dataset(format!(
343 "valid-object mask shape {:?} differs from reconstruction shape {:?}",
344 mask.dim(),
345 configuration.reconstruction_shape
346 )));
347 }
348 if mask.iter().any(|&value| value > 1) || !mask.iter().any(|&value| value == 1) {
349 return Err(Error::Dataset(
350 "valid-object mask must contain only zero and one and select at least one pixel"
351 .into(),
352 ));
353 }
354 }
355 if provenance
356 .iter()
357 .any(|(key, value)| key.trim().is_empty() || value.trim().is_empty())
358 {
359 return Err(Error::Dataset(
360 "dataset provenance keys and values must not be empty".into(),
361 ));
362 }
363 if measurement_units.is_some_and(|units| units.trim().is_empty()) {
364 return Err(Error::Dataset(
365 "measurement units must not be empty when provided".into(),
366 ));
367 }
368 Ok(())
369}