1use super::DatasetLoader;
3use crate::{
4 Complex64, Error, Result,
5 experiment::SpectralAcquisitionPlan,
6 measurements::{ImageSet, MeasurementSpec, MeasurementStack},
7 model::{ImagePlaneModel, ObjectCoupling, SpectralImagePlaneModel, SpectralModelChannel},
8 reconstruction::SpectralReconstructionProblem,
9};
10use ndarray::Array2;
11use serde::{Deserialize, Serialize};
12use std::{
13 collections::BTreeMap,
14 fs::File,
15 io::BufReader,
16 path::{Path, PathBuf},
17};
18
19pub const SPECTRAL_DATASET_FORMAT_VERSION: u32 = 2;
21#[derive(Clone, Debug, Serialize, Deserialize)]
23#[serde(deny_unknown_fields)]
24pub struct SpectralDatasetChannel {
25 pub channel_id: String,
27 pub wavelength_vacuum_m: f64,
29 pub compiled_model: PathBuf,
31 pub response_provenance: String,
33 #[serde(default)]
35 pub ground_truth_object: Option<PathBuf>,
36 #[serde(default)]
38 pub valid_object_mask: Option<PathBuf>,
39}
40#[derive(Clone, Debug, Serialize, Deserialize)]
42#[serde(deny_unknown_fields)]
43pub struct SpectralDatasetManifest {
44 pub format_version: u32,
46 pub measurement_manifest: PathBuf,
48 pub channels: Vec<SpectralDatasetChannel>,
50 pub object_coupling: ObjectCoupling,
52 pub acquisition: SpectralAcquisitionPlan,
54 #[serde(default)]
56 pub provenance: BTreeMap<String, String>,
57 #[serde(default)]
59 pub measurement_units: Option<String>,
60}
61#[derive(Clone, Debug)]
63pub struct SpectralDataset {
64 source_path: PathBuf,
65 measurements: MeasurementStack,
66 model: SpectralImagePlaneModel,
67 manifest: SpectralDatasetManifest,
68 truth: Vec<Option<Array2<Complex64>>>,
69 masks: Vec<Option<Array2<u8>>>,
70}
71impl SpectralDataset {
72 pub fn source_path(&self) -> &Path {
74 &self.source_path
75 }
76 pub fn measurements(&self) -> &MeasurementStack {
78 &self.measurements
79 }
80 pub fn model(&self) -> &SpectralImagePlaneModel {
82 &self.model
83 }
84 pub fn manifest(&self) -> &SpectralDatasetManifest {
86 &self.manifest
87 }
88 pub fn ground_truth_objects(&self) -> &[Option<Array2<Complex64>>] {
90 &self.truth
91 }
92 pub fn valid_object_masks(&self) -> &[Option<Array2<u8>>] {
94 &self.masks
95 }
96 pub fn reconstruction_problem(
98 &self,
99 ) -> Result<SpectralReconstructionProblem<MeasurementStack>> {
100 SpectralReconstructionProblem::new(self.measurements.clone(), self.model.clone())
101 }
102}
103impl DatasetLoader {
104 pub fn load_spectral(&self) -> Result<SpectralDataset> {
121 let root = self.root();
122 let manifest_path = contained(root, Path::new("dataset.json"))?;
123 let manifest: SpectralDatasetManifest =
124 serde_json::from_reader(BufReader::new(File::open(manifest_path)?))?;
125 if manifest.format_version != SPECTRAL_DATASET_FORMAT_VERSION {
126 return Err(Error::Dataset(
127 "load_spectral requires dataset format_version 2; use load for version 1".into(),
128 ));
129 }
130 if manifest
131 .measurement_units
132 .as_ref()
133 .is_some_and(|v| v.trim().is_empty())
134 || manifest
135 .provenance
136 .iter()
137 .any(|(k, v)| k.trim().is_empty() || v.trim().is_empty())
138 {
139 return Err(Error::Dataset(
140 "units and provenance entries must be nonempty".into(),
141 ));
142 }
143 let measurement_path = contained(root, &manifest.measurement_manifest)?;
144 let spec = MeasurementSpec::load(&measurement_path)?;
145 let base = measurement_path.parent().unwrap_or(root);
146 validate_measurement_files(base, &spec)?;
147 let measurements = MeasurementStack::from_manifest_definition(spec, base)?;
148 let mut channels = Vec::new();
149 let mut truth = Vec::new();
150 let mut masks = Vec::new();
151 for channel in &manifest.channels {
152 if channel.response_provenance.trim().is_empty() {
153 return Err(Error::Dataset(format!(
154 "channel '{}' requires response_provenance",
155 channel.channel_id
156 )));
157 }
158 let model: ImagePlaneModel = serde_json::from_reader(BufReader::new(File::open(
159 contained(root, &channel.compiled_model)?,
160 )?))?;
161 if model.sampling().wavelength != Some(channel.wavelength_vacuum_m) {
162 return Err(Error::Dataset(format!(
163 "channel '{}' wavelength differs from compiled kernel",
164 channel.channel_id
165 )));
166 }
167 if channel.valid_object_mask.is_some() && channel.ground_truth_object.is_none() {
168 return Err(Error::Dataset(
169 "spectral valid_object_mask requires ground_truth_object".into(),
170 ));
171 }
172 let object = channel
173 .ground_truth_object
174 .as_ref()
175 .map(|p| super::loader::load_json_array(contained(root, p)?, "spectral truth"))
176 .transpose()?;
177 let mask = channel
178 .valid_object_mask
179 .as_ref()
180 .map(|p| super::loader::load_json_array(contained(root, p)?, "spectral truth mask"))
181 .transpose()?;
182 if object.as_ref().is_some_and(|a: &Array2<Complex64>| {
183 a.dim() != model.reconstruction_shape()
184 || a.iter().any(|v| !v.re.is_finite() || !v.im.is_finite())
185 }) || mask.as_ref().is_some_and(|a: &Array2<u8>| {
186 a.dim() != model.reconstruction_shape() || a.iter().any(|&v| v > 1)
187 }) {
188 return Err(Error::Dataset(format!(
189 "channel '{}' truth or mask has invalid shape/values",
190 channel.channel_id
191 )));
192 }
193 channels.push(SpectralModelChannel {
194 channel_id: channel.channel_id.clone(),
195 model,
196 });
197 truth.push(object);
198 masks.push(mask);
199 }
200 let model = SpectralImagePlaneModel::from_compiled_channels(
201 channels,
202 manifest.acquisition.clone(),
203 manifest.object_coupling,
204 )?;
205 SpectralReconstructionProblem::new(&measurements, model.clone())?;
206 Ok(SpectralDataset {
207 source_path: root.to_owned(),
208 measurements,
209 model,
210 manifest,
211 truth,
212 masks,
213 })
214 }
215 pub fn format_version(&self) -> Result<u32> {
217 let value: serde_json::Value =
218 serde_json::from_reader(BufReader::new(File::open(self.manifest_path())?))?;
219 value
220 .get("format_version")
221 .and_then(|v| v.as_u64())
222 .and_then(|v| u32::try_from(v).ok())
223 .ok_or_else(|| Error::Dataset("dataset.json requires an integer format_version".into()))
224 }
225 pub(super) fn validate_profile(&self, expected: u32) -> Result<()> {
226 if self.format_version()? != expected {
227 return Err(Error::Dataset(
228 "registry format_version differs from dataset.json".into(),
229 ));
230 }
231 match expected {
232 1 => {
233 self.load()?;
234 }
235 2 => {
236 self.load_spectral()?;
237 }
238 _ => return Err(Error::Dataset("unsupported dataset profile".into())),
239 };
240 Ok(())
241 }
242}
243fn contained(root: &Path, path: &Path) -> Result<PathBuf> {
244 super::loader::validate_contained_path("spectral dataset artifact", path)?;
245 let resolved = root.join(path).canonicalize()?;
246 if !resolved.starts_with(root.canonicalize()?) {
247 return Err(Error::Dataset(format!(
248 "spectral artifact escapes its manifest directory: {}",
249 path.display()
250 )));
251 }
252 Ok(resolved)
253}
254fn validate_measurement_files(base: &Path, spec: &MeasurementSpec) -> Result<()> {
255 for frame in &spec.frames {
256 contained(base, &frame.path)?;
257 }
258 for path in [&spec.dark_frame, &spec.flat_field].into_iter().flatten() {
259 contained(base, path)?;
260 }
261 for set in [&spec.background, &spec.mask].into_iter().flatten() {
262 match set {
263 ImageSet::Single(p) => {
264 contained(base, p)?;
265 }
266 ImageSet::PerFrame(paths) => {
267 for p in paths {
268 contained(base, p)?;
269 }
270 }
271 }
272 }
273 Ok(())
274}