Skip to main content

fpm_rs/datasets/
registry.rs

1use std::{
2    collections::BTreeSet,
3    env, fs,
4    fs::{File, OpenOptions},
5    io::{Read, Write},
6    path::{Component, Path, PathBuf},
7    time::{SystemTime, UNIX_EPOCH},
8};
9
10use fs2::FileExt;
11use serde::{Deserialize, Serialize};
12use sha2::{Digest, Sha256};
13
14use crate::{Error, Result};
15
16use super::{
17    DATASET_FORMAT_VERSION, Dataset, DatasetLoader, SPECTRAL_DATASET_FORMAT_VERSION,
18    SpectralDataset,
19};
20
21/// Registry document version understood by this release.
22pub const DATASET_REGISTRY_VERSION: u32 = 1;
23/// Default registry source used when no API or environment override is set.
24pub const DEFAULT_DATASET_REGISTRY_URL: &str =
25    "https://raw.githubusercontent.com/hgrecco/fpm-rs/main/dataset_registry.json";
26/// Environment variable overriding [`DEFAULT_DATASET_REGISTRY_URL`].
27pub const DATASET_REGISTRY_URL_ENV: &str = "FPM_RS_DATASET_REGISTRY_URL";
28/// Environment variable overriding the platform dataset cache directory.
29pub const DATASET_CACHE_DIR_ENV: &str = "FPM_RS_DATASET_CACHE_DIR";
30
31const CACHE_MARKER: &str = ".fpm-rs-dataset-cache";
32const CACHE_MARKER_CONTENTS: &[u8] = b"fpm-rs managed dataset cache v1\n";
33const INSTALL_METADATA: &str = ".fpm-rs-install.json";
34const MAX_REGISTRY_BYTES: usize = 16 * 1024 * 1024;
35const MAX_ARCHIVE_ENTRIES: u64 = 100_000;
36const MAX_EXTRACTED_BYTES: u64 = 256 * 1024 * 1024 * 1024;
37
38/// Immutable compressed bundle information from a registry entry.
39#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(deny_unknown_fields)]
41pub struct DatasetArchive {
42    /// HTTPS or file URL for the immutable compressed tar archive.
43    pub url: String,
44    /// Lowercase hexadecimal SHA-256 digest of the archive bytes.
45    pub sha256: String,
46    /// Exact compressed archive size in bytes.
47    pub size_bytes: u64,
48}
49
50/// License metadata from a registry entry.
51#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(deny_unknown_fields)]
53pub struct DatasetLicense {
54    /// SPDX license identifier for the distributed dataset.
55    pub spdx: String,
56    /// Authoritative license text or record URL.
57    pub url: String,
58}
59
60/// Preferred citation metadata from a registry entry.
61#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
62#[serde(deny_unknown_fields)]
63pub struct DatasetCitation {
64    /// Publication DOI identifier recorded by the registry contract.
65    pub doi: String,
66    /// Complete human-readable bibliographic citation.
67    pub text: String,
68}
69
70/// Original-source provenance from a registry entry.
71#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
72#[serde(deny_unknown_fields)]
73pub struct DatasetSource {
74    /// Authoritative original dataset or project URL.
75    pub url: String,
76    /// Description of source acquisition and external conversion provenance.
77    pub description: String,
78}
79
80/// One current immutable dataset version advertised by a registry.
81#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
82#[serde(deny_unknown_fields)]
83pub struct DatasetRegistryEntry {
84    /// Stable path-safe dataset identifier.
85    pub id: String,
86    /// Immutable path-safe dataset version.
87    pub version: String,
88    /// Short human-readable dataset title.
89    pub title: String,
90    /// Human-readable scientific and acquisition summary.
91    pub description: String,
92    /// Dataset profile expected after extraction: one for ordinary, two for spectral.
93    pub format_version: u32,
94    /// Immutable download location, digest, and byte size.
95    pub archive: DatasetArchive,
96    /// License identifier and authoritative URL.
97    pub license: DatasetLicense,
98    /// Preferred publication citation metadata.
99    pub citation: DatasetCitation,
100    /// Original-source provenance.
101    pub source: DatasetSource,
102    /// Search and filtering tags.
103    pub tags: Vec<String>,
104}
105
106impl DatasetRegistryEntry {
107    /// Validates registry-version-one metadata and supported dataset profile versions.
108    pub fn validate(&self) -> Result<()> {
109        validate_component("dataset id", &self.id)?;
110        validate_component("dataset version", &self.version)?;
111        for (label, value) in [
112            ("dataset title", self.title.as_str()),
113            ("dataset description", self.description.as_str()),
114            ("archive URL", self.archive.url.as_str()),
115            ("license SPDX identifier", self.license.spdx.as_str()),
116            ("license URL", self.license.url.as_str()),
117            ("citation DOI", self.citation.doi.as_str()),
118            ("citation text", self.citation.text.as_str()),
119            ("source URL", self.source.url.as_str()),
120            ("source description", self.source.description.as_str()),
121        ] {
122            validate_nonempty(label, value)?;
123        }
124        if !matches!(
125            self.format_version,
126            DATASET_FORMAT_VERSION | SPECTRAL_DATASET_FORMAT_VERSION
127        ) {
128            return Err(Error::Dataset(format!(
129                "dataset '{}' declares unsupported format version {}; expected 1 or 2",
130                self.id, self.format_version
131            )));
132        }
133        if self.archive.size_bytes == 0 {
134            return Err(Error::Dataset(format!(
135                "dataset '{}' archive size must be positive",
136                self.id
137            )));
138        }
139        if self.archive.sha256.len() != 64
140            || !self
141                .archive
142                .sha256
143                .bytes()
144                .all(|byte| byte.is_ascii_hexdigit())
145        {
146            return Err(Error::Dataset(format!(
147                "dataset '{}' SHA-256 must contain 64 hexadecimal characters",
148                self.id
149            )));
150        }
151        let archive_path = self
152            .archive
153            .url
154            .split(['?', '#'])
155            .next()
156            .unwrap_or_default();
157        if !archive_path.ends_with(".tar.zst") {
158            return Err(Error::Dataset(format!(
159                "dataset '{}' archive URL must identify a .tar.zst file",
160                self.id
161            )));
162        }
163        if self.tags.is_empty() {
164            return Err(Error::Dataset(format!(
165                "dataset '{}' must contain at least one tag",
166                self.id
167            )));
168        }
169        let mut tags = BTreeSet::new();
170        for tag in &self.tags {
171            validate_nonempty("dataset tag", tag)?;
172            if !tags.insert(tag) {
173                return Err(Error::Dataset(format!(
174                    "dataset '{}' contains duplicate tag '{tag}'",
175                    self.id
176                )));
177            }
178        }
179        Ok(())
180    }
181}
182
183/// Strict versioned dataset registry document.
184#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
185#[serde(deny_unknown_fields)]
186pub struct DatasetRegistryDocument {
187    /// Registry schema version; must equal [`DATASET_REGISTRY_VERSION`].
188    pub registry_version: u32,
189    /// Current immutable dataset versions, with unique [`DatasetRegistryEntry::id`] values.
190    pub datasets: Vec<DatasetRegistryEntry>,
191}
192
193impl DatasetRegistryDocument {
194    /// Parses and validates a registry JSON document.
195    pub fn from_slice(bytes: &[u8]) -> Result<Self> {
196        let document: Self = serde_json::from_slice(bytes)?;
197        document.validate()?;
198        Ok(document)
199    }
200
201    /// Validates the registry version, entry metadata, and ID uniqueness.
202    pub fn validate(&self) -> Result<()> {
203        if self.registry_version != DATASET_REGISTRY_VERSION {
204            return Err(Error::Dataset(format!(
205                "unsupported dataset registry version {}; expected {}",
206                self.registry_version, DATASET_REGISTRY_VERSION
207            )));
208        }
209        let mut identifiers = BTreeSet::new();
210        for entry in &self.datasets {
211            entry.validate()?;
212            if !identifiers.insert(&entry.id) {
213                return Err(Error::Dataset(format!(
214                    "dataset registry contains duplicate id '{}'",
215                    entry.id
216                )));
217            }
218        }
219        Ok(())
220    }
221
222    fn entry(&self, id: &str) -> Result<&DatasetRegistryEntry> {
223        self.datasets
224            .iter()
225            .find(|entry| entry.id == id)
226            .ok_or_else(|| Error::Dataset(format!("dataset registry has no entry '{id}'")))
227    }
228}
229
230/// A registry entry annotated with its current managed-cache status.
231#[derive(Clone, Debug, PartialEq, Eq)]
232pub struct DatasetListing {
233    /// Validated registry metadata.
234    pub entry: DatasetRegistryEntry,
235    /// Whether the current version is present and valid in the managed cache.
236    pub cached: bool,
237    /// Installed version directory when `cached` is true.
238    pub cache_path: Option<PathBuf>,
239}
240
241/// Registry client with verified downloads and a managed local cache.
242#[derive(Clone, Debug)]
243pub struct DatasetRegistry {
244    registry_url: String,
245    cache_dir: PathBuf,
246}
247
248impl DatasetRegistry {
249    /// Resolves the registry source and cache directory from environment and
250    /// platform defaults.
251    pub fn from_defaults() -> Result<Self> {
252        let registry_url = env::var(DATASET_REGISTRY_URL_ENV)
253            .ok()
254            .filter(|value| !value.trim().is_empty())
255            .unwrap_or_else(|| DEFAULT_DATASET_REGISTRY_URL.to_owned());
256        let cache_dir = env::var_os(DATASET_CACHE_DIR_ENV)
257            .filter(|value| !value.is_empty())
258            .map(PathBuf::from)
259            .or_else(|| {
260                dirs::cache_dir().map(|directory| directory.join("fpm-rs").join("datasets"))
261            })
262            .ok_or_else(|| {
263                Error::Dataset("could not determine the platform cache directory".into())
264            })?;
265        Self::new(registry_url, cache_dir)
266    }
267
268    /// Creates a registry using explicit source and cache values.
269    pub fn new(registry_url: impl Into<String>, cache_dir: impl Into<PathBuf>) -> Result<Self> {
270        let registry_url = registry_url.into();
271        validate_nonempty("dataset registry URL", &registry_url)?;
272        let cache_dir = cache_dir.into();
273        if cache_dir.as_os_str().is_empty() {
274            return Err(Error::Dataset(
275                "dataset cache directory must not be empty".into(),
276            ));
277        }
278        Ok(Self {
279            registry_url,
280            cache_dir,
281        })
282    }
283
284    /// Returns the configured registry URL, file URL, or filesystem path.
285    pub fn registry_url(&self) -> &str {
286        &self.registry_url
287    }
288
289    /// Returns the root of the managed cache.
290    pub fn cache_dir(&self) -> &Path {
291        &self.cache_dir
292    }
293
294    /// Fetches the registry and reports whether each current version is cached.
295    pub fn list(&self) -> Result<Vec<DatasetListing>> {
296        let registry = self.load_registry()?;
297        let _lock = self.lock_cache()?;
298        registry
299            .datasets
300            .into_iter()
301            .map(|entry| {
302                let path = self.dataset_path(&entry);
303                let cached = self.is_cached(&entry)?;
304                Ok(DatasetListing {
305                    entry,
306                    cached,
307                    cache_path: cached.then_some(path),
308                })
309            })
310            .collect()
311    }
312
313    /// Downloads, verifies, validates, and caches the current version of `id`.
314    ///
315    /// An already-valid current installation is returned unchanged.
316    pub fn download(&self, id: &str) -> Result<PathBuf> {
317        validate_component("dataset id", id)?;
318        let registry = self.load_registry()?;
319        let entry = registry.entry(id)?.clone();
320        self.install(&entry)
321    }
322
323    /// Downloads every registry entry sequentially and stops on the first error.
324    pub fn download_all(&self) -> Result<Vec<PathBuf>> {
325        let registry = self.load_registry()?;
326        registry
327            .datasets
328            .iter()
329            .map(|entry| self.install(entry))
330            .collect()
331    }
332
333    /// Opens `id`, downloading it first when necessary.
334    ///
335    /// A corrupt current installation is removed and installed once more.
336    pub fn open(&self, id: &str) -> Result<Dataset> {
337        let path = self.download(id)?;
338        if matches!(
339            DatasetLoader::new(&path)?.format_version(),
340            Ok(SPECTRAL_DATASET_FORMAT_VERSION)
341        ) {
342            return Err(Error::Dataset(
343                "spectral dataset requires open_spectral".into(),
344            ));
345        }
346        match self.load_cached_dataset(&path) {
347            Ok(dataset) => Ok(dataset),
348            Err(first_error) => {
349                let registry = self.load_registry()?;
350                let entry = registry.entry(id)?.clone();
351                {
352                    let _lock = self.lock_cache()?;
353                    if path.exists() {
354                        self.validate_managed_tree(&path)?;
355                        fs::remove_dir_all(&path)?;
356                    }
357                }
358                let repaired = self.install(&entry)?;
359                self.load_cached_dataset(&repaired).map_err(|second_error| {
360                    Error::Dataset(format!(
361                        "cached dataset '{id}' was invalid ({first_error}); reinstall also failed: {second_error}"
362                    ))
363                })
364            }
365        }
366    }
367
368    /// Opens an explicit spectral registry profile, downloading a registered ID if needed.
369    /// Uses the same verified archive and managed cache as ordinary data. Version-one
370    /// profiles must use `open`; local converted profiles use `DatasetLoader::load_spectral`.
371    pub fn open_spectral(&self, id: &str) -> Result<SpectralDataset> {
372        let path = self.download(id)?;
373        if matches!(
374            DatasetLoader::new(&path)?.format_version(),
375            Ok(DATASET_FORMAT_VERSION)
376        ) {
377            return Err(Error::Dataset("ordinary dataset requires open".into()));
378        }
379        let load = |path: &Path| -> Result<SpectralDataset> {
380            let _lock = self.lock_cache()?;
381            DatasetLoader::new(path)?.load_spectral()
382        };
383        match load(&path) {
384            Ok(dataset) => Ok(dataset),
385            Err(first_error) => {
386                let registry = self.load_registry()?;
387                let entry = registry.entry(id)?.clone();
388                {
389                    let _lock = self.lock_cache()?;
390                    if path.exists() {
391                        self.validate_managed_tree(&path)?;
392                        fs::remove_dir_all(&path)?;
393                    }
394                }
395                let repaired = self.install(&entry)?;
396                load(&repaired).map_err(|second_error| Error::Dataset(format!(
397                    "cached spectral dataset '{id}' was invalid ({first_error}); reinstall also failed: {second_error}"
398                )))
399            }
400        }
401    }
402
403    fn load_cached_dataset(&self, path: &Path) -> Result<Dataset> {
404        let _lock = self.lock_cache()?;
405        DatasetLoader::new(path)?.load()
406    }
407
408    /// Removes every cached version of `id`.
409    pub fn clean(&self, id: &str) -> Result<bool> {
410        validate_component("dataset id", id)?;
411        let _lock = self.lock_cache()?;
412        if !self.cache_dir.exists() {
413            return Ok(false);
414        }
415        self.validate_cache_root()?;
416        let path = self.cache_dir.join("datasets").join(id);
417        if !path.exists() {
418            return Ok(false);
419        }
420        self.validate_managed_tree(&path)?;
421        fs::remove_dir_all(path)?;
422        Ok(true)
423    }
424
425    /// Removes the complete marked managed cache and returns the version count.
426    pub fn clean_all(&self) -> Result<usize> {
427        let _lock = self.lock_cache()?;
428        if !self.cache_dir.exists() {
429            return Ok(0);
430        }
431        self.validate_cache_root()?;
432        self.validate_managed_tree(&self.cache_dir)?;
433        let count = count_cached_versions(&self.cache_dir.join("datasets"))?;
434        fs::remove_dir_all(&self.cache_dir)?;
435        Ok(count)
436    }
437
438    fn load_registry(&self) -> Result<DatasetRegistryDocument> {
439        match read_source_limited(&self.registry_url, MAX_REGISTRY_BYTES as u64) {
440            Ok(bytes) => {
441                let registry = DatasetRegistryDocument::from_slice(&bytes)?;
442                self.write_registry_snapshot(&bytes)?;
443                Ok(registry)
444            }
445            Err(fetch_error) => match self.read_registry_snapshot() {
446                Ok(bytes) => DatasetRegistryDocument::from_slice(&bytes),
447                Err(snapshot_error) => Err(Error::Dataset(format!(
448                    "failed to fetch registry '{}': {fetch_error}; no usable cached snapshot: {snapshot_error}",
449                    self.registry_url
450                ))),
451            },
452        }
453    }
454
455    fn install(&self, entry: &DatasetRegistryEntry) -> Result<PathBuf> {
456        let _lock = self.lock_cache()?;
457        self.ensure_cache_root()?;
458        let destination = self.dataset_path(entry);
459        if self.is_cached(entry)? {
460            return Ok(destination);
461        }
462        if destination.exists() {
463            self.validate_managed_tree(&destination)?;
464            fs::remove_dir_all(&destination)?;
465        }
466
467        let partial_root = self.cache_dir.join("partial");
468        fs::create_dir_all(&partial_root)?;
469        let nonce = unique_nonce();
470        let archive_path = partial_root.join(format!("{}-{nonce}.tar.zst", entry.id));
471        let staging_path = partial_root.join(format!("{}-{nonce}.bundle", entry.id));
472        fs::create_dir(&staging_path)?;
473
474        let result = (|| {
475            download_verified_archive(entry, &archive_path)?;
476            extract_archive(&archive_path, &staging_path)?;
477            if !staging_path.join("dataset.json").is_file() {
478                return Err(Error::Dataset(format!(
479                    "dataset '{}' archive must contain dataset.json at its root",
480                    entry.id
481                )));
482            }
483            DatasetLoader::new(&staging_path)?.validate_profile(entry.format_version)?;
484            let metadata = InstallMetadata {
485                cache_format_version: 1,
486                id: entry.id.clone(),
487                version: entry.version.clone(),
488                archive_sha256: entry.archive.sha256.to_ascii_lowercase(),
489            };
490            serde_json::to_writer_pretty(
491                File::create(staging_path.join(INSTALL_METADATA))?,
492                &metadata,
493            )?;
494            let parent = destination
495                .parent()
496                .ok_or_else(|| Error::Dataset("invalid destination for cached dataset".into()))?;
497            fs::create_dir_all(parent)?;
498            fs::rename(&staging_path, &destination)?;
499            Ok(destination.clone())
500        })();
501
502        let _ = fs::remove_file(&archive_path);
503        if staging_path.exists() {
504            let _ = fs::remove_dir_all(&staging_path);
505        }
506        result
507    }
508
509    fn dataset_path(&self, entry: &DatasetRegistryEntry) -> PathBuf {
510        self.cache_dir
511            .join("datasets")
512            .join(&entry.id)
513            .join(&entry.version)
514    }
515
516    fn is_cached(&self, entry: &DatasetRegistryEntry) -> Result<bool> {
517        let path = self.dataset_path(entry);
518        if !path.is_dir() || !path.join("dataset.json").is_file() {
519            return Ok(false);
520        }
521        self.validate_managed_tree(&path)?;
522        let metadata_path = path.join(INSTALL_METADATA);
523        if !metadata_path.is_file() {
524            return Ok(false);
525        }
526        let Ok(metadata) =
527            serde_json::from_reader::<_, InstallMetadata>(File::open(metadata_path)?)
528        else {
529            return Ok(false);
530        };
531        Ok(metadata.cache_format_version == 1
532            && metadata.id == entry.id
533            && metadata.version == entry.version
534            && metadata.archive_sha256 == entry.archive.sha256.to_ascii_lowercase())
535    }
536
537    fn snapshot_path(&self) -> PathBuf {
538        let digest = format!("{:x}", Sha256::digest(self.registry_url.as_bytes()));
539        self.cache_dir
540            .join("registries")
541            .join(format!("{digest}.json"))
542    }
543
544    fn write_registry_snapshot(&self, bytes: &[u8]) -> Result<()> {
545        let _lock = self.lock_cache()?;
546        self.ensure_cache_root()?;
547        let path = self.snapshot_path();
548        let parent = path
549            .parent()
550            .ok_or_else(|| Error::Dataset("invalid registry snapshot path".into()))?;
551        fs::create_dir_all(parent)?;
552        let temporary = path.with_extension(format!("json.{}.partial", unique_nonce()));
553        fs::write(&temporary, bytes)?;
554        replace_file(&temporary, &path)?;
555        Ok(())
556    }
557
558    fn read_registry_snapshot(&self) -> Result<Vec<u8>> {
559        if !self.cache_dir.exists() {
560            return Err(Error::Dataset("dataset cache does not exist".into()));
561        }
562        self.validate_cache_root()?;
563        fs::read(self.snapshot_path()).map_err(Into::into)
564    }
565
566    fn ensure_cache_root(&self) -> Result<()> {
567        if !self.cache_dir.exists() {
568            fs::create_dir_all(&self.cache_dir)?;
569        }
570        let metadata = fs::symlink_metadata(&self.cache_dir)?;
571        if metadata.file_type().is_symlink() || !metadata.is_dir() {
572            return Err(Error::Dataset(format!(
573                "dataset cache root must be a real directory: {}",
574                self.cache_dir.display()
575            )));
576        }
577        let marker = self.cache_dir.join(CACHE_MARKER);
578        if marker.exists() {
579            return validate_marker(&marker);
580        }
581        if fs::read_dir(&self.cache_dir)?.next().is_some() {
582            return Err(Error::Dataset(format!(
583                "refusing to manage non-empty unmarked cache directory {}",
584                self.cache_dir.display()
585            )));
586        }
587        let mut file = OpenOptions::new()
588            .write(true)
589            .create_new(true)
590            .open(marker)?;
591        file.write_all(CACHE_MARKER_CONTENTS)?;
592        file.sync_all()?;
593        Ok(())
594    }
595
596    fn validate_cache_root(&self) -> Result<()> {
597        let metadata = fs::symlink_metadata(&self.cache_dir)?;
598        if metadata.file_type().is_symlink() || !metadata.is_dir() {
599            return Err(Error::Dataset(format!(
600                "dataset cache root must be a real directory: {}",
601                self.cache_dir.display()
602            )));
603        }
604        validate_marker(&self.cache_dir.join(CACHE_MARKER))
605    }
606
607    fn validate_managed_tree(&self, path: &Path) -> Result<()> {
608        let relative = path.strip_prefix(&self.cache_dir).map_err(|_| {
609            Error::Dataset(format!(
610                "managed dataset path escapes cache root: {}",
611                path.display()
612            ))
613        })?;
614        if relative
615            .components()
616            .any(|component| !matches!(component, Component::Normal(_)))
617        {
618            return Err(Error::Dataset(format!(
619                "managed dataset path is unsafe: {}",
620                path.display()
621            )));
622        }
623        reject_symlinks(path)
624    }
625
626    fn lock_cache(&self) -> Result<File> {
627        let parent = self.cache_dir.parent().ok_or_else(|| {
628            Error::Dataset("dataset cache root must have a parent directory".into())
629        })?;
630        fs::create_dir_all(parent)?;
631        let name = self
632            .cache_dir
633            .file_name()
634            .and_then(|value| value.to_str())
635            .unwrap_or("datasets");
636        let path = parent.join(format!(".{name}.lock"));
637        let file = OpenOptions::new()
638            .read(true)
639            .write(true)
640            .create(true)
641            .truncate(false)
642            .open(path)?;
643        FileExt::lock_exclusive(&file)?;
644        Ok(file)
645    }
646}
647
648/// Opens a registry dataset using environment and platform defaults.
649pub fn open_dataset(id: &str) -> Result<Dataset> {
650    DatasetRegistry::from_defaults()?.open(id)
651}
652
653#[derive(Debug, Serialize, Deserialize)]
654#[serde(deny_unknown_fields)]
655struct InstallMetadata {
656    cache_format_version: u32,
657    id: String,
658    version: String,
659    archive_sha256: String,
660}
661
662fn download_verified_archive(entry: &DatasetRegistryEntry, destination: &Path) -> Result<()> {
663    let mut reader = open_source_reader(&entry.archive.url)?;
664    let mut output = OpenOptions::new()
665        .write(true)
666        .create_new(true)
667        .open(destination)?;
668    let mut digest = Sha256::new();
669    let mut buffer = [0_u8; 64 * 1024];
670    let mut total = 0_u64;
671    loop {
672        let count = reader.read(&mut buffer)?;
673        if count == 0 {
674            break;
675        }
676        total = total
677            .checked_add(count as u64)
678            .ok_or_else(|| Error::Dataset("downloaded archive size overflowed".into()))?;
679        if total > entry.archive.size_bytes {
680            return Err(Error::Dataset(format!(
681                "dataset '{}' archive exceeds declared size {} bytes",
682                entry.id, entry.archive.size_bytes
683            )));
684        }
685        digest.update(&buffer[..count]);
686        output.write_all(&buffer[..count])?;
687    }
688    output.sync_all()?;
689    if total != entry.archive.size_bytes {
690        return Err(Error::Dataset(format!(
691            "dataset '{}' archive size mismatch: expected {}, got {total}",
692            entry.id, entry.archive.size_bytes
693        )));
694    }
695    let actual = format!("{:x}", digest.finalize());
696    if actual != entry.archive.sha256.to_ascii_lowercase() {
697        return Err(Error::Dataset(format!(
698            "dataset '{}' SHA-256 mismatch: expected {}, got {actual}",
699            entry.id, entry.archive.sha256
700        )));
701    }
702    Ok(())
703}
704
705fn extract_archive(archive_path: &Path, destination: &Path) -> Result<()> {
706    let decoder = zstd::Decoder::new(File::open(archive_path)?)?;
707    let mut archive = tar::Archive::new(decoder);
708    let mut paths = BTreeSet::new();
709    let mut remaining_entries = MAX_ARCHIVE_ENTRIES;
710    let mut remaining_bytes = MAX_EXTRACTED_BYTES;
711    for entry in archive.entries()? {
712        let mut entry = entry?;
713        remaining_entries = remaining_entries.checked_sub(1).ok_or_else(|| {
714            Error::Dataset("dataset archive exceeds the extraction entry limit".into())
715        })?;
716        let relative = entry.path()?.into_owned();
717        validate_archive_path(&relative)?;
718        if !paths.insert(relative.clone()) {
719            return Err(Error::Dataset(format!(
720                "dataset archive contains duplicate path {}",
721                relative.display()
722            )));
723        }
724        let entry_type = entry.header().entry_type();
725        let target = destination.join(&relative);
726        if entry_type.is_dir() {
727            fs::create_dir_all(target)?;
728            continue;
729        }
730        if !entry_type.is_file() {
731            return Err(Error::Dataset(format!(
732                "dataset archive entry is not a regular file or directory: {}",
733                relative.display()
734            )));
735        }
736        let size = entry.size();
737        remaining_bytes = remaining_bytes.checked_sub(size).ok_or_else(|| {
738            Error::Dataset("dataset archive exceeds the extraction byte limit".into())
739        })?;
740        if let Some(parent) = target.parent() {
741            fs::create_dir_all(parent)?;
742        }
743        let mut output = OpenOptions::new()
744            .write(true)
745            .create_new(true)
746            .open(&target)?;
747        let copied = std::io::copy(&mut entry, &mut output)?;
748        if copied != size {
749            return Err(Error::Dataset(format!(
750                "dataset archive entry size mismatch for {}",
751                relative.display()
752            )));
753        }
754    }
755    Ok(())
756}
757
758fn read_source_limited(source: &str, limit: u64) -> Result<Vec<u8>> {
759    let mut reader = open_source_reader(source)?;
760    let mut bytes = Vec::new();
761    let copied = std::io::copy(&mut reader.by_ref().take(limit + 1), &mut bytes)?;
762    if copied > limit {
763        return Err(Error::Dataset(format!(
764            "source exceeds the {limit}-byte limit: {source}"
765        )));
766    }
767    Ok(bytes)
768}
769
770fn open_source_reader(source: &str) -> Result<Box<dyn Read>> {
771    if source.starts_with("http://") || source.starts_with("https://") {
772        let response = ureq::get(source)
773            .call()
774            .map_err(|error| Error::Dataset(format!("request failed for {source}: {error}")))?;
775        return Ok(Box::new(response.into_body().into_reader()));
776    }
777    let path = source.strip_prefix("file://").unwrap_or(source);
778    Ok(Box::new(File::open(path).map_err(|error| {
779        Error::Dataset(format!("failed to open source {source}: {error}"))
780    })?))
781}
782
783fn validate_archive_path(path: &Path) -> Result<()> {
784    if path.as_os_str().is_empty()
785        || path.is_absolute()
786        || path
787            .components()
788            .any(|component| !matches!(component, Component::Normal(_)))
789    {
790        return Err(Error::Dataset(format!(
791            "dataset archive entry path is unsafe: {}",
792            path.display()
793        )));
794    }
795    Ok(())
796}
797
798fn validate_component(label: &str, value: &str) -> Result<()> {
799    if value.is_empty()
800        || value == "."
801        || value == ".."
802        || !value
803            .bytes()
804            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
805    {
806        return Err(Error::Dataset(format!(
807            "{label} must contain only letters, digits, '.', '_', or '-'"
808        )));
809    }
810    Ok(())
811}
812
813fn validate_nonempty(label: &str, value: &str) -> Result<()> {
814    if value.trim().is_empty() {
815        return Err(Error::Dataset(format!("{label} must not be empty")));
816    }
817    Ok(())
818}
819
820fn validate_marker(path: &Path) -> Result<()> {
821    let metadata = fs::symlink_metadata(path).map_err(|error| {
822        Error::Dataset(format!(
823            "dataset cache ownership marker is missing at {}: {error}",
824            path.display()
825        ))
826    })?;
827    if metadata.file_type().is_symlink() || !metadata.is_file() {
828        return Err(Error::Dataset(format!(
829            "dataset cache ownership marker must be a regular file: {}",
830            path.display()
831        )));
832    }
833    if fs::read(path)? != CACHE_MARKER_CONTENTS {
834        return Err(Error::Dataset(format!(
835            "dataset cache ownership marker is invalid: {}",
836            path.display()
837        )));
838    }
839    Ok(())
840}
841
842fn reject_symlinks(path: &Path) -> Result<()> {
843    let metadata = fs::symlink_metadata(path)?;
844    if metadata.file_type().is_symlink() {
845        return Err(Error::Dataset(format!(
846            "managed cache path contains a symlink: {}",
847            path.display()
848        )));
849    }
850    if metadata.is_dir() {
851        for entry in fs::read_dir(path)? {
852            reject_symlinks(&entry?.path())?;
853        }
854    }
855    Ok(())
856}
857
858fn count_cached_versions(root: &Path) -> Result<usize> {
859    if !root.is_dir() {
860        return Ok(0);
861    }
862    let mut count = 0;
863    for dataset in fs::read_dir(root)? {
864        let dataset = dataset?;
865        if dataset.file_type()?.is_dir() {
866            count += fs::read_dir(dataset.path())?
867                .filter_map(std::result::Result::ok)
868                .filter_map(|entry| entry.file_type().ok())
869                .filter(|kind| kind.is_dir())
870                .count();
871        }
872    }
873    Ok(count)
874}
875
876fn unique_nonce() -> String {
877    let nanos = SystemTime::now()
878        .duration_since(UNIX_EPOCH)
879        .map_or(0, |duration| duration.as_nanos());
880    format!("{}-{nanos}", std::process::id())
881}
882
883fn replace_file(source: &Path, destination: &Path) -> Result<()> {
884    match fs::rename(source, destination) {
885        Ok(()) => Ok(()),
886        Err(error)
887            if destination.exists()
888                && matches!(
889                    error.kind(),
890                    std::io::ErrorKind::AlreadyExists | std::io::ErrorKind::PermissionDenied
891                ) =>
892        {
893            fs::remove_file(destination)?;
894            fs::rename(source, destination)?;
895            Ok(())
896        }
897        Err(error) => Err(error.into()),
898    }
899}
900
901#[cfg(test)]
902mod tests {
903    use super::*;
904
905    #[test]
906    fn archive_paths_reject_escape_and_non_normal_components() {
907        for path in [
908            "",
909            ".",
910            "./dataset.json",
911            "../dataset.json",
912            "/dataset.json",
913        ] {
914            assert!(
915                validate_archive_path(Path::new(path)).is_err(),
916                "accepted {path:?}"
917            );
918        }
919        assert!(validate_archive_path(Path::new("frames/frame.tiff")).is_ok());
920    }
921
922    #[test]
923    fn archive_extraction_rejects_links_and_duplicate_paths() -> Result<()> {
924        let temporary = tempfile::tempdir()?;
925        let link_archive = temporary.path().join("link.tar.zst");
926        {
927            let encoder = zstd::Encoder::new(File::create(&link_archive)?, 1)?;
928            let mut archive = tar::Builder::new(encoder);
929            let mut header = tar::Header::new_gnu();
930            header.set_entry_type(tar::EntryType::Symlink);
931            header.set_size(0);
932            header.set_mode(0o777);
933            header.set_link_name("outside")?;
934            header.set_cksum();
935            archive.append_data(&mut header, "link", std::io::empty())?;
936            archive.into_inner()?.finish()?;
937        }
938        let destination = temporary.path().join("link-output");
939        fs::create_dir(&destination)?;
940        assert!(
941            extract_archive(&link_archive, &destination)
942                .unwrap_err()
943                .to_string()
944                .contains("not a regular file or directory")
945        );
946
947        let duplicate_archive = temporary.path().join("duplicate.tar.zst");
948        {
949            let encoder = zstd::Encoder::new(File::create(&duplicate_archive)?, 1)?;
950            let mut archive = tar::Builder::new(encoder);
951            for payload in [b"first".as_slice(), b"second".as_slice()] {
952                let mut header = tar::Header::new_gnu();
953                header.set_size(payload.len() as u64);
954                header.set_mode(0o644);
955                header.set_cksum();
956                archive.append_data(&mut header, "duplicate", payload)?;
957            }
958            archive.into_inner()?.finish()?;
959        }
960        let destination = temporary.path().join("duplicate-output");
961        fs::create_dir(&destination)?;
962        assert!(
963            extract_archive(&duplicate_archive, &destination)
964                .unwrap_err()
965                .to_string()
966                .contains("duplicate path")
967        );
968        Ok(())
969    }
970}