1use std::{
2 collections::BTreeSet,
3 env, fs,
4 fs::{File, OpenOptions},
5 io::{Read, Write},
6 path::{Component, Path, PathBuf},
7 time::{SystemTime, UNIX_EPOCH},
8};
9
10use fs2::FileExt;
11use serde::{Deserialize, Serialize};
12use sha2::{Digest, Sha256};
13
14use crate::{Error, Result};
15
16use super::{
17 DATASET_FORMAT_VERSION, Dataset, DatasetLoader, SPECTRAL_DATASET_FORMAT_VERSION,
18 SpectralDataset,
19};
20
21pub const DATASET_REGISTRY_VERSION: u32 = 1;
23pub const DEFAULT_DATASET_REGISTRY_URL: &str =
25 "https://raw.githubusercontent.com/hgrecco/fpm-rs/main/dataset_registry.json";
26pub const DATASET_REGISTRY_URL_ENV: &str = "FPM_RS_DATASET_REGISTRY_URL";
28pub const DATASET_CACHE_DIR_ENV: &str = "FPM_RS_DATASET_CACHE_DIR";
30
31const CACHE_MARKER: &str = ".fpm-rs-dataset-cache";
32const CACHE_MARKER_CONTENTS: &[u8] = b"fpm-rs managed dataset cache v1\n";
33const INSTALL_METADATA: &str = ".fpm-rs-install.json";
34const MAX_REGISTRY_BYTES: usize = 16 * 1024 * 1024;
35const MAX_ARCHIVE_ENTRIES: u64 = 100_000;
36const MAX_EXTRACTED_BYTES: u64 = 256 * 1024 * 1024 * 1024;
37
38#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
40#[serde(deny_unknown_fields)]
41pub struct DatasetArchive {
42 pub url: String,
44 pub sha256: String,
46 pub size_bytes: u64,
48}
49
50#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
52#[serde(deny_unknown_fields)]
53pub struct DatasetLicense {
54 pub spdx: String,
56 pub url: String,
58}
59
60#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
62#[serde(deny_unknown_fields)]
63pub struct DatasetCitation {
64 pub doi: String,
66 pub text: String,
68}
69
70#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
72#[serde(deny_unknown_fields)]
73pub struct DatasetSource {
74 pub url: String,
76 pub description: String,
78}
79
80#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
82#[serde(deny_unknown_fields)]
83pub struct DatasetRegistryEntry {
84 pub id: String,
86 pub version: String,
88 pub title: String,
90 pub description: String,
92 pub format_version: u32,
94 pub archive: DatasetArchive,
96 pub license: DatasetLicense,
98 pub citation: DatasetCitation,
100 pub source: DatasetSource,
102 pub tags: Vec<String>,
104}
105
106impl DatasetRegistryEntry {
107 pub fn validate(&self) -> Result<()> {
109 validate_component("dataset id", &self.id)?;
110 validate_component("dataset version", &self.version)?;
111 for (label, value) in [
112 ("dataset title", self.title.as_str()),
113 ("dataset description", self.description.as_str()),
114 ("archive URL", self.archive.url.as_str()),
115 ("license SPDX identifier", self.license.spdx.as_str()),
116 ("license URL", self.license.url.as_str()),
117 ("citation DOI", self.citation.doi.as_str()),
118 ("citation text", self.citation.text.as_str()),
119 ("source URL", self.source.url.as_str()),
120 ("source description", self.source.description.as_str()),
121 ] {
122 validate_nonempty(label, value)?;
123 }
124 if !matches!(
125 self.format_version,
126 DATASET_FORMAT_VERSION | SPECTRAL_DATASET_FORMAT_VERSION
127 ) {
128 return Err(Error::Dataset(format!(
129 "dataset '{}' declares unsupported format version {}; expected 1 or 2",
130 self.id, self.format_version
131 )));
132 }
133 if self.archive.size_bytes == 0 {
134 return Err(Error::Dataset(format!(
135 "dataset '{}' archive size must be positive",
136 self.id
137 )));
138 }
139 if self.archive.sha256.len() != 64
140 || !self
141 .archive
142 .sha256
143 .bytes()
144 .all(|byte| byte.is_ascii_hexdigit())
145 {
146 return Err(Error::Dataset(format!(
147 "dataset '{}' SHA-256 must contain 64 hexadecimal characters",
148 self.id
149 )));
150 }
151 let archive_path = self
152 .archive
153 .url
154 .split(['?', '#'])
155 .next()
156 .unwrap_or_default();
157 if !archive_path.ends_with(".tar.zst") {
158 return Err(Error::Dataset(format!(
159 "dataset '{}' archive URL must identify a .tar.zst file",
160 self.id
161 )));
162 }
163 if self.tags.is_empty() {
164 return Err(Error::Dataset(format!(
165 "dataset '{}' must contain at least one tag",
166 self.id
167 )));
168 }
169 let mut tags = BTreeSet::new();
170 for tag in &self.tags {
171 validate_nonempty("dataset tag", tag)?;
172 if !tags.insert(tag) {
173 return Err(Error::Dataset(format!(
174 "dataset '{}' contains duplicate tag '{tag}'",
175 self.id
176 )));
177 }
178 }
179 Ok(())
180 }
181}
182
183#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
185#[serde(deny_unknown_fields)]
186pub struct DatasetRegistryDocument {
187 pub registry_version: u32,
189 pub datasets: Vec<DatasetRegistryEntry>,
191}
192
193impl DatasetRegistryDocument {
194 pub fn from_slice(bytes: &[u8]) -> Result<Self> {
196 let document: Self = serde_json::from_slice(bytes)?;
197 document.validate()?;
198 Ok(document)
199 }
200
201 pub fn validate(&self) -> Result<()> {
203 if self.registry_version != DATASET_REGISTRY_VERSION {
204 return Err(Error::Dataset(format!(
205 "unsupported dataset registry version {}; expected {}",
206 self.registry_version, DATASET_REGISTRY_VERSION
207 )));
208 }
209 let mut identifiers = BTreeSet::new();
210 for entry in &self.datasets {
211 entry.validate()?;
212 if !identifiers.insert(&entry.id) {
213 return Err(Error::Dataset(format!(
214 "dataset registry contains duplicate id '{}'",
215 entry.id
216 )));
217 }
218 }
219 Ok(())
220 }
221
222 fn entry(&self, id: &str) -> Result<&DatasetRegistryEntry> {
223 self.datasets
224 .iter()
225 .find(|entry| entry.id == id)
226 .ok_or_else(|| Error::Dataset(format!("dataset registry has no entry '{id}'")))
227 }
228}
229
230#[derive(Clone, Debug, PartialEq, Eq)]
232pub struct DatasetListing {
233 pub entry: DatasetRegistryEntry,
235 pub cached: bool,
237 pub cache_path: Option<PathBuf>,
239}
240
241#[derive(Clone, Debug)]
243pub struct DatasetRegistry {
244 registry_url: String,
245 cache_dir: PathBuf,
246}
247
248impl DatasetRegistry {
249 pub fn from_defaults() -> Result<Self> {
252 let registry_url = env::var(DATASET_REGISTRY_URL_ENV)
253 .ok()
254 .filter(|value| !value.trim().is_empty())
255 .unwrap_or_else(|| DEFAULT_DATASET_REGISTRY_URL.to_owned());
256 let cache_dir = env::var_os(DATASET_CACHE_DIR_ENV)
257 .filter(|value| !value.is_empty())
258 .map(PathBuf::from)
259 .or_else(|| {
260 dirs::cache_dir().map(|directory| directory.join("fpm-rs").join("datasets"))
261 })
262 .ok_or_else(|| {
263 Error::Dataset("could not determine the platform cache directory".into())
264 })?;
265 Self::new(registry_url, cache_dir)
266 }
267
268 pub fn new(registry_url: impl Into<String>, cache_dir: impl Into<PathBuf>) -> Result<Self> {
270 let registry_url = registry_url.into();
271 validate_nonempty("dataset registry URL", ®istry_url)?;
272 let cache_dir = cache_dir.into();
273 if cache_dir.as_os_str().is_empty() {
274 return Err(Error::Dataset(
275 "dataset cache directory must not be empty".into(),
276 ));
277 }
278 Ok(Self {
279 registry_url,
280 cache_dir,
281 })
282 }
283
284 pub fn registry_url(&self) -> &str {
286 &self.registry_url
287 }
288
289 pub fn cache_dir(&self) -> &Path {
291 &self.cache_dir
292 }
293
294 pub fn list(&self) -> Result<Vec<DatasetListing>> {
296 let registry = self.load_registry()?;
297 let _lock = self.lock_cache()?;
298 registry
299 .datasets
300 .into_iter()
301 .map(|entry| {
302 let path = self.dataset_path(&entry);
303 let cached = self.is_cached(&entry)?;
304 Ok(DatasetListing {
305 entry,
306 cached,
307 cache_path: cached.then_some(path),
308 })
309 })
310 .collect()
311 }
312
313 pub fn download(&self, id: &str) -> Result<PathBuf> {
317 validate_component("dataset id", id)?;
318 let registry = self.load_registry()?;
319 let entry = registry.entry(id)?.clone();
320 self.install(&entry)
321 }
322
323 pub fn download_all(&self) -> Result<Vec<PathBuf>> {
325 let registry = self.load_registry()?;
326 registry
327 .datasets
328 .iter()
329 .map(|entry| self.install(entry))
330 .collect()
331 }
332
333 pub fn open(&self, id: &str) -> Result<Dataset> {
337 let path = self.download(id)?;
338 if matches!(
339 DatasetLoader::new(&path)?.format_version(),
340 Ok(SPECTRAL_DATASET_FORMAT_VERSION)
341 ) {
342 return Err(Error::Dataset(
343 "spectral dataset requires open_spectral".into(),
344 ));
345 }
346 match self.load_cached_dataset(&path) {
347 Ok(dataset) => Ok(dataset),
348 Err(first_error) => {
349 let registry = self.load_registry()?;
350 let entry = registry.entry(id)?.clone();
351 {
352 let _lock = self.lock_cache()?;
353 if path.exists() {
354 self.validate_managed_tree(&path)?;
355 fs::remove_dir_all(&path)?;
356 }
357 }
358 let repaired = self.install(&entry)?;
359 self.load_cached_dataset(&repaired).map_err(|second_error| {
360 Error::Dataset(format!(
361 "cached dataset '{id}' was invalid ({first_error}); reinstall also failed: {second_error}"
362 ))
363 })
364 }
365 }
366 }
367
368 pub fn open_spectral(&self, id: &str) -> Result<SpectralDataset> {
372 let path = self.download(id)?;
373 if matches!(
374 DatasetLoader::new(&path)?.format_version(),
375 Ok(DATASET_FORMAT_VERSION)
376 ) {
377 return Err(Error::Dataset("ordinary dataset requires open".into()));
378 }
379 let load = |path: &Path| -> Result<SpectralDataset> {
380 let _lock = self.lock_cache()?;
381 DatasetLoader::new(path)?.load_spectral()
382 };
383 match load(&path) {
384 Ok(dataset) => Ok(dataset),
385 Err(first_error) => {
386 let registry = self.load_registry()?;
387 let entry = registry.entry(id)?.clone();
388 {
389 let _lock = self.lock_cache()?;
390 if path.exists() {
391 self.validate_managed_tree(&path)?;
392 fs::remove_dir_all(&path)?;
393 }
394 }
395 let repaired = self.install(&entry)?;
396 load(&repaired).map_err(|second_error| Error::Dataset(format!(
397 "cached spectral dataset '{id}' was invalid ({first_error}); reinstall also failed: {second_error}"
398 )))
399 }
400 }
401 }
402
403 fn load_cached_dataset(&self, path: &Path) -> Result<Dataset> {
404 let _lock = self.lock_cache()?;
405 DatasetLoader::new(path)?.load()
406 }
407
408 pub fn clean(&self, id: &str) -> Result<bool> {
410 validate_component("dataset id", id)?;
411 let _lock = self.lock_cache()?;
412 if !self.cache_dir.exists() {
413 return Ok(false);
414 }
415 self.validate_cache_root()?;
416 let path = self.cache_dir.join("datasets").join(id);
417 if !path.exists() {
418 return Ok(false);
419 }
420 self.validate_managed_tree(&path)?;
421 fs::remove_dir_all(path)?;
422 Ok(true)
423 }
424
425 pub fn clean_all(&self) -> Result<usize> {
427 let _lock = self.lock_cache()?;
428 if !self.cache_dir.exists() {
429 return Ok(0);
430 }
431 self.validate_cache_root()?;
432 self.validate_managed_tree(&self.cache_dir)?;
433 let count = count_cached_versions(&self.cache_dir.join("datasets"))?;
434 fs::remove_dir_all(&self.cache_dir)?;
435 Ok(count)
436 }
437
438 fn load_registry(&self) -> Result<DatasetRegistryDocument> {
439 match read_source_limited(&self.registry_url, MAX_REGISTRY_BYTES as u64) {
440 Ok(bytes) => {
441 let registry = DatasetRegistryDocument::from_slice(&bytes)?;
442 self.write_registry_snapshot(&bytes)?;
443 Ok(registry)
444 }
445 Err(fetch_error) => match self.read_registry_snapshot() {
446 Ok(bytes) => DatasetRegistryDocument::from_slice(&bytes),
447 Err(snapshot_error) => Err(Error::Dataset(format!(
448 "failed to fetch registry '{}': {fetch_error}; no usable cached snapshot: {snapshot_error}",
449 self.registry_url
450 ))),
451 },
452 }
453 }
454
455 fn install(&self, entry: &DatasetRegistryEntry) -> Result<PathBuf> {
456 let _lock = self.lock_cache()?;
457 self.ensure_cache_root()?;
458 let destination = self.dataset_path(entry);
459 if self.is_cached(entry)? {
460 return Ok(destination);
461 }
462 if destination.exists() {
463 self.validate_managed_tree(&destination)?;
464 fs::remove_dir_all(&destination)?;
465 }
466
467 let partial_root = self.cache_dir.join("partial");
468 fs::create_dir_all(&partial_root)?;
469 let nonce = unique_nonce();
470 let archive_path = partial_root.join(format!("{}-{nonce}.tar.zst", entry.id));
471 let staging_path = partial_root.join(format!("{}-{nonce}.bundle", entry.id));
472 fs::create_dir(&staging_path)?;
473
474 let result = (|| {
475 download_verified_archive(entry, &archive_path)?;
476 extract_archive(&archive_path, &staging_path)?;
477 if !staging_path.join("dataset.json").is_file() {
478 return Err(Error::Dataset(format!(
479 "dataset '{}' archive must contain dataset.json at its root",
480 entry.id
481 )));
482 }
483 DatasetLoader::new(&staging_path)?.validate_profile(entry.format_version)?;
484 let metadata = InstallMetadata {
485 cache_format_version: 1,
486 id: entry.id.clone(),
487 version: entry.version.clone(),
488 archive_sha256: entry.archive.sha256.to_ascii_lowercase(),
489 };
490 serde_json::to_writer_pretty(
491 File::create(staging_path.join(INSTALL_METADATA))?,
492 &metadata,
493 )?;
494 let parent = destination
495 .parent()
496 .ok_or_else(|| Error::Dataset("invalid destination for cached dataset".into()))?;
497 fs::create_dir_all(parent)?;
498 fs::rename(&staging_path, &destination)?;
499 Ok(destination.clone())
500 })();
501
502 let _ = fs::remove_file(&archive_path);
503 if staging_path.exists() {
504 let _ = fs::remove_dir_all(&staging_path);
505 }
506 result
507 }
508
509 fn dataset_path(&self, entry: &DatasetRegistryEntry) -> PathBuf {
510 self.cache_dir
511 .join("datasets")
512 .join(&entry.id)
513 .join(&entry.version)
514 }
515
516 fn is_cached(&self, entry: &DatasetRegistryEntry) -> Result<bool> {
517 let path = self.dataset_path(entry);
518 if !path.is_dir() || !path.join("dataset.json").is_file() {
519 return Ok(false);
520 }
521 self.validate_managed_tree(&path)?;
522 let metadata_path = path.join(INSTALL_METADATA);
523 if !metadata_path.is_file() {
524 return Ok(false);
525 }
526 let Ok(metadata) =
527 serde_json::from_reader::<_, InstallMetadata>(File::open(metadata_path)?)
528 else {
529 return Ok(false);
530 };
531 Ok(metadata.cache_format_version == 1
532 && metadata.id == entry.id
533 && metadata.version == entry.version
534 && metadata.archive_sha256 == entry.archive.sha256.to_ascii_lowercase())
535 }
536
537 fn snapshot_path(&self) -> PathBuf {
538 let digest = format!("{:x}", Sha256::digest(self.registry_url.as_bytes()));
539 self.cache_dir
540 .join("registries")
541 .join(format!("{digest}.json"))
542 }
543
544 fn write_registry_snapshot(&self, bytes: &[u8]) -> Result<()> {
545 let _lock = self.lock_cache()?;
546 self.ensure_cache_root()?;
547 let path = self.snapshot_path();
548 let parent = path
549 .parent()
550 .ok_or_else(|| Error::Dataset("invalid registry snapshot path".into()))?;
551 fs::create_dir_all(parent)?;
552 let temporary = path.with_extension(format!("json.{}.partial", unique_nonce()));
553 fs::write(&temporary, bytes)?;
554 replace_file(&temporary, &path)?;
555 Ok(())
556 }
557
558 fn read_registry_snapshot(&self) -> Result<Vec<u8>> {
559 if !self.cache_dir.exists() {
560 return Err(Error::Dataset("dataset cache does not exist".into()));
561 }
562 self.validate_cache_root()?;
563 fs::read(self.snapshot_path()).map_err(Into::into)
564 }
565
566 fn ensure_cache_root(&self) -> Result<()> {
567 if !self.cache_dir.exists() {
568 fs::create_dir_all(&self.cache_dir)?;
569 }
570 let metadata = fs::symlink_metadata(&self.cache_dir)?;
571 if metadata.file_type().is_symlink() || !metadata.is_dir() {
572 return Err(Error::Dataset(format!(
573 "dataset cache root must be a real directory: {}",
574 self.cache_dir.display()
575 )));
576 }
577 let marker = self.cache_dir.join(CACHE_MARKER);
578 if marker.exists() {
579 return validate_marker(&marker);
580 }
581 if fs::read_dir(&self.cache_dir)?.next().is_some() {
582 return Err(Error::Dataset(format!(
583 "refusing to manage non-empty unmarked cache directory {}",
584 self.cache_dir.display()
585 )));
586 }
587 let mut file = OpenOptions::new()
588 .write(true)
589 .create_new(true)
590 .open(marker)?;
591 file.write_all(CACHE_MARKER_CONTENTS)?;
592 file.sync_all()?;
593 Ok(())
594 }
595
596 fn validate_cache_root(&self) -> Result<()> {
597 let metadata = fs::symlink_metadata(&self.cache_dir)?;
598 if metadata.file_type().is_symlink() || !metadata.is_dir() {
599 return Err(Error::Dataset(format!(
600 "dataset cache root must be a real directory: {}",
601 self.cache_dir.display()
602 )));
603 }
604 validate_marker(&self.cache_dir.join(CACHE_MARKER))
605 }
606
607 fn validate_managed_tree(&self, path: &Path) -> Result<()> {
608 let relative = path.strip_prefix(&self.cache_dir).map_err(|_| {
609 Error::Dataset(format!(
610 "managed dataset path escapes cache root: {}",
611 path.display()
612 ))
613 })?;
614 if relative
615 .components()
616 .any(|component| !matches!(component, Component::Normal(_)))
617 {
618 return Err(Error::Dataset(format!(
619 "managed dataset path is unsafe: {}",
620 path.display()
621 )));
622 }
623 reject_symlinks(path)
624 }
625
626 fn lock_cache(&self) -> Result<File> {
627 let parent = self.cache_dir.parent().ok_or_else(|| {
628 Error::Dataset("dataset cache root must have a parent directory".into())
629 })?;
630 fs::create_dir_all(parent)?;
631 let name = self
632 .cache_dir
633 .file_name()
634 .and_then(|value| value.to_str())
635 .unwrap_or("datasets");
636 let path = parent.join(format!(".{name}.lock"));
637 let file = OpenOptions::new()
638 .read(true)
639 .write(true)
640 .create(true)
641 .truncate(false)
642 .open(path)?;
643 FileExt::lock_exclusive(&file)?;
644 Ok(file)
645 }
646}
647
648pub fn open_dataset(id: &str) -> Result<Dataset> {
650 DatasetRegistry::from_defaults()?.open(id)
651}
652
653#[derive(Debug, Serialize, Deserialize)]
654#[serde(deny_unknown_fields)]
655struct InstallMetadata {
656 cache_format_version: u32,
657 id: String,
658 version: String,
659 archive_sha256: String,
660}
661
662fn download_verified_archive(entry: &DatasetRegistryEntry, destination: &Path) -> Result<()> {
663 let mut reader = open_source_reader(&entry.archive.url)?;
664 let mut output = OpenOptions::new()
665 .write(true)
666 .create_new(true)
667 .open(destination)?;
668 let mut digest = Sha256::new();
669 let mut buffer = [0_u8; 64 * 1024];
670 let mut total = 0_u64;
671 loop {
672 let count = reader.read(&mut buffer)?;
673 if count == 0 {
674 break;
675 }
676 total = total
677 .checked_add(count as u64)
678 .ok_or_else(|| Error::Dataset("downloaded archive size overflowed".into()))?;
679 if total > entry.archive.size_bytes {
680 return Err(Error::Dataset(format!(
681 "dataset '{}' archive exceeds declared size {} bytes",
682 entry.id, entry.archive.size_bytes
683 )));
684 }
685 digest.update(&buffer[..count]);
686 output.write_all(&buffer[..count])?;
687 }
688 output.sync_all()?;
689 if total != entry.archive.size_bytes {
690 return Err(Error::Dataset(format!(
691 "dataset '{}' archive size mismatch: expected {}, got {total}",
692 entry.id, entry.archive.size_bytes
693 )));
694 }
695 let actual = format!("{:x}", digest.finalize());
696 if actual != entry.archive.sha256.to_ascii_lowercase() {
697 return Err(Error::Dataset(format!(
698 "dataset '{}' SHA-256 mismatch: expected {}, got {actual}",
699 entry.id, entry.archive.sha256
700 )));
701 }
702 Ok(())
703}
704
705fn extract_archive(archive_path: &Path, destination: &Path) -> Result<()> {
706 let decoder = zstd::Decoder::new(File::open(archive_path)?)?;
707 let mut archive = tar::Archive::new(decoder);
708 let mut paths = BTreeSet::new();
709 let mut remaining_entries = MAX_ARCHIVE_ENTRIES;
710 let mut remaining_bytes = MAX_EXTRACTED_BYTES;
711 for entry in archive.entries()? {
712 let mut entry = entry?;
713 remaining_entries = remaining_entries.checked_sub(1).ok_or_else(|| {
714 Error::Dataset("dataset archive exceeds the extraction entry limit".into())
715 })?;
716 let relative = entry.path()?.into_owned();
717 validate_archive_path(&relative)?;
718 if !paths.insert(relative.clone()) {
719 return Err(Error::Dataset(format!(
720 "dataset archive contains duplicate path {}",
721 relative.display()
722 )));
723 }
724 let entry_type = entry.header().entry_type();
725 let target = destination.join(&relative);
726 if entry_type.is_dir() {
727 fs::create_dir_all(target)?;
728 continue;
729 }
730 if !entry_type.is_file() {
731 return Err(Error::Dataset(format!(
732 "dataset archive entry is not a regular file or directory: {}",
733 relative.display()
734 )));
735 }
736 let size = entry.size();
737 remaining_bytes = remaining_bytes.checked_sub(size).ok_or_else(|| {
738 Error::Dataset("dataset archive exceeds the extraction byte limit".into())
739 })?;
740 if let Some(parent) = target.parent() {
741 fs::create_dir_all(parent)?;
742 }
743 let mut output = OpenOptions::new()
744 .write(true)
745 .create_new(true)
746 .open(&target)?;
747 let copied = std::io::copy(&mut entry, &mut output)?;
748 if copied != size {
749 return Err(Error::Dataset(format!(
750 "dataset archive entry size mismatch for {}",
751 relative.display()
752 )));
753 }
754 }
755 Ok(())
756}
757
758fn read_source_limited(source: &str, limit: u64) -> Result<Vec<u8>> {
759 let mut reader = open_source_reader(source)?;
760 let mut bytes = Vec::new();
761 let copied = std::io::copy(&mut reader.by_ref().take(limit + 1), &mut bytes)?;
762 if copied > limit {
763 return Err(Error::Dataset(format!(
764 "source exceeds the {limit}-byte limit: {source}"
765 )));
766 }
767 Ok(bytes)
768}
769
770fn open_source_reader(source: &str) -> Result<Box<dyn Read>> {
771 if source.starts_with("http://") || source.starts_with("https://") {
772 let response = ureq::get(source)
773 .call()
774 .map_err(|error| Error::Dataset(format!("request failed for {source}: {error}")))?;
775 return Ok(Box::new(response.into_body().into_reader()));
776 }
777 let path = source.strip_prefix("file://").unwrap_or(source);
778 Ok(Box::new(File::open(path).map_err(|error| {
779 Error::Dataset(format!("failed to open source {source}: {error}"))
780 })?))
781}
782
783fn validate_archive_path(path: &Path) -> Result<()> {
784 if path.as_os_str().is_empty()
785 || path.is_absolute()
786 || path
787 .components()
788 .any(|component| !matches!(component, Component::Normal(_)))
789 {
790 return Err(Error::Dataset(format!(
791 "dataset archive entry path is unsafe: {}",
792 path.display()
793 )));
794 }
795 Ok(())
796}
797
798fn validate_component(label: &str, value: &str) -> Result<()> {
799 if value.is_empty()
800 || value == "."
801 || value == ".."
802 || !value
803 .bytes()
804 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
805 {
806 return Err(Error::Dataset(format!(
807 "{label} must contain only letters, digits, '.', '_', or '-'"
808 )));
809 }
810 Ok(())
811}
812
813fn validate_nonempty(label: &str, value: &str) -> Result<()> {
814 if value.trim().is_empty() {
815 return Err(Error::Dataset(format!("{label} must not be empty")));
816 }
817 Ok(())
818}
819
820fn validate_marker(path: &Path) -> Result<()> {
821 let metadata = fs::symlink_metadata(path).map_err(|error| {
822 Error::Dataset(format!(
823 "dataset cache ownership marker is missing at {}: {error}",
824 path.display()
825 ))
826 })?;
827 if metadata.file_type().is_symlink() || !metadata.is_file() {
828 return Err(Error::Dataset(format!(
829 "dataset cache ownership marker must be a regular file: {}",
830 path.display()
831 )));
832 }
833 if fs::read(path)? != CACHE_MARKER_CONTENTS {
834 return Err(Error::Dataset(format!(
835 "dataset cache ownership marker is invalid: {}",
836 path.display()
837 )));
838 }
839 Ok(())
840}
841
842fn reject_symlinks(path: &Path) -> Result<()> {
843 let metadata = fs::symlink_metadata(path)?;
844 if metadata.file_type().is_symlink() {
845 return Err(Error::Dataset(format!(
846 "managed cache path contains a symlink: {}",
847 path.display()
848 )));
849 }
850 if metadata.is_dir() {
851 for entry in fs::read_dir(path)? {
852 reject_symlinks(&entry?.path())?;
853 }
854 }
855 Ok(())
856}
857
858fn count_cached_versions(root: &Path) -> Result<usize> {
859 if !root.is_dir() {
860 return Ok(0);
861 }
862 let mut count = 0;
863 for dataset in fs::read_dir(root)? {
864 let dataset = dataset?;
865 if dataset.file_type()?.is_dir() {
866 count += fs::read_dir(dataset.path())?
867 .filter_map(std::result::Result::ok)
868 .filter_map(|entry| entry.file_type().ok())
869 .filter(|kind| kind.is_dir())
870 .count();
871 }
872 }
873 Ok(count)
874}
875
876fn unique_nonce() -> String {
877 let nanos = SystemTime::now()
878 .duration_since(UNIX_EPOCH)
879 .map_or(0, |duration| duration.as_nanos());
880 format!("{}-{nanos}", std::process::id())
881}
882
883fn replace_file(source: &Path, destination: &Path) -> Result<()> {
884 match fs::rename(source, destination) {
885 Ok(()) => Ok(()),
886 Err(error)
887 if destination.exists()
888 && matches!(
889 error.kind(),
890 std::io::ErrorKind::AlreadyExists | std::io::ErrorKind::PermissionDenied
891 ) =>
892 {
893 fs::remove_file(destination)?;
894 fs::rename(source, destination)?;
895 Ok(())
896 }
897 Err(error) => Err(error.into()),
898 }
899}
900
901#[cfg(test)]
902mod tests {
903 use super::*;
904
905 #[test]
906 fn archive_paths_reject_escape_and_non_normal_components() {
907 for path in [
908 "",
909 ".",
910 "./dataset.json",
911 "../dataset.json",
912 "/dataset.json",
913 ] {
914 assert!(
915 validate_archive_path(Path::new(path)).is_err(),
916 "accepted {path:?}"
917 );
918 }
919 assert!(validate_archive_path(Path::new("frames/frame.tiff")).is_ok());
920 }
921
922 #[test]
923 fn archive_extraction_rejects_links_and_duplicate_paths() -> Result<()> {
924 let temporary = tempfile::tempdir()?;
925 let link_archive = temporary.path().join("link.tar.zst");
926 {
927 let encoder = zstd::Encoder::new(File::create(&link_archive)?, 1)?;
928 let mut archive = tar::Builder::new(encoder);
929 let mut header = tar::Header::new_gnu();
930 header.set_entry_type(tar::EntryType::Symlink);
931 header.set_size(0);
932 header.set_mode(0o777);
933 header.set_link_name("outside")?;
934 header.set_cksum();
935 archive.append_data(&mut header, "link", std::io::empty())?;
936 archive.into_inner()?.finish()?;
937 }
938 let destination = temporary.path().join("link-output");
939 fs::create_dir(&destination)?;
940 assert!(
941 extract_archive(&link_archive, &destination)
942 .unwrap_err()
943 .to_string()
944 .contains("not a regular file or directory")
945 );
946
947 let duplicate_archive = temporary.path().join("duplicate.tar.zst");
948 {
949 let encoder = zstd::Encoder::new(File::create(&duplicate_archive)?, 1)?;
950 let mut archive = tar::Builder::new(encoder);
951 for payload in [b"first".as_slice(), b"second".as_slice()] {
952 let mut header = tar::Header::new_gnu();
953 header.set_size(payload.len() as u64);
954 header.set_mode(0o644);
955 header.set_cksum();
956 archive.append_data(&mut header, "duplicate", payload)?;
957 }
958 archive.into_inner()?.finish()?;
959 }
960 let destination = temporary.path().join("duplicate-output");
961 fs::create_dir(&destination)?;
962 assert!(
963 extract_archive(&duplicate_archive, &destination)
964 .unwrap_err()
965 .to_string()
966 .contains("duplicate path")
967 );
968 Ok(())
969 }
970}