aboutsummaryrefslogtreecommitdiff
path: root/crates/fparkan-corpus/src/lib.rs
diff options
context:
space:
mode:
authorValentin Popov <valentin@popov.link>2026-09-06 04:22:12 +0300
committerValentin Popov <valentin@popov.link>2026-09-06 04:22:12 +0300
commitaa51f3574d3d4d4e0fa59653eedd718fceb5d1bd (patch)
tree9a43f6a3f3c86767e4e2b7316051562d96789f06 /crates/fparkan-corpus/src/lib.rs
parentda03e77f53bb33b2fe203ac45578289953a1b524 (diff)
downloadfparkan-aa51f3574d3d4d4e0fa59653eedd718fceb5d1bd.tar.xz
fparkan-aa51f3574d3d4d4e0fa59653eedd718fceb5d1bd.zip
chore: simplify project to engine docs and tests
Remove planning and acceptance scaffolding while retaining the native Vulkan mission preview, format readers, runtime algorithms, and ordinary Rust tests. Keep the book aligned with the runnable project and validate checked-in shaders without generated tool metadata.
Diffstat (limited to 'crates/fparkan-corpus/src/lib.rs')
-rw-r--r--crates/fparkan-corpus/src/lib.rs1239
1 files changed, 0 insertions, 1239 deletions
diff --git a/crates/fparkan-corpus/src/lib.rs b/crates/fparkan-corpus/src/lib.rs
deleted file mode 100644
index 48eaf9e..0000000
--- a/crates/fparkan-corpus/src/lib.rs
+++ /dev/null
@@ -1,1239 +0,0 @@
-#![forbid(unsafe_code)]
-#![cfg_attr(
- test,
- allow(
- clippy::cast_possible_truncation,
- clippy::cast_possible_wrap,
- clippy::cast_precision_loss,
- clippy::expect_used,
- clippy::float_cmp,
- clippy::identity_op,
- clippy::too_many_lines,
- clippy::uninlined_format_args,
- clippy::map_unwrap_or,
- clippy::needless_raw_string_hashes,
- clippy::semicolon_if_nothing_returned,
- clippy::type_complexity,
- clippy::panic,
- clippy::unwrap_used
- )
-)]
-//! Licensed corpus discovery and aggregate reports.
-
-use fparkan_binary::{sha256, sha256_hex, Sha256Digest};
-use fparkan_fx::{decode_fxid, FXID_KIND};
-use fparkan_material::{decode_mat0, decode_wear, MAT0_KIND, WEAR_KIND};
-use fparkan_mission_format::{decode_tma, TmaProfile};
-use fparkan_msh::{decode_msh, validate_msh};
-use fparkan_nres::NresDocument;
-use fparkan_path::{ascii_lookup_key, normalize_relative, PathPolicy};
-use fparkan_prototype::{decode_unit_dat, decode_unit_dat_binding};
-use fparkan_rsli::{decode as decode_rsli, ReadProfile};
-use fparkan_terrain_format::{decode_land_map, decode_land_msh};
-use fparkan_texm::decode_texm;
-use std::collections::{BTreeMap, BTreeSet};
-use std::fmt;
-use std::fs;
-use std::io::Write;
-#[cfg(unix)]
-use std::os::unix::ffi::OsStrExt;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-const TEXM_KIND: u32 = 0x6d78_6554;
-
-/// Corpus kind.
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum CorpusKind {
- /// Demo corpus.
- Demo,
- /// Part 1 full game.
- Part1,
- /// Part 2 full game.
- Part2,
- /// Unknown local directory.
- Unknown,
-}
-
-/// Corpus root.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct CorpusRoot(pub PathBuf);
-
-/// Discovery options.
-#[derive(Clone, Copy, Debug, Default)]
-pub struct DiscoverOptions {
- /// Whether symlinks may be traversed.
- pub follow_symlinks: bool,
-}
-
-/// File manifest entry.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct ManifestEntry {
- /// Normalized relative path.
- pub path: String,
- /// Byte-exact relative host path used for reopening corpus files.
- pub host_rel_path: PathBuf,
- /// File size in bytes.
- pub size: u64,
- /// SHA-256 content fingerprint.
- pub hash: Sha256Digest,
-}
-
-/// Corpus manifest.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct CorpusManifest {
- /// Kind.
- pub kind: CorpusKind,
- /// Sorted files.
- pub files: Vec<ManifestEntry>,
- /// Casefold collisions.
- pub casefold_collisions: Vec<Vec<String>>,
-}
-
-/// Aggregate report.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct CorpusReport {
- /// Schema version.
- pub schema: u32,
- /// Kind.
- pub kind: CorpusKind,
- /// Total files.
- pub files: usize,
- /// Total bytes.
- pub bytes: u64,
- /// Metrics.
- pub metrics: BTreeMap<String, u64>,
- /// Casefold collision count.
- pub casefold_collisions: usize,
- /// Manifest fingerprint.
- pub fingerprint: Sha256Digest,
- /// Per-file status records.
- pub records: Vec<CorpusFileRecord>,
- /// Number of files with report errors.
- pub failures: usize,
-}
-
-/// Per-file report status.
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum CorpusFileStatus {
- /// File was inspected successfully.
- Ok,
- /// File was inspected but produced a non-fatal warning.
- Warning,
- /// File could not be inspected.
- Error,
-}
-
-/// Per-file report record.
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct CorpusFileRecord {
- /// Normalized relative path.
- pub path: String,
- /// Inspection status.
- pub status: CorpusFileStatus,
- /// Detected file variant.
- pub variant: String,
- /// Optional status message.
- pub message: Option<String>,
-}
-
-/// Corpus error.
-#[derive(Debug)]
-pub enum CorpusError {
- /// I/O failure.
- Io {
- /// Path where I/O failed.
- path: PathBuf,
- /// Source error.
- source: std::io::Error,
- },
- /// Invalid root.
- InvalidRoot(PathBuf),
- /// Invalid path.
- InvalidPath(String),
- /// Aggregate report failure.
- Report {
- /// Path where reporting failed.
- path: String,
- /// Failure message.
- message: String,
- },
-}
-
-impl fmt::Display for CorpusError {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- match self {
- Self::Io { path, source } => write!(f, "{}: {source}", path.display()),
- Self::InvalidRoot(path) => write!(f, "invalid corpus root: {}", path.display()),
- Self::InvalidPath(path) => write!(f, "invalid corpus path: {path}"),
- Self::Report { path, message } => write!(f, "{path}: {message}"),
- }
- }
-}
-
-impl std::error::Error for CorpusError {
- fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
- match self {
- Self::Io { source, .. } => Some(source),
- Self::InvalidRoot(_) | Self::InvalidPath(_) | Self::Report { .. } => None,
- }
- }
-}
-
-/// Discovers a corpus under a root directory.
-///
-/// # Errors
-///
-/// Returns [`CorpusError`] if the root is invalid, traversal encounters an I/O
-/// error, or a discovered path cannot be represented by the legacy path policy.
-pub fn discover(root: &Path, options: DiscoverOptions) -> Result<CorpusManifest, CorpusError> {
- if !root.is_dir() {
- return Err(CorpusError::InvalidRoot(root.to_path_buf()));
- }
- let mut files = Vec::new();
- walk(root, root, options, &mut files)?;
- files.sort_by(|a, b| a.host_rel_path.cmp(&b.host_rel_path));
-
- let kind = classify(root, &files);
- let casefold_collisions = detect_casefold_collisions(&files);
- Ok(CorpusManifest {
- kind,
- files,
- casefold_collisions,
- })
-}
-
-fn walk(
- root: &Path,
- dir: &Path,
- options: DiscoverOptions,
- out: &mut Vec<ManifestEntry>,
-) -> Result<(), CorpusError> {
- let read_dir = fs::read_dir(dir).map_err(|source| CorpusError::Io {
- path: dir.to_path_buf(),
- source,
- })?;
- let mut entries = Vec::new();
- for entry in read_dir {
- let entry = entry.map_err(|source| CorpusError::Io {
- path: dir.to_path_buf(),
- source,
- })?;
- entries.push(entry.path());
- }
- entries.sort();
- for path in entries {
- if path
- .file_name()
- .and_then(|name| name.to_str())
- .is_some_and(|name| name.starts_with('.'))
- {
- continue;
- }
- let metadata = fs::symlink_metadata(&path).map_err(|source| CorpusError::Io {
- path: path.clone(),
- source,
- })?;
- if metadata.file_type().is_symlink() && !options.follow_symlinks {
- continue;
- }
- if metadata.is_dir() {
- walk(root, &path, options, out)?;
- continue;
- }
- if !metadata.is_file() {
- continue;
- }
- let rel = path
- .strip_prefix(root)
- .map_err(|_| CorpusError::InvalidPath(path.display().to_string()))?;
- #[cfg(unix)]
- let rel_bytes = rel.as_os_str().as_bytes();
- #[cfg(not(unix))]
- let rel_bytes = rel
- .to_str()
- .ok_or_else(|| CorpusError::InvalidPath(path.display().to_string()))?
- .as_bytes();
- let normalized = normalize_relative(rel_bytes, PathPolicy::HostCompatible)
- .map_err(|_| CorpusError::InvalidPath(path.display().to_string()))?;
- let bytes = fs::read(&path).map_err(|source| CorpusError::Io {
- path: path.clone(),
- source,
- })?;
- out.push(ManifestEntry {
- path: normalized.display_lossy().to_string(),
- host_rel_path: rel.to_path_buf(),
- size: metadata.len(),
- hash: sha256(&bytes),
- });
- }
- Ok(())
-}
-
-fn classify(root: &Path, files: &[ManifestEntry]) -> CorpusKind {
- let name = root
- .file_name()
- .and_then(|v| v.to_str())
- .unwrap_or_default()
- .to_ascii_uppercase();
- if name == "IS" {
- CorpusKind::Part1
- } else if name == "IS2" {
- CorpusKind::Part2
- } else if files
- .iter()
- .any(|f| f.path.eq_ignore_ascii_case("iron_3d.exe"))
- {
- CorpusKind::Part1
- } else {
- CorpusKind::Unknown
- }
-}
-
-fn detect_casefold_collisions(files: &[ManifestEntry]) -> Vec<Vec<String>> {
- let mut grouped: BTreeMap<Vec<u8>, BTreeSet<String>> = BTreeMap::new();
- for file in files {
- grouped
- .entry(ascii_lookup_key(path_identity_bytes(&file.host_rel_path)).0)
- .or_default()
- .insert(file.path.clone());
- }
- grouped
- .into_values()
- .filter(|paths| paths.len() > 1)
- .map(|paths| paths.into_iter().collect())
- .collect()
-}
-
-/// Builds aggregate report.
-///
-/// # Errors
-///
-/// Returns [`CorpusError`] when the aggregate report cannot be constructed.
-/// Per-file inspection failures are represented in [`CorpusReport::records`]
-/// and counted in [`CorpusReport::failures`].
-pub fn report(root: &Path, manifest: &CorpusManifest) -> Result<CorpusReport, CorpusError> {
- let mut metrics = empty_report_metrics();
- let mut records = Vec::with_capacity(manifest.files.len());
- let mut failures = 0usize;
-
- for entry in &manifest.files {
- let record = inspect_report_file(root, entry, &mut metrics);
- if record.status == CorpusFileStatus::Error {
- failures = failures.saturating_add(1);
- }
- records.push(record);
- }
-
- Ok(CorpusReport {
- schema: 1,
- kind: manifest.kind,
- files: manifest.files.len(),
- bytes: manifest.files.iter().map(|f| f.size).sum(),
- metrics,
- casefold_collisions: manifest.casefold_collisions.len(),
- fingerprint: fingerprint(manifest),
- records,
- failures,
- })
-}
-
-fn empty_report_metrics() -> BTreeMap<String, u64> {
- let mut metrics = BTreeMap::new();
- metrics.insert("nres_files".to_string(), 0);
- metrics.insert("nres_entries".to_string(), 0);
- metrics.insert("rsli_files".to_string(), 0);
- metrics.insert("tma_files".to_string(), 0);
- metrics.insert("land_msh_files".to_string(), 0);
- metrics.insert("land_map_files".to_string(), 0);
- metrics.insert("unit_dat_files".to_string(), 0);
- metrics.insert("msh_entries".to_string(), 0);
- metrics.insert("mat0_entries".to_string(), 0);
- metrics.insert("texm_entries".to_string(), 0);
- metrics.insert("fxid_entries".to_string(), 0);
- metrics.insert("wear_entries".to_string(), 0);
- metrics
-}
-
-fn inspect_report_file(
- root: &Path,
- entry: &ManifestEntry,
- metrics: &mut BTreeMap<String, u64>,
-) -> CorpusFileRecord {
- let lower = entry.path.to_ascii_lowercase();
- let mut variant = inspect_path_metrics(&lower, metrics);
- let path = root.join(&entry.host_rel_path);
- let bytes = match fs::read(&path) {
- Ok(bytes) => bytes,
- Err(source) => {
- return CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Error,
- variant,
- message: Some(source.to_string()),
- };
- }
- };
- if bytes.starts_with(b"NRes") {
- if variant == "file" {
- variant = "nres".to_string();
- }
- bump(metrics, "nres_files", 1);
- if let Err(message) = inspect_nres_metrics(&bytes, metrics) {
- return CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Error,
- variant,
- message: Some(message),
- };
- }
- if variant == "land_msh" {
- if let Err(message) = inspect_land_metrics(&bytes, false) {
- return CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Error,
- variant,
- message: Some(message),
- };
- }
- }
- if variant == "land_map" {
- if let Err(message) = inspect_land_metrics(&bytes, true) {
- return CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Error,
- variant,
- message: Some(message),
- };
- }
- }
- } else if bytes.starts_with(b"NL") {
- variant = "rsli".to_string();
- bump(metrics, "rsli_files", 1);
- if let Err(message) = inspect_rsli_metrics(&bytes) {
- return CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Error,
- variant,
- message: Some(message),
- };
- }
- } else if lower.ends_with("data.tma") {
- if let Err(message) = inspect_tma_metrics(&bytes) {
- return CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Error,
- variant: "tma".to_string(),
- message: Some(message),
- };
- }
- } else if has_extension(&lower, "dat")
- && (lower.starts_with("units/") || lower.contains("/units/"))
- {
- variant = "unit_dat".to_string();
- if let Err(message) = inspect_unit_dat_metrics(&bytes) {
- return CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Error,
- variant,
- message: Some(message),
- };
- }
- }
- CorpusFileRecord {
- path: entry.path.clone(),
- status: CorpusFileStatus::Ok,
- variant,
- message: None,
- }
-}
-
-fn path_identity_bytes(path: &Path) -> &[u8] {
- #[cfg(unix)]
- {
- path.as_os_str().as_bytes()
- }
- #[cfg(not(unix))]
- {
- path.to_str().unwrap_or_default().as_bytes()
- }
-}
-
-fn inspect_path_metrics(lower: &str, metrics: &mut BTreeMap<String, u64>) -> String {
- let mut variant = "file";
- if lower.ends_with("data.tma") {
- bump(metrics, "tma_files", 1);
- variant = "tma";
- }
- if lower.ends_with("land.msh") {
- bump(metrics, "land_msh_files", 1);
- variant = "land_msh";
- }
- if lower.ends_with("land.map") {
- bump(metrics, "land_map_files", 1);
- variant = "land_map";
- }
- if has_extension(lower, "dat") && (lower.starts_with("units/") || lower.contains("/units/")) {
- bump(metrics, "unit_dat_files", 1);
- variant = "unit_dat";
- }
- variant.to_string()
-}
-
-fn inspect_nres_metrics(bytes: &[u8], metrics: &mut BTreeMap<String, u64>) -> Result<(), String> {
- let document = inspect_nres_document(bytes)?;
- bump(metrics, "nres_entries", document.entries().len() as u64);
- for entry in document.entries() {
- let name = String::from_utf8_lossy(entry.name_bytes()).to_ascii_lowercase();
- if has_extension(&name, "msh") {
- bump(metrics, "msh_entries", 1);
- validate_nres_msh_payload(&document, entry)?;
- }
- match entry.meta().type_id {
- MAT0_KIND => {
- bump(metrics, "mat0_entries", 1);
- validate_nres_mat0_payload(&document, entry)?;
- }
- TEXM_KIND => {
- bump(metrics, "texm_entries", 1);
- validate_nres_texm_payload(&document, entry)?;
- }
- FXID_KIND => {
- bump(metrics, "fxid_entries", 1);
- validate_nres_fxid_payload(&document, entry)?;
- }
- WEAR_KIND => {
- bump(metrics, "wear_entries", 1);
- validate_nres_wear_payload(&document, entry)?;
- }
- _ => {}
- }
- }
- Ok(())
-}
-
-fn validate_nres_msh_payload(
- document: &NresDocument,
- entry: &fparkan_nres::NresEntry,
-) -> Result<(), String> {
- let payload = document
- .payload(entry.id())
- .map_err(|err| err.to_string())?;
- let nested = fparkan_nres::decode(
- Arc::from(payload.to_vec().into_boxed_slice()),
- fparkan_nres::ReadProfile::Compatible,
- )
- .map_err(|err| err.to_string())?;
- let model = decode_msh(&nested).map_err(|err| err.to_string())?;
- validate_msh(&model).map_err(|err| err.to_string())?;
- Ok(())
-}
-
-fn validate_nres_mat0_payload(
- document: &NresDocument,
- entry: &fparkan_nres::NresEntry,
-) -> Result<(), String> {
- let payload = document
- .payload(entry.id())
- .map_err(|err| err.to_string())?;
- decode_mat0(payload, entry.meta().attr2).map_err(|err| err.to_string())?;
- Ok(())
-}
-
-fn validate_nres_wear_payload(
- document: &NresDocument,
- entry: &fparkan_nres::NresEntry,
-) -> Result<(), String> {
- let payload = document
- .payload(entry.id())
- .map_err(|err| err.to_string())?;
- decode_wear(payload).map_err(|err| err.to_string())?;
- Ok(())
-}
-
-fn validate_nres_texm_payload(
- document: &NresDocument,
- entry: &fparkan_nres::NresEntry,
-) -> Result<(), String> {
- let payload = document
- .payload(entry.id())
- .map_err(|err| err.to_string())?;
- decode_texm(Arc::from(payload.to_vec().into_boxed_slice())).map_err(|err| err.to_string())?;
- Ok(())
-}
-
-fn validate_nres_fxid_payload(
- document: &NresDocument,
- entry: &fparkan_nres::NresEntry,
-) -> Result<(), String> {
- let payload = document
- .payload(entry.id())
- .map_err(|err| err.to_string())?;
- decode_fxid(Arc::from(payload.to_vec().into_boxed_slice())).map_err(|err| err.to_string())?;
- Ok(())
-}
-
-fn inspect_rsli_metrics(bytes: &[u8]) -> Result<(), String> {
- let _ = decode_rsli(
- Arc::from(bytes.to_vec().into_boxed_slice()),
- ReadProfile::Compatible,
- )
- .map_err(|err| err.to_string())?;
- Ok(())
-}
-
-fn inspect_tma_metrics(bytes: &[u8]) -> Result<(), String> {
- let _ = decode_tma(
- Arc::from(bytes.to_vec().into_boxed_slice()),
- TmaProfile::Strict,
- )
- .map_err(|err| err.to_string())?;
- Ok(())
-}
-
-fn inspect_unit_dat_metrics(bytes: &[u8]) -> Result<(), String> {
- if decode_unit_dat(bytes).is_err() && decode_unit_dat_binding(bytes).is_err() {
- return Err("failed to parse unit.dat payload as unit or binding format".to_string());
- }
- Ok(())
-}
-
-fn inspect_land_metrics(bytes: &[u8], is_map: bool) -> Result<(), String> {
- let document = inspect_nres_document(bytes)?;
- if is_map {
- decode_land_map(&document).map_err(|err| err.to_string())?;
- } else {
- decode_land_msh(&document).map_err(|err| err.to_string())?;
- }
- Ok(())
-}
-
-fn inspect_nres_document(bytes: &[u8]) -> Result<NresDocument, String> {
- fparkan_nres::decode(
- Arc::from(bytes.to_vec().into_boxed_slice()),
- fparkan_nres::ReadProfile::Compatible,
- )
- .map_err(|err| err.to_string())
-}
-
-fn bump(metrics: &mut BTreeMap<String, u64>, key: &str, delta: u64) {
- if let Some(value) = metrics.get_mut(key) {
- *value = value.saturating_add(delta);
- }
-}
-
-fn has_extension(path: &str, expected: &str) -> bool {
- Path::new(path)
- .extension()
- .is_some_and(|extension| extension.eq_ignore_ascii_case(expected))
-}
-
-/// Computes stable manifest fingerprint.
-#[must_use]
-pub fn fingerprint(manifest: &CorpusManifest) -> Sha256Digest {
- let mut bytes = Vec::new();
- for file in &manifest.files {
- bytes.extend_from_slice(file.path.as_bytes());
- bytes.push(0);
- bytes.extend_from_slice(&file.size.to_le_bytes());
- bytes.extend_from_slice(&file.hash);
- }
- sha256(&bytes)
-}
-
-/// Writes report atomically.
-///
-/// # Errors
-///
-/// Returns [`CorpusError`] if the parent directory, temporary file, write, or
-/// final rename operation fails.
-pub fn write_report_atomic(path: &Path, report: &CorpusReport) -> Result<(), CorpusError> {
- let tmp = path.with_extension("tmp");
- if let Some(parent) = path.parent() {
- fs::create_dir_all(parent).map_err(|source| CorpusError::Io {
- path: parent.to_path_buf(),
- source,
- })?;
- }
- let mut file = fs::File::create(&tmp).map_err(|source| CorpusError::Io {
- path: tmp.clone(),
- source,
- })?;
- file.write_all(render_report_json(report).as_bytes())
- .map_err(|source| CorpusError::Io {
- path: tmp.clone(),
- source,
- })?;
- file.sync_all().map_err(|source| CorpusError::Io {
- path: tmp.clone(),
- source,
- })?;
- fs::rename(&tmp, path).map_err(|source| CorpusError::Io {
- path: path.to_path_buf(),
- source,
- })?;
- Ok(())
-}
-
-/// Renders report JSON.
-#[must_use]
-pub fn render_report_json(report: &CorpusReport) -> String {
- let mut out = format!(
- "{{\"schema_version\":\"fparkan-corpus-report-v1\",\"schema\":{},\"kind\":\"{:?}\",\"files\":{},\"bytes\":{},\"casefold_collisions\":{},\"fingerprint\":\"{}\",\"failures\":{},\"record_count\":{},\"metrics\":{{",
- report.schema,
- report.kind,
- report.files,
- report.bytes,
- report.casefold_collisions,
- sha256_hex(&report.fingerprint),
- report.failures,
- report.records.len()
- );
- for (idx, (key, value)) in report.metrics.iter().enumerate() {
- if idx > 0 {
- out.push(',');
- }
- out.push('"');
- out.push_str(key);
- out.push_str("\":");
- out.push_str(&value.to_string());
- }
- out.push_str("}}");
- out.push('}');
- out
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use fparkan_path::join_under;
- use std::time::{SystemTime, UNIX_EPOCH};
-
- #[test]
- #[ignore = "requires licensed corpus"]
- fn report_for_testdata_roots() {
- let root = licensed_root("IS");
- let manifest = discover(&root, DiscoverOptions::default()).expect("manifest");
- let report = report(&root, &manifest).expect("report");
- assert!(report.files > 0);
- assert!(report.metrics["nres_files"] > 0);
- }
-
- #[test]
- #[ignore = "requires licensed corpus"]
- fn licensed_part1_manifest_profile_and_counts_match_baseline() {
- let root = testdata_root("IS");
- let manifest = discover(&root, DiscoverOptions::default()).expect("part 1 manifest");
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(manifest.kind, CorpusKind::Part1);
- assert_eq!(report.files, 1_017);
- assert_eq!(report.metrics["nres_files"], 120);
- assert_eq!(report.metrics["rsli_files"], 2);
- assert_eq!(report.metrics["tma_files"], 29);
- assert_eq!(report.metrics["land_msh_files"], 33);
- assert_eq!(report.metrics["land_map_files"], 33);
- assert_eq!(report.metrics["unit_dat_files"], 425);
- }
-
- #[test]
- #[ignore = "requires licensed corpus"]
- fn licensed_part2_manifest_profile_and_counts_match_baseline() {
- let root = testdata_root("IS2");
- let manifest = discover(&root, DiscoverOptions::default()).expect("part 2 manifest");
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(manifest.kind, CorpusKind::Part2);
- assert_eq!(report.files, 1_302);
- assert_eq!(report.metrics["nres_files"], 134);
- assert_eq!(report.metrics["rsli_files"], 2);
- assert_eq!(report.metrics["tma_files"], 31);
- assert_eq!(report.metrics["land_msh_files"], 32);
- assert_eq!(report.metrics["land_map_files"], 32);
- assert_eq!(report.metrics["unit_dat_files"], 676);
- }
-
- #[test]
- #[ignore = "requires licensed corpus"]
- fn licensed_part1_has_no_casefold_relative_path_collisions() {
- let root = testdata_root("IS");
- let manifest = discover(&root, DiscoverOptions::default()).expect("part 1 manifest");
-
- assert!(manifest.casefold_collisions.is_empty());
- }
-
- #[test]
- #[ignore = "requires licensed corpus"]
- fn licensed_part2_has_no_casefold_relative_path_collisions() {
- let root = testdata_root("IS2");
- let manifest = discover(&root, DiscoverOptions::default()).expect("part 2 manifest");
-
- assert!(manifest.casefold_collisions.is_empty());
- }
-
- #[test]
- #[ignore = "requires licensed corpus"]
- fn licensed_part1_paths_stay_under_root() {
- assert_discovered_paths_stay_under_root("IS");
- }
-
- #[test]
- #[ignore = "requires licensed corpus"]
- fn licensed_part2_paths_stay_under_root() {
- assert_discovered_paths_stay_under_root("IS2");
- }
-
- #[test]
- fn report_json_contains_metrics_and_hashes_not_paths_or_payloads() {
- let manifest = CorpusManifest {
- kind: CorpusKind::Part1,
- files: vec![manifest_entry("secret/payload.bin", 4, sha256(b"DATA"))],
- casefold_collisions: Vec::new(),
- };
- let report = report(Path::new("."), &manifest).expect("report");
- let json = render_report_json(&report);
-
- assert!(json.contains("\"schema_version\":\"fparkan-corpus-report-v1\""));
- assert!(json.contains("\"fingerprint\":"));
- assert!(json.contains("\"failures\":1"));
- assert!(json.contains("\"record_count\":1"));
- assert!(json.contains("\"metrics\":"));
- assert!(!json.contains("secret/payload.bin"));
- assert!(!json.contains("DATA"));
- }
-
- #[test]
- fn report_records_missing_manifest_files_as_failures() {
- let root = temp_dir("report-missing");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry("missing.lib", 1, sha256(b"missing"))],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 1);
- assert_eq!(report.records.len(), 1);
- assert_eq!(report.records[0].path, "missing.lib");
- assert_eq!(report.records[0].status, CorpusFileStatus::Error);
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn report_records_malformed_nres_as_failure() {
- let root = temp_dir("report-bad-nres");
- fs::write(root.join("bad.lib"), b"NRes").expect("bad nres");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry("bad.lib", 4, sha256(b"NRes"))],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 1);
- assert_eq!(report.records[0].status, CorpusFileStatus::Error);
- assert_eq!(report.records[0].variant, "nres");
- assert!(report.records[0]
- .message
- .as_deref()
- .is_some_and(|message| message.contains("NRes")));
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn report_uses_production_nres_parser_for_entry_metrics() {
- let root = temp_dir("report-nres");
- let archive = build_nres(&[
- TestNresEntry {
- name: "mesh.msh",
- type_id: 0,
- payload: b"mesh",
- },
- TestNresEntry {
- name: "mat.bin",
- type_id: 0x3054_414D,
- payload: b"mat0",
- },
- TestNresEntry {
- name: "texture.bin",
- type_id: 0x6D78_6554,
- payload: b"texm",
- },
- ]);
- fs::write(root.join("archive.lib"), &archive).expect("archive");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry(
- "archive.lib",
- u64::try_from(archive.len()).expect("archive size"),
- sha256(&archive),
- )],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 1);
- assert_eq!(report.records.len(), 1);
- assert_eq!(report.records[0].status, CorpusFileStatus::Error);
- assert_eq!(report.records[0].variant, "nres");
- assert_eq!(report.metrics["nres_files"], 1);
- assert_eq!(report.metrics["nres_entries"], 3);
- assert_eq!(report.metrics["msh_entries"], 1);
- assert_eq!(report.metrics["mat0_entries"], 0);
- assert_eq!(report.metrics["texm_entries"], 0);
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn report_land_map_paths_use_production_land_parser() {
- let root = temp_dir("report-land-map");
- fs::create_dir_all(root.join("WORLD/MAP")).expect("land map dir");
- fs::write(root.join("WORLD/MAP/land.map"), build_nres(&[])).expect("land map");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry(
- "WORLD/MAP/land.map",
- 16,
- sha256(b"land.map"),
- )],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 1);
- assert_eq!(report.records[0].status, CorpusFileStatus::Error);
- assert_eq!(report.records[0].variant, "land_map");
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn report_land_msh_paths_use_production_land_parser() {
- let root = temp_dir("report-land-msh");
- fs::create_dir_all(root.join("WORLD/MAP")).expect("land msh dir");
- fs::write(root.join("WORLD/MAP/land.msh"), build_nres(&[])).expect("land msh");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry(
- "WORLD/MAP/land.msh",
- 16,
- sha256(b"land.msh"),
- )],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 1);
- assert_eq!(report.records[0].status, CorpusFileStatus::Error);
- assert_eq!(report.records[0].variant, "land_msh");
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn report_tma_paths_use_production_tma_parser() {
- let root = temp_dir("report-tma");
- fs::create_dir_all(root.join("MISSIONS/test")).expect("tma dir");
- fs::write(root.join("MISSIONS/test/data.tma"), b"malformed tma").expect("tma");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry(
- "MISSIONS/test/data.tma",
- 12,
- sha256(b"malformed tma"),
- )],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 1);
- assert_eq!(report.records[0].status, CorpusFileStatus::Error);
- assert_eq!(report.records[0].variant, "tma");
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn report_unit_dat_paths_use_production_unit_parser() {
- let root = temp_dir("report-unit");
- fs::create_dir_all(root.join("units")).expect("unit dir");
- fs::write(root.join("units/unit.dat"), vec![0u8; 120]).expect("unit");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry("units/unit.dat", 120, sha256(&[0u8; 120]))],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 0);
- assert_eq!(report.records[0].status, CorpusFileStatus::Ok);
- assert_eq!(report.records[0].variant, "unit_dat");
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn report_rsli_paths_use_production_rsli_parser() {
- let root = temp_dir("report-rsli");
- fs::write(root.join("patch.nl"), b"NL malformed").expect("rsli");
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry("patch.nl", 12, sha256(b"NL malformed"))],
- casefold_collisions: Vec::new(),
- };
-
- let report = report(&root, &manifest).expect("report");
-
- assert_eq!(report.failures, 1);
- assert_eq!(report.records[0].status, CorpusFileStatus::Error);
- assert_eq!(report.records[0].variant, "rsli");
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn deterministic_traversal_is_creation_order_independent() {
- let first = temp_dir("order-first");
- let second = temp_dir("order-second");
- fs::create_dir_all(first.join("nested")).expect("first nested");
- fs::create_dir_all(second.join("nested")).expect("second nested");
-
- fs::write(first.join("b.bin"), b"b").expect("first b");
- fs::write(first.join("nested").join("a.bin"), b"a").expect("first a");
- fs::write(second.join("nested").join("a.bin"), b"a").expect("second a");
- fs::write(second.join("b.bin"), b"b").expect("second b");
-
- let first_manifest = discover(&first, DiscoverOptions::default()).expect("first manifest");
- let second_manifest =
- discover(&second, DiscoverOptions::default()).expect("second manifest");
-
- assert_eq!(first_manifest.files, second_manifest.files);
- let _ = fs::remove_dir_all(first);
- let _ = fs::remove_dir_all(second);
- }
-
- #[cfg(unix)]
- #[test]
- fn unreadable_directory_produces_error() {
- use std::os::unix::fs::PermissionsExt;
-
- let root = temp_dir("unreadable");
- let child = root.join("locked");
- fs::create_dir_all(&child).expect("locked dir");
- fs::set_permissions(&child, fs::Permissions::from_mode(0o000)).expect("lock dir");
-
- let result = discover(&root, DiscoverOptions::default());
-
- fs::set_permissions(&child, fs::Permissions::from_mode(0o700)).expect("unlock dir");
- let _ = fs::remove_dir_all(root);
- assert!(matches!(result, Err(CorpusError::Io { path, .. }) if path.ends_with("locked")));
- }
-
- #[cfg(unix)]
- #[test]
- fn symlink_loop_is_not_traversed_by_default() {
- use std::os::unix::fs::symlink;
-
- let root = temp_dir("symlink-loop");
- fs::write(root.join("real.bin"), b"real").expect("real file");
- symlink(&root, root.join("loop")).expect("loop symlink");
-
- let manifest = discover(&root, DiscoverOptions::default()).expect("manifest");
-
- assert_eq!(manifest.files.len(), 1);
- assert_eq!(manifest.files[0].path, "real.bin");
- let _ = fs::remove_dir_all(root);
- }
-
- #[test]
- fn casefold_collisions_are_registered() {
- let manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![
- manifest_entry("Textures/Foo.TEX", 1, sha256(b"first")),
- manifest_entry("textures/foo.tex", 1, sha256(b"second")),
- ],
- casefold_collisions: Vec::new(),
- };
-
- let collisions = detect_casefold_collisions(&manifest.files);
-
- assert_eq!(
- collisions,
- vec![vec![
- "Textures/Foo.TEX".to_string(),
- "textures/foo.tex".to_string()
- ]]
- );
- }
-
- #[test]
- fn fingerprint_changes() {
- let mut manifest = CorpusManifest {
- kind: CorpusKind::Unknown,
- files: vec![manifest_entry("a", 1, sha256(b"before"))],
- casefold_collisions: Vec::new(),
- };
- let a = fingerprint(&manifest);
- manifest.files[0].hash = sha256(b"after");
- assert_ne!(a, fingerprint(&manifest));
- }
-
- #[test]
- fn atomic_report_write() {
- let tmp = std::env::temp_dir().join(format!(
- "fparkan-report-{}.json",
- SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .expect("clock")
- .as_nanos()
- ));
- let report = CorpusReport {
- schema: 1,
- kind: CorpusKind::Unknown,
- files: 0,
- bytes: 0,
- metrics: BTreeMap::new(),
- casefold_collisions: 0,
- fingerprint: sha256(b"empty-report"),
- records: Vec::new(),
- failures: 0,
- };
- write_report_atomic(&tmp, &report).expect("write");
- assert!(tmp.is_file());
- let _ = fs::remove_file(tmp);
- }
-
- #[cfg(unix)]
- #[test]
- fn discover_supports_non_utf8_host_paths() {
- use std::ffi::OsString;
- use std::os::unix::ffi::OsStringExt;
-
- let root = temp_dir("non-utf8");
- let file_name = OsString::from_vec(vec![0xFF, b'.', b'b', b'i', b'n']);
- let file_path = root.join(&file_name);
- if let Err(err) = fs::write(&file_path, b"raw") {
- assert_eq!(err.kind(), std::io::ErrorKind::PermissionDenied);
- let _ = fs::remove_dir_all(root);
- return;
- }
-
- let manifest = discover(&root, DiscoverOptions::default()).expect("manifest");
-
- assert_eq!(manifest.files.len(), 1);
- assert_eq!(manifest.files[0].path, "\u{FFFD}.bin");
- assert_eq!(manifest.files[0].host_rel_path, PathBuf::from(&file_name));
- let _ = fs::remove_dir_all(root);
- }
-
- struct TestNresEntry<'a> {
- name: &'a str,
- type_id: u32,
- payload: &'a [u8],
- }
-
- fn build_nres(entries: &[TestNresEntry<'_>]) -> Vec<u8> {
- let mut out = vec![0; 16];
- let mut offsets = Vec::with_capacity(entries.len());
- for entry in entries {
- offsets.push(u32::try_from(out.len()).expect("offset"));
- out.extend_from_slice(entry.payload);
- let padding = (8 - (out.len() % 8)) % 8;
- out.resize(out.len() + padding, 0);
- }
- let mut order: Vec<usize> = (0..entries.len()).collect();
- order.sort_by(|left, right| {
- entries[*left]
- .name
- .as_bytes()
- .cmp(entries[*right].name.as_bytes())
- });
- for (index, entry) in entries.iter().enumerate() {
- push_u32(&mut out, entry.type_id);
- push_u32(&mut out, 0);
- push_u32(&mut out, 0);
- push_u32(
- &mut out,
- u32::try_from(entry.payload.len()).expect("payload size"),
- );
- push_u32(&mut out, 0);
- let mut name = [0; 36];
- let name_bytes = entry.name.as_bytes();
- name[..name_bytes.len()].copy_from_slice(name_bytes);
- out.extend_from_slice(&name);
- push_u32(&mut out, offsets[index]);
- push_u32(&mut out, u32::try_from(order[index]).expect("sort index"));
- }
- out[0..4].copy_from_slice(b"NRes");
- out[4..8].copy_from_slice(&0x100_u32.to_le_bytes());
- out[8..12].copy_from_slice(&u32::try_from(entries.len()).expect("count").to_le_bytes());
- let total_size = u32::try_from(out.len()).expect("total size");
- out[12..16].copy_from_slice(&total_size.to_le_bytes());
- out
- }
-
- fn manifest_entry(path: &str, size: u64, hash: Sha256Digest) -> ManifestEntry {
- ManifestEntry {
- path: path.to_string(),
- host_rel_path: PathBuf::from(path),
- size,
- hash,
- }
- }
-
- fn push_u32(out: &mut Vec<u8>, value: u32) {
- out.extend_from_slice(&value.to_le_bytes());
- }
-
- fn temp_dir(name: &str) -> PathBuf {
- let path = std::env::temp_dir().join(format!(
- "fparkan-corpus-{name}-{}",
- SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .expect("clock")
- .as_nanos()
- ));
- fs::create_dir_all(&path).expect("temp dir");
- path
- }
-
- fn testdata_root(part: &str) -> PathBuf {
- licensed_root(part)
- }
-
- fn licensed_root(part: &str) -> PathBuf {
- let variable = match part {
- "IS" => "FPARKAN_CORPUS_PART1_ROOT",
- "IS2" => "FPARKAN_CORPUS_PART2_ROOT",
- _ => panic!("unknown licensed corpus part: {part}"),
- };
- let root = std::env::var_os(variable)
- .map(PathBuf::from)
- .unwrap_or_else(|| panic!("{variable} is required for licensed corpus tests"));
- assert!(
- root.is_dir(),
- "licensed corpus root is missing: {}",
- root.display()
- );
- root
- }
-
- fn assert_discovered_paths_stay_under_root(part: &str) {
- let root = testdata_root(part);
- let manifest = discover(&root, DiscoverOptions::default()).expect("licensed manifest");
-
- for entry in &manifest.files {
- let normalized = normalize_relative(entry.path.as_bytes(), PathPolicy::HostCompatible)
- .expect("discovered path should re-normalize");
- let joined = join_under(&root, &normalized).expect("discovered path should join");
- assert!(
- joined.starts_with(&root),
- "discovered path escaped root: {}",
- entry.path
- );
- }
- }
-}