diff --git a/docs/native-ci-helper-migration.adoc b/docs/native-ci-helper-migration.adoc index ba2534c..70c0a4f 100644 --- a/docs/native-ci-helper-migration.adoc +++ b/docs/native-ci-helper-migration.adoc @@ -37,3 +37,12 @@ all-targets Clippy with warnings denied before testing the real inventory. Creusot with Pearlite contracts: `OPEN`, not run for these CI utilities. Verus: `OPEN`, not run. Executable contract tests are not formal proofs; framework provenance tiers are retained and no independent witness is claimed. + +== Review follow-up + +The recon reader uses a typed baseline envelope for `assail` and `pons`. +Descriptive metadata such as the repository's `_comment` array is ignored; +malformed scanner counts remain input errors. The CLI regression loads the +actual committed baseline as well as invalid counts and unavailable reports. +Ledger derivation is split into inventory, taxonomy, assignment, grading and +absence checks with the same claim/input exit codes and report semantics. diff --git a/tools/ci/src/bin/recon-compare.rs b/tools/ci/src/bin/recon-compare.rs index 91c89b5..082e6fb 100644 --- a/tools/ci/src/bin/recon-compare.rs +++ b/tools/ci/src/bin/recon-compare.rs @@ -17,12 +17,19 @@ struct Args { #[arg(long)] no_fail: bool, } +#[derive(Default, serde::Deserialize)] +struct Baseline { + #[serde(default)] + assail: BTreeMap, + #[serde(default)] + pons: BTreeMap, +} fn run() -> Result<()> { let args = Args::parse(); - let baseline: BTreeMap> = if args.baseline.exists() { + let baseline: Baseline = if args.baseline.exists() { serde_json::from_slice(&fs::read(&args.baseline)?)? } else { - BTreeMap::new() + Baseline::default() }; let mut text = String::from("## Recon: two independent scans\n\n"); let mut observed = BTreeMap::new(); @@ -48,8 +55,12 @@ fn run() -> Result<()> { continue; } }; - let frozen = baseline.get(name).cloned().unwrap_or_default(); - regressions += regression_count(&counts, &frozen); + let frozen = if name == "assail" { + &baseline.assail + } else { + &baseline.pons + }; + regressions += regression_count(&counts, frozen); text.push_str("| category | observed | baseline | verdict |\n|---|---|---|---|\n"); for (category, count) in &counts { let old = frozen.get(category); diff --git a/tools/ci/src/lib.rs b/tools/ci/src/lib.rs index 3562208..1cb68d6 100644 --- a/tools/ci/src/lib.rs +++ b/tools/ci/src/lib.rs @@ -89,7 +89,11 @@ fn regex(pattern: Option<&str>) -> Result> { .map_err(|e| input(format!("invalid selection regex: {e}"))) } -pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result { +type Inventory = BTreeMap>; +type Assignments<'a> = BTreeMap<(String, String), (&'a Cell, bool)>; +type Coordinates = BTreeSet<(String, String)>; + +fn parse_inventory(inventory: &Value) -> Result { let inventory: BTreeMap> = serde_json::from_value(inventory.clone())?; if inventory .iter() @@ -100,7 +104,10 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result() == 0 { return Err(input("inventory contains no executed tests")); } - let ledger: Ledger = serde_json::from_value(ledger.clone())?; + Ok(inventory) +} + +fn check_taxonomy(ledger: &Ledger) -> Result<()> { for cell in ledger.entries.iter().chain(&ledger.buckets) { if !ledger.taxonomy.categories.contains(&cell.category) || !ledger.taxonomy.aspects.contains(&cell.aspect) @@ -108,11 +115,15 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result(ledger: &'a Ledger, inventory: &Inventory) -> Result> { let mut assignments = BTreeMap::<(String, String), (&Cell, bool)>::new(); for cell in &ledger.entries { let pattern = regex(cell.pattern.as_deref())?; let mut matched = 0; - for (binary, names) in &inventory { + for (binary, names) in inventory { if !cell.bins.contains(binary) { continue; } @@ -133,6 +144,14 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result( + ledger: &'a Ledger, + inventory: &Inventory, + assignments: &mut Assignments<'a>, +) -> Result<()> { let buckets: Vec<_> = ledger .buckets .iter() @@ -144,7 +163,7 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result>()?; let mut remainder = 0; - for (binary, names) in &inventory { + for (binary, names) in inventory { for name in names { let key = (binary.clone(), name.clone()); if assignments.contains_key(&key) { @@ -170,6 +189,13 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result, + root: &Path, +) -> Result<([usize; 3], Coordinates, Vec)> { let mut tiers = [0usize; 3]; let mut cells = BTreeSet::new(); let mut entries = Vec::new(); @@ -200,11 +226,26 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result Result<()> { for absent in &ledger.declared_absent { if cells.contains(&(absent.category.clone(), absent.aspect.clone())) { return Err(claim("a coordinate is both declared absent and covered")); } } + Ok(()) +} + +pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result { + let inventory = parse_inventory(inventory)?; + let ledger: Ledger = serde_json::from_value(ledger.clone())?; + check_taxonomy(&ledger)?; + let mut assignments = assign_entries(&ledger, &inventory)?; + assign_buckets(&ledger, &inventory, &mut assignments)?; + let (tiers, cells, entries) = grade(assignments, root)?; + check_absent(&ledger, &cells)?; Ok( json!({"schema_version":1, "suite":ledger.suite.as_deref().unwrap_or("unknown"), "summary":{"passed":entries.len(), "total":entries.len(), "actually_proven":tiers[0], "provisionally_proven":tiers[1], "unproven":tiers[2], "covered_cells":cells.len(), "cat_aspect_cells":ledger.taxonomy.categories.len()*ledger.taxonomy.aspects.len(), "lattice_covered":cells.len()}, "entries":entries}), diff --git a/tools/ci/tests/cli.rs b/tools/ci/tests/cli.rs index d66b286..e775438 100644 --- a/tools/ci/tests/cli.rs +++ b/tools/ci/tests/cli.rs @@ -80,7 +80,10 @@ fn recon_cli_distinguishes_regressions_and_unavailable_reports() -> Result<()> { &json!({"weak_points":[{"category":"A"},{"category":"A"}]}), )?; let pons = temp.write("pons.json", &json!({"findings":[]}))?; - let baseline = temp.write("baseline.json", &json!({"assail":{"A":1},"pons":{}}))?; + let baseline = temp.write( + "baseline.json", + &json!({"_comment":["metadata is not a scanner"],"assail":{"A":1},"pons":{}}), + )?; let run = |advisory: bool| { let mut command = Command::new(env!("CARGO_BIN_EXE_recon-compare")); command @@ -134,3 +137,31 @@ fn repository_ledger_taxonomy_and_selectors_are_readable() -> Result<()> { } Ok(()) } + +#[test] +fn recon_cli_accepts_the_actual_repository_baseline() -> Result<()> { + let temp = Temp::new()?; + let assail = temp.write("assail.json", &json!({"weak_points":[]}))?; + let pons = temp.write("pons.json", &json!({"findings":[]}))?; + let baseline: Value = serde_json::from_str(include_str!("../../../recon/baseline.json"))?; + let baseline = temp.write("baseline.json", &baseline)?; + let run = || { + Command::new(env!("CARGO_BIN_EXE_recon-compare")) + .args([ + "--baseline", + baseline.to_str().unwrap(), + "--assail", + assail.to_str().unwrap(), + "--pons", + pons.to_str().unwrap(), + ]) + .output() + }; + assert!(run()?.status.success()); + temp.write( + "baseline.json", + &json!({"_comment":[], "assail":{"A":"invalid"},"pons":{}}), + )?; + assert_eq!(run()?.status.code(), Some(2)); + Ok(()) +}