Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/native-ci-helper-migration.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,12 @@ all-targets Clippy with warnings denied before testing the real inventory.
Creusot with Pearlite contracts: `OPEN`, not run for these CI utilities.
Verus: `OPEN`, not run. Executable contract tests are not formal proofs;
framework provenance tiers are retained and no independent witness is claimed.

== Review follow-up

The recon reader uses a typed baseline envelope for `assail` and `pons`.
Descriptive metadata such as the repository's `_comment` array is ignored;
malformed scanner counts remain input errors. The CLI regression loads the
actual committed baseline as well as invalid counts and unavailable reports.
Ledger derivation is split into inventory, taxonomy, assignment, grading and
absence checks with the same claim/input exit codes and report semantics.
19 changes: 15 additions & 4 deletions tools/ci/src/bin/recon-compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,19 @@ struct Args {
#[arg(long)]
no_fail: bool,
}
#[derive(Default, serde::Deserialize)]
struct Baseline {
#[serde(default)]
assail: BTreeMap<String, u64>,
#[serde(default)]
pons: BTreeMap<String, u64>,
}
fn run() -> Result<()> {
let args = Args::parse();
let baseline: BTreeMap<String, BTreeMap<String, u64>> = if args.baseline.exists() {
let baseline: Baseline = if args.baseline.exists() {
serde_json::from_slice(&fs::read(&args.baseline)?)?
} else {
BTreeMap::new()
Baseline::default()
};
let mut text = String::from("## Recon: two independent scans\n\n");
let mut observed = BTreeMap::new();
Expand All @@ -48,8 +55,12 @@ fn run() -> Result<()> {
continue;
}
};
let frozen = baseline.get(name).cloned().unwrap_or_default();
regressions += regression_count(&counts, &frozen);
let frozen = if name == "assail" {
&baseline.assail
} else {
&baseline.pons
};
regressions += regression_count(&counts, frozen);
text.push_str("| category | observed | baseline | verdict |\n|---|---|---|---|\n");
for (category, count) in &counts {
let old = frozen.get(category);
Expand Down
49 changes: 45 additions & 4 deletions tools/ci/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,11 @@ fn regex(pattern: Option<&str>) -> Result<Option<Regex>> {
.map_err(|e| input(format!("invalid selection regex: {e}")))
}

pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result<Value> {
type Inventory = BTreeMap<String, BTreeSet<String>>;
type Assignments<'a> = BTreeMap<(String, String), (&'a Cell, bool)>;
type Coordinates = BTreeSet<(String, String)>;

fn parse_inventory(inventory: &Value) -> Result<Inventory> {
let inventory: BTreeMap<String, BTreeSet<String>> = serde_json::from_value(inventory.clone())?;
if inventory
.iter()
Expand All @@ -100,19 +104,26 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result<V
if inventory.values().map(BTreeSet::len).sum::<usize>() == 0 {
return Err(input("inventory contains no executed tests"));
}
let ledger: Ledger = serde_json::from_value(ledger.clone())?;
Ok(inventory)
}

fn check_taxonomy(ledger: &Ledger) -> Result<()> {
for cell in ledger.entries.iter().chain(&ledger.buckets) {
if !ledger.taxonomy.categories.contains(&cell.category)
|| !ledger.taxonomy.aspects.contains(&cell.aspect)
{
return Err(input("a ledger cell is outside the declared taxonomy"));
}
}
Ok(())
}

fn assign_entries<'a>(ledger: &'a Ledger, inventory: &Inventory) -> Result<Assignments<'a>> {
let mut assignments = BTreeMap::<(String, String), (&Cell, bool)>::new();
for cell in &ledger.entries {
let pattern = regex(cell.pattern.as_deref())?;
let mut matched = 0;
for (binary, names) in &inventory {
for (binary, names) in inventory {
if !cell.bins.contains(binary) {
continue;
}
Expand All @@ -133,6 +144,14 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result<V
)));
}
}
Ok(assignments)
}

fn assign_buckets<'a>(
ledger: &'a Ledger,
inventory: &Inventory,
assignments: &mut Assignments<'a>,
) -> Result<()> {
let buckets: Vec<_> = ledger
.buckets
.iter()
Expand All @@ -144,7 +163,7 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result<V
.map(|d| Regex::new(&d.r#match).map_err(|e| input(e.to_string())))
.collect::<Result<_>>()?;
let mut remainder = 0;
for (binary, names) in &inventory {
for (binary, names) in inventory {
for name in names {
let key = (binary.clone(), name.clone());
if assignments.contains_key(&key) {
Expand All @@ -170,6 +189,13 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result<V
"{remainder} tests exceed the declared remainder ratchet"
)));
}
Ok(())
}

fn grade(
assignments: Assignments<'_>,
root: &Path,
) -> Result<([usize; 3], Coordinates, Vec<Value>)> {
let mut tiers = [0usize; 3];
let mut cells = BTreeSet::new();
let mut entries = Vec::new();
Expand Down Expand Up @@ -200,11 +226,26 @@ pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result<V
cells.insert((cell.category.clone(), cell.aspect.clone()));
entries.push(json!({"test_id":format!("{binary}::{name}"), "category":cell.category, "aspect":cell.aspect, "provenance":tier, "proof_ladder":[], "result":"passed"}));
}
Ok((tiers, cells, entries))
}

fn check_absent(ledger: &Ledger, cells: &Coordinates) -> Result<()> {
for absent in &ledger.declared_absent {
if cells.contains(&(absent.category.clone(), absent.aspect.clone())) {
return Err(claim("a coordinate is both declared absent and covered"));
}
}
Ok(())
}

pub fn derive_report(inventory: &Value, ledger: &Value, root: &Path) -> Result<Value> {
let inventory = parse_inventory(inventory)?;
let ledger: Ledger = serde_json::from_value(ledger.clone())?;
check_taxonomy(&ledger)?;
let mut assignments = assign_entries(&ledger, &inventory)?;
assign_buckets(&ledger, &inventory, &mut assignments)?;
let (tiers, cells, entries) = grade(assignments, root)?;
check_absent(&ledger, &cells)?;
Ok(
json!({"schema_version":1, "suite":ledger.suite.as_deref().unwrap_or("unknown"),
"summary":{"passed":entries.len(), "total":entries.len(), "actually_proven":tiers[0], "provisionally_proven":tiers[1], "unproven":tiers[2], "covered_cells":cells.len(), "cat_aspect_cells":ledger.taxonomy.categories.len()*ledger.taxonomy.aspects.len(), "lattice_covered":cells.len()}, "entries":entries}),
Expand Down
33 changes: 32 additions & 1 deletion tools/ci/tests/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,10 @@ fn recon_cli_distinguishes_regressions_and_unavailable_reports() -> Result<()> {
&json!({"weak_points":[{"category":"A"},{"category":"A"}]}),
)?;
let pons = temp.write("pons.json", &json!({"findings":[]}))?;
let baseline = temp.write("baseline.json", &json!({"assail":{"A":1},"pons":{}}))?;
let baseline = temp.write(
"baseline.json",
&json!({"_comment":["metadata is not a scanner"],"assail":{"A":1},"pons":{}}),
)?;
let run = |advisory: bool| {
let mut command = Command::new(env!("CARGO_BIN_EXE_recon-compare"));
command
Expand Down Expand Up @@ -134,3 +137,31 @@ fn repository_ledger_taxonomy_and_selectors_are_readable() -> Result<()> {
}
Ok(())
}

#[test]
fn recon_cli_accepts_the_actual_repository_baseline() -> Result<()> {
let temp = Temp::new()?;
let assail = temp.write("assail.json", &json!({"weak_points":[]}))?;
let pons = temp.write("pons.json", &json!({"findings":[]}))?;
let baseline: Value = serde_json::from_str(include_str!("../../../recon/baseline.json"))?;
let baseline = temp.write("baseline.json", &baseline)?;
let run = || {
Command::new(env!("CARGO_BIN_EXE_recon-compare"))
.args([
"--baseline",
baseline.to_str().unwrap(),
"--assail",
assail.to_str().unwrap(),
"--pons",
pons.to_str().unwrap(),
])
.output()
};
assert!(run()?.status.success());
temp.write(
"baseline.json",
&json!({"_comment":[], "assail":{"A":"invalid"},"pons":{}}),
)?;
assert_eq!(run()?.status.code(), Some(2));
Ok(())
}
Loading