changes claude never committed
This commit is contained in:
@@ -0,0 +1,325 @@
|
||||
//! The `ci` binary: runs the full adversarial framework against the reference
|
||||
//! and runtime-under-test, writes the eight required reports (JSON + markdown),
|
||||
//! and exits nonzero if any gate fails.
|
||||
|
||||
use ci_reports::json::Json;
|
||||
use ci_reports::{run_all, CiResults, Scale};
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
fn arr_f(vals: &[f64]) -> Json {
|
||||
Json::Arr(vals.iter().map(|&v| Json::Num(v)).collect())
|
||||
}
|
||||
fn fails(v: &[String]) -> Json {
|
||||
Json::Arr(v.iter().map(|s| Json::s(s.clone())).collect())
|
||||
}
|
||||
fn pass_field(v: &[String]) -> Json {
|
||||
Json::Bool(v.is_empty())
|
||||
}
|
||||
|
||||
fn write_report(dir: &Path, name: &str, j: &Json) {
|
||||
let path = dir.join(format!("{name}.json"));
|
||||
let mut f = fs::File::create(&path).expect("create report");
|
||||
f.write_all(j.to_pretty().as_bytes()).expect("write report");
|
||||
}
|
||||
|
||||
fn build_reports(dir: &Path, r: &CiResults) {
|
||||
// 1. domain_participation_report
|
||||
write_report(
|
||||
dir,
|
||||
"domain_participation_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.domain.failures)),
|
||||
("appears_fraction".into(), arr_f(&r.domain.appears)),
|
||||
("influences_fraction".into(), arr_f(&r.domain.influences)),
|
||||
("mutated_fraction".into(), arr_f(&r.domain.mutated)),
|
||||
("removal_diversity_loss".into(), arr_f(&r.domain.removal_loss)),
|
||||
("min_merge_loss".into(), Json::Num(r.domain.min_merge_loss)),
|
||||
(
|
||||
"read_only_domains".into(),
|
||||
Json::Arr(r.domain.read_only.iter().map(|&d| Json::Int(d as i64)).collect()),
|
||||
),
|
||||
(
|
||||
"write_only_domains".into(),
|
||||
Json::Arr(r.domain.write_only.iter().map(|&d| Json::Int(d as i64)).collect()),
|
||||
),
|
||||
("failures".into(), fails(&r.domain.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 2. causal_rank_report (trace gates)
|
||||
write_report(
|
||||
dir,
|
||||
"causal_rank_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.trace.failures)),
|
||||
("median_causal_rank".into(), Json::Num(r.trace.median_rank)),
|
||||
("p95_causal_rank".into(), Json::Num(r.trace.p95_causal_rank)),
|
||||
("median_causal_edges".into(), Json::Num(r.trace.median_causal_edges)),
|
||||
("median_touched_domains".into(), Json::Num(r.trace.median_touched)),
|
||||
("p95_touched_domains".into(), Json::Num(r.trace.p95_touched)),
|
||||
("fp_collision_rate".into(), Json::Num(r.trace.fp_collision_rate)),
|
||||
("largest_cluster".into(), Json::Num(r.trace.largest_cluster)),
|
||||
("failures".into(), fails(&r.trace.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 3. compression_resistance_report
|
||||
let attack_json: Vec<Json> = r
|
||||
.collapse
|
||||
.reports
|
||||
.iter()
|
||||
.map(|rep| {
|
||||
Json::Obj(vec![
|
||||
("attack".into(), Json::s(rep.attack.clone())),
|
||||
("predicts".into(), Json::Num(rep.predicts)),
|
||||
("info_loss".into(), Json::Num(rep.info_loss)),
|
||||
("detail".into(), Json::s(rep.detail.clone())),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
write_report(
|
||||
dir,
|
||||
"compression_resistance_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.collapse.failures)),
|
||||
("best_1factor".into(), Json::Num(r.collapse.best_1factor)),
|
||||
("best_2factor".into(), Json::Num(r.collapse.best_2factor)),
|
||||
("best_4factor".into(), Json::Num(r.collapse.best_4factor)),
|
||||
("max_single_domain".into(), Json::Num(r.collapse.max_single_domain)),
|
||||
("max_pair".into(), Json::Num(r.collapse.max_pair)),
|
||||
("min_info_loss".into(), Json::Num(r.collapse.min_info_loss)),
|
||||
("attacks".into(), Json::Arr(attack_json)),
|
||||
("failures".into(), fails(&r.collapse.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 4. metamorphic_response_report
|
||||
write_report(
|
||||
dir,
|
||||
"metamorphic_response_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.metamorphic.failures)),
|
||||
("perturbations".into(), Json::Int(r.metamorphic.total as i64)),
|
||||
("altered_trace".into(), Json::Num(r.metamorphic.altered_trace)),
|
||||
("altered_delta".into(), Json::Num(r.metamorphic.altered_delta)),
|
||||
("altered_future".into(), Json::Num(r.metamorphic.altered_future)),
|
||||
("neutral_unexplained".into(), Json::Num(r.metamorphic.neutral_unexplained)),
|
||||
("failures".into(), fails(&r.metamorphic.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 5. mutation_survivor_report
|
||||
let survivors: Vec<Json> = r
|
||||
.mutation
|
||||
.survivors
|
||||
.iter()
|
||||
.map(|(id, name)| {
|
||||
Json::Obj(vec![
|
||||
("id".into(), Json::Int(*id as i64)),
|
||||
("name".into(), Json::s(name.clone())),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
write_report(
|
||||
dir,
|
||||
"mutation_survivor_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), Json::Bool(r.mutation.passed())),
|
||||
("total_mutants".into(), Json::Int(r.mutation.total as i64)),
|
||||
("killed".into(), Json::Int(r.mutation.killed as i64)),
|
||||
("survivors".into(), Json::Arr(survivors)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 6. runtime_equivalence_report
|
||||
write_report(
|
||||
dir,
|
||||
"runtime_equivalence_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.equivalence.failures)),
|
||||
("total".into(), Json::Int(r.equivalence.total as i64)),
|
||||
("matched".into(), Json::Int(r.equivalence.matched as i64)),
|
||||
("failures".into(), fails(&r.equivalence.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 7. replay_report
|
||||
write_report(
|
||||
dir,
|
||||
"replay_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.replay.failures)),
|
||||
("total".into(), Json::Int(r.replay.total as i64)),
|
||||
("deterministic".into(), Json::Int(r.replay.deterministic as i64)),
|
||||
("drift".into(), Json::Int(r.replay.drift as i64)),
|
||||
("failures".into(), fails(&r.replay.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 8. coverage_report
|
||||
write_report(
|
||||
dir,
|
||||
"coverage_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.coverage.failures)),
|
||||
("generated_cases".into(), Json::Int(r.coverage.generated_cases as i64)),
|
||||
("generated_rejected".into(), Json::Int(r.coverage.generated_rejected as i64)),
|
||||
("contract_rejected".into(), Json::Int(r.coverage.contract_rejected as i64)),
|
||||
("executions".into(), Json::Int(r.coverage.executions as i64)),
|
||||
("perturbations".into(), Json::Int(r.coverage.perturbations as i64)),
|
||||
("contracts_passed".into(), Json::Int(r.contract.passed as i64)),
|
||||
("contracts_total".into(), Json::Int(r.contract.total as i64)),
|
||||
("failures".into(), fails(&r.coverage.failures)),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
fn status(v: bool) -> &'static str {
|
||||
if v {
|
||||
"PASS"
|
||||
} else {
|
||||
"FAIL"
|
||||
}
|
||||
}
|
||||
|
||||
fn write_markdown(dir: &Path, r: &CiResults) {
|
||||
let mut s = String::new();
|
||||
s.push_str("# Magicka VM — Phase 0/1 CI Report\n\n");
|
||||
s.push_str(&format!(
|
||||
"Overall: **{}**\n\n",
|
||||
status(r.passed())
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"Scale: executions={}, mutants={}, replay={}, collapse_samples={}\n\n",
|
||||
r.scale.executions, r.scale.mutants, r.scale.replay_cases, r.scale.collapse_samples
|
||||
));
|
||||
s.push_str("| Report | Status | Key metrics |\n|---|---|---|\n");
|
||||
s.push_str(&format!(
|
||||
"| runtime_equivalence | {} | {}/{} matched |\n",
|
||||
status(r.equivalence.failures.is_empty()),
|
||||
r.equivalence.matched,
|
||||
r.equivalence.total
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| causal_rank/trace | {} | rank med={} p95={}, edges med={}, touched med={} |\n",
|
||||
status(r.trace.failures.is_empty()),
|
||||
r.trace.median_rank,
|
||||
r.trace.p95_causal_rank,
|
||||
r.trace.median_causal_edges,
|
||||
r.trace.median_touched
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| domain_participation | {} | min_merge_loss={:.3} |\n",
|
||||
status(r.domain.failures.is_empty()),
|
||||
r.domain.min_merge_loss
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| metamorphic_response | {} | trace={:.3} delta={:.3} future={:.3} |\n",
|
||||
status(r.metamorphic.failures.is_empty()),
|
||||
r.metamorphic.altered_trace,
|
||||
r.metamorphic.altered_delta,
|
||||
r.metamorphic.altered_future
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| compression_resistance | {} | 1f={:.3} 2f={:.3} 4f={:.3} single={:.3} pair={:.3} info_loss={:.3} |\n",
|
||||
status(r.collapse.failures.is_empty()),
|
||||
r.collapse.best_1factor,
|
||||
r.collapse.best_2factor,
|
||||
r.collapse.best_4factor,
|
||||
r.collapse.max_single_domain,
|
||||
r.collapse.max_pair,
|
||||
r.collapse.min_info_loss
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| mutation_survivor | {} | killed {}/{} |\n",
|
||||
status(r.mutation.passed()),
|
||||
r.mutation.killed,
|
||||
r.mutation.total
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| contract | {} | {}/{} cases |\n",
|
||||
status(r.contract.failures.is_empty()),
|
||||
r.contract.passed,
|
||||
r.contract.total
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| replay | {} | {}/{} deterministic, drift={} |\n",
|
||||
status(r.replay.failures.is_empty()),
|
||||
r.replay.deterministic,
|
||||
r.replay.total,
|
||||
r.replay.drift
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| coverage | {} | exec={}, perturb={}, rejected={} |\n",
|
||||
status(r.coverage.failures.is_empty()),
|
||||
r.coverage.executions,
|
||||
r.coverage.perturbations,
|
||||
r.coverage.generated_rejected
|
||||
));
|
||||
|
||||
s.push_str("\n## Failures\n\n");
|
||||
let mut any = false;
|
||||
for (name, f) in r.all_failures() {
|
||||
for msg in f {
|
||||
any = true;
|
||||
s.push_str(&format!("- **{}**: {}\n", name, msg));
|
||||
}
|
||||
}
|
||||
for (id, name) in &r.mutation.survivors {
|
||||
any = true;
|
||||
s.push_str(&format!("- **mutation_survivor**: mutant {} ({}) survived\n", id, name));
|
||||
}
|
||||
if !any {
|
||||
s.push_str("None. The fake universe failed to collapse. ✅\n");
|
||||
}
|
||||
|
||||
let path = dir.join("ci_summary.md");
|
||||
fs::write(path, s).expect("write markdown");
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let scale = Scale::from_env();
|
||||
let out_dir = std::env::var("MAGICKA_OUT").unwrap_or_else(|_| "ci_out".to_string());
|
||||
let dir = Path::new(&out_dir);
|
||||
fs::create_dir_all(dir).expect("create out dir");
|
||||
|
||||
eprintln!(
|
||||
"running CI: executions={} mutants={} replay={} collapse_samples={}",
|
||||
scale.executions, scale.mutants, scale.replay_cases, scale.collapse_samples
|
||||
);
|
||||
let start = Instant::now();
|
||||
let results = run_all(scale);
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
build_reports(dir, &results);
|
||||
write_markdown(dir, &results);
|
||||
|
||||
println!("\n=== Magicka VM CI ({:?}) ===", elapsed);
|
||||
for (name, f) in results.all_failures() {
|
||||
println!(" {:<24} {}", name, status(f.is_empty()));
|
||||
}
|
||||
println!(
|
||||
" {:<24} {} ({} killed / {} mutants{})",
|
||||
"mutation_survivor",
|
||||
status(results.mutation.passed()),
|
||||
results.mutation.killed,
|
||||
results.mutation.total,
|
||||
if results.mutation.survivors.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(", {} survivors", results.mutation.survivors.len())
|
||||
}
|
||||
);
|
||||
println!("reports written to {}/", out_dir);
|
||||
|
||||
if results.passed() {
|
||||
println!("\nOVERALL: PASS — the adversarial framework could not collapse the universe.");
|
||||
std::process::exit(0);
|
||||
} else {
|
||||
println!("\nOVERALL: FAIL");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user