//! The `ci` binary: runs the full adversarial framework against the reference //! and the independently-implemented runtime under test, writes the required //! reports (JSON + markdown) including a provenance report binding the numbers //! to executed work, and exits nonzero if any gate fails. use ci_reports::json::Json; use ci_reports::{run_all_to, CiResults, Profile, Scale}; use std::fs; use std::io::Write; use std::path::Path; use std::time::Instant; fn arr_f(vals: &[f64]) -> Json { Json::Arr(vals.iter().map(|&v| Json::Num(v)).collect()) } fn fails(v: &[String]) -> Json { Json::Arr(v.iter().map(|s| Json::s(s.clone())).collect()) } fn pass_field(v: &[String]) -> Json { Json::Bool(v.is_empty()) } fn hex(h: world_model::Hash) -> Json { Json::s(format!("{:016x}", h.0)) } fn write_report(dir: &Path, name: &str, j: &Json) { let path = dir.join(format!("{name}.json")); let mut f = fs::File::create(&path).expect("create report"); f.write_all(j.to_pretty().as_bytes()).expect("write report"); } /// Write the raw evidence the independent `attest` binary verifies (findings /// 2, 3): every Merkle leaf, plus the producer's claims. The attestor recomputes /// the root from these leaves and checks it against the claimed root, in a /// separate process that never reads the compliance report. fn write_evidence(dir: &Path, r: &CiResults) { let ev = dir.join("evidence"); fs::create_dir_all(&ev).expect("create evidence dir"); let mut leaves = String::from("leaf\n"); for h in &r.merkle_leaves { leaves.push_str(&format!("{:016x}\n", h.0)); } fs::write(ev.join("leaves.tsv"), leaves).expect("write leaves"); let claims = format!( "# evidence claims for independent attestation\n\ root\t{:016x}\n\ leaf_count\t{}\n\ total_comparisons\t{}\n\ reference_engine_id\t{:016x}\n\ rut_engine_id\t{:016x}\n\ engines_agree\t{}\n", r.provenance.execution_merkle_root.0, r.provenance.merkle_leaf_count, r.provenance.total_comparisons, r.provenance.reference_engine_id.0, r.provenance.rut_engine_id.0, r.provenance.engines_agree, ); fs::write(ev.join("claims.tsv"), claims).expect("write claims"); // Note: evidence/traces.tsv (the FULL per-execution trace corpus) is streamed // during the run in main(), covering 100% of base executions — not written // here from a capped sample. } fn build_reports(dir: &Path, r: &CiResults) { // 1. domain_participation_report write_report( dir, "domain_participation_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.domain.failures)), ("appears_fraction".into(), arr_f(&r.domain.appears)), ("influences_fraction".into(), arr_f(&r.domain.influences)), ("mutated_fraction".into(), arr_f(&r.domain.mutated)), ("removal_diversity_loss".into(), arr_f(&r.domain.removal_loss)), ("min_merge_loss".into(), Json::Num(r.domain.min_merge_loss)), ( "read_only_domains".into(), Json::Arr(r.domain.read_only.iter().map(|&d| Json::Int(d as i64)).collect()), ), ( "write_only_domains".into(), Json::Arr(r.domain.write_only.iter().map(|&d| Json::Int(d as i64)).collect()), ), ("failures".into(), fails(&r.domain.failures)), ]), ); // 2. causal_rank_report (trace gates) write_report( dir, "causal_rank_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.trace.failures)), ("median_causal_rank".into(), Json::Num(r.trace.median_rank)), ("p95_causal_rank".into(), Json::Num(r.trace.p95_causal_rank)), ("median_causal_edges".into(), Json::Num(r.trace.median_causal_edges)), ("median_touched_domains".into(), Json::Num(r.trace.median_touched)), ("p95_touched_domains".into(), Json::Num(r.trace.p95_touched)), ("fp_collision_rate".into(), Json::Num(r.trace.fp_collision_rate)), ("largest_cluster".into(), Json::Num(r.trace.largest_cluster)), ("failures".into(), fails(&r.trace.failures)), ]), ); // 3. compression_resistance_report let attack_json: Vec = r .collapse .reports .iter() .map(|rep| { Json::Obj(vec![ ("attack".into(), Json::s(rep.attack.clone())), ("reconstructs".into(), Json::Num(rep.predicts)), ("info_loss".into(), Json::Num(rep.info_loss)), ("detail".into(), Json::s(rep.detail.clone())), ]) }) .collect(); write_report( dir, "compression_resistance_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.collapse.failures)), ("measures".into(), Json::s("real serialized trace structure")), ("best_1factor".into(), Json::Num(r.collapse.best_1factor)), ("best_2factor".into(), Json::Num(r.collapse.best_2factor)), ("best_4factor".into(), Json::Num(r.collapse.best_4factor)), ("max_single_domain".into(), Json::Num(r.collapse.max_single_domain)), ("max_pair".into(), Json::Num(r.collapse.max_pair)), ("min_info_loss".into(), Json::Num(r.collapse.min_info_loss)), ("attacks".into(), Json::Arr(attack_json)), ("failures".into(), fails(&r.collapse.failures)), ]), ); // 4. metamorphic_response_report write_report( dir, "metamorphic_response_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.metamorphic.failures)), ("perturbations".into(), Json::Int(r.metamorphic.total as i64)), ("altered_trace".into(), Json::Num(r.metamorphic.altered_trace)), ("altered_delta".into(), Json::Num(r.metamorphic.altered_delta)), ("altered_future".into(), Json::Num(r.metamorphic.altered_future)), ("consumed_perturbation_trace_violations".into(), Json::Num(r.metamorphic.expectation_violations)), ("legitimately_neutral".into(), Json::Num(r.metamorphic.explained_neutral)), ("consumed_perturbations".into(), Json::Int(r.metamorphic.consumed as i64)), ("failures".into(), fails(&r.metamorphic.failures)), ]), ); // 4b. causal_explanation_report (finding 10: intervention-confirmed edges) write_report( dir, "causal_explanation_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.causal_explanation.failures)), ("edges_tested".into(), Json::Int(r.causal_explanation.edges_tested as i64)), ("edges_confirmed".into(), Json::Int(r.causal_explanation.edges_confirmed as i64)), ("confirmed_fraction".into(), Json::Num(r.causal_explanation.confirmed_fraction)), ("method".into(), Json::s("ablate recorded causal source lane; require destination delta to change")), ("failures".into(), fails(&r.causal_explanation.failures)), ]), ); // 5. mutation_survivor_report let survivors: Vec = r .mutation .survivors .iter() .map(|(id, reason)| { Json::Obj(vec![ ("id".into(), Json::Int(*id as i64)), ("reason".into(), Json::s(reason.clone())), ]) }) .collect(); write_report( dir, "mutation_survivor_report", &Json::Obj(vec![ ("pass".into(), Json::Bool(r.mutation.passed())), ("killed_by".into(), Json::s("named acceptance gate")), ("total_mutants".into(), Json::Int(r.mutation.total as i64)), ("killed".into(), Json::Int(r.mutation.killed as i64)), ("survivors".into(), Json::Arr(survivors)), ]), ); // 6. runtime_equivalence_report write_report( dir, "runtime_equivalence_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.equivalence.failures)), ("independent_implementations".into(), Json::Bool(r.equivalence.independent)), ("reference_engine_id".into(), hex(r.provenance.reference_engine_id)), ("rut_engine_id".into(), hex(r.provenance.rut_engine_id)), ("engines_agree".into(), Json::Bool(r.provenance.engines_agree)), ("total".into(), Json::Int(r.equivalence.total as i64)), ("matched".into(), Json::Int(r.equivalence.matched as i64)), ("failures".into(), fails(&r.equivalence.failures)), ]), ); // 7. replay_report write_report( dir, "replay_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.replay.failures)), ("loaded_from_committed_corpus".into(), Json::Bool(r.replay.loaded_from_disk)), ("total".into(), Json::Int(r.replay.total as i64)), ("deterministic".into(), Json::Int(r.replay.deterministic as i64)), ("drift".into(), Json::Int(r.replay.drift as i64)), ("retained_failures_present".into(), Json::Bool(r.replay.retained_present)), ("retained_failures_total".into(), Json::Int(r.replay.retained_total as i64)), ("retained_failures_regressions".into(), Json::Int(r.replay.retained_regressions as i64)), ("failures".into(), fails(&r.replay.failures)), ]), ); // 8. coverage_report write_report( dir, "coverage_report", &Json::Obj(vec![ ("pass".into(), pass_field(&r.coverage.failures)), ("generated_cases".into(), Json::Int(r.coverage.generated_cases as i64)), ("generated_rejected".into(), Json::Int(r.coverage.generated_rejected as i64)), ("contract_rejected".into(), Json::Int(r.coverage.contract_rejected as i64)), ("executions".into(), Json::Int(r.coverage.executions as i64)), ("perturbations".into(), Json::Int(r.coverage.perturbations as i64)), ("contracts_passed".into(), Json::Int(r.contract.passed as i64)), ("contracts_total".into(), Json::Int(r.contract.total as i64)), ("failures".into(), fails(&r.coverage.failures)), ]), ); // 9. provenance_report — binds the run to executed work. let p = &r.provenance; write_report( dir, "provenance_report", &Json::Obj(vec![ ("pass".into(), pass_field(&p.failures)), ("profile".into(), Json::s(p.profile.name())), ( "merge_floor".into(), Json::Obj(vec![ ("worlds".into(), Json::Int(p.floor.worlds as i64)), ("programs".into(), Json::Int(p.floor.programs as i64)), ("executions".into(), Json::Int(p.floor.executions as i64)), ("perturbations_per_exec".into(), Json::Int(p.floor.perturbations_per_exec as i64)), ("mutants".into(), Json::Int(p.floor.mutants as i64)), ("replay_cases".into(), Json::Int(p.floor.replay_cases as i64)), ]), ), ("worlds_generated".into(), Json::Int(p.worlds_generated as i64)), ("programs_generated".into(), Json::Int(p.programs_generated as i64)), ("actual_executions".into(), Json::Int(p.actual_executions as i64)), ("min_perturbations_per_exec".into(), Json::Int(p.min_perturbations_per_exec as i64)), ("total_comparisons".into(), Json::Int(p.total_comparisons as i64)), ("actual_mutants".into(), Json::Int(p.actual_mutants as i64)), ("replay_total".into(), Json::Int(p.replay_total as i64)), ("reference_engine_id".into(), hex(p.reference_engine_id)), ("rut_engine_id".into(), hex(p.rut_engine_id)), ("engines_agree".into(), Json::Bool(p.engines_agree)), ("execution_merkle_root".into(), hex(p.execution_merkle_root)), ("merkle_leaf_count".into(), Json::Int(p.merkle_leaf_count as i64)), ("collapse_feature_width".into(), Json::Int(p.collapse_feature_width as i64)), ("failures".into(), fails(&p.failures)), ]), ); } /// The reports the spec requires every CI run to produce. A missing or empty /// artifact is itself an acceptance failure (compliance rule 4). const REQUIRED_REPORTS: [&str; 10] = [ "domain_participation_report", "causal_rank_report", "causal_explanation_report", "compression_resistance_report", "metamorphic_response_report", "mutation_survivor_report", "runtime_equivalence_report", "replay_report", "coverage_report", "provenance_report", ]; fn report_present(dir: &Path, name: &str) -> bool { fs::metadata(dir.join(format!("{name}.json"))) .map(|m| m.len() > 0) .unwrap_or(false) } /// One acceptance obligation: a measured artifact, a provenance chain, a /// merge-blocking enforcement point, and a failure if the artifact/provenance is /// absent. This is the compliance model made machine-checkable. struct Obligation { requirement: &'static str, artifact: &'static str, floor: i64, actual: i64, gate_pass: bool, } fn obligations(r: &CiResults) -> Vec { let p = &r.provenance; let merge = p.profile == Profile::Merge; let f = |v: usize| v as i64; let floor = |v: usize| if merge { v as i64 } else { 0 }; vec![ Obligation { requirement: "generated worlds >= 50,000", artifact: "provenance_report", floor: floor(p.floor.worlds), actual: f(p.worlds_generated), gate_pass: !merge || p.worlds_generated >= p.floor.worlds, }, Obligation { requirement: "generated programs >= 250,000", artifact: "provenance_report", floor: floor(p.floor.programs), actual: f(p.programs_generated), gate_pass: !merge || p.programs_generated >= p.floor.programs, }, Obligation { requirement: "executions >= 1,000,000", artifact: "provenance_report", floor: floor(p.floor.executions), actual: f(p.actual_executions), gate_pass: !merge || p.actual_executions >= p.floor.executions, }, Obligation { requirement: "perturbations per execution >= 10", artifact: "metamorphic_response_report", floor: floor(p.floor.perturbations_per_exec), actual: f(p.min_perturbations_per_exec), gate_pass: !merge || p.min_perturbations_per_exec >= p.floor.perturbations_per_exec, }, Obligation { requirement: "reference/runtime comparison = 100% of executions", artifact: "runtime_equivalence_report", floor: floor(p.floor.executions * (1 + p.floor.perturbations_per_exec)), actual: f(p.total_comparisons), gate_pass: r.equivalence.failures.is_empty(), }, Obligation { requirement: "semantic mutants >= 500, 0 survivors, killed by named gate", artifact: "mutation_survivor_report", floor: floor(p.floor.mutants), actual: f(r.mutation.total), gate_pass: r.mutation.passed() && (!merge || r.mutation.total >= p.floor.mutants), }, Obligation { requirement: "replay corpus >= 10,000, persisted, 0 drift", artifact: "replay_report", floor: floor(p.floor.replay_cases), actual: f(r.replay.total), gate_pass: r.replay.failures.is_empty() && r.replay.loaded_from_disk, }, Obligation { requirement: "collapse attacks fail to simplify (real trace info)", artifact: "compression_resistance_report", floor: 0, actual: 0, gate_pass: r.collapse.failures.is_empty(), }, Obligation { requirement: "trace gates (causal edges/rank/touched/fingerprints)", artifact: "causal_rank_report", floor: 0, actual: 0, gate_pass: r.trace.failures.is_empty(), }, Obligation { requirement: "domain participation (no decorative/read-only/write-only)", artifact: "domain_participation_report", floor: 0, actual: 0, gate_pass: r.domain.failures.is_empty(), }, Obligation { requirement: "metamorphic response thresholds", artifact: "metamorphic_response_report", floor: 0, actual: 0, gate_pass: r.metamorphic.failures.is_empty(), }, Obligation { requirement: "recorded causal edges are intervention-confirmed (not counted)", artifact: "causal_explanation_report", floor: 0, actual: f(r.causal_explanation.edges_confirmed), gate_pass: r.causal_explanation.failures.is_empty(), }, Obligation { requirement: "every admitted case satisfies its contract", artifact: "coverage_report", floor: 0, actual: 0, gate_pass: r.contract.failures.is_empty() && r.coverage.failures.is_empty(), }, Obligation { requirement: "provenance binds reports to executed work", artifact: "provenance_report", floor: 0, actual: 0, gate_pass: r.provenance.failures.is_empty(), }, ] } /// Write the compliance report and return whether the compliance model holds: /// every obligation's artifact is present and every obligation passes. fn build_compliance_report(dir: &Path, r: &CiResults) -> bool { let items = obligations(r); let mut all_ok = true; let mut json_items = Vec::new(); for ob in &items { let present = report_present(dir, ob.artifact); let pass = present && ob.gate_pass; if !pass { all_ok = false; } json_items.push(Json::Obj(vec![ ("requirement".into(), Json::s(ob.requirement)), ("measured_artifact".into(), Json::s(format!("{}.json", ob.artifact))), ("artifact_present".into(), Json::Bool(present)), ("provenance".into(), Json::s("provenance_report.json (merkle root + engine ids)")), ("merge_blocking".into(), Json::Bool(true)), ("floor".into(), Json::Int(ob.floor)), ("actual".into(), Json::Int(ob.actual)), ("pass".into(), Json::Bool(pass)), ])); } // Every required report must exist and be non-empty. let mut missing = Vec::new(); for rep in REQUIRED_REPORTS { if !report_present(dir, rep) { missing.push(rep.to_string()); all_ok = false; } } write_report( dir, "compliance_report", &Json::Obj(vec![ ("pass".into(), Json::Bool(all_ok)), ("profile".into(), Json::s(r.provenance.profile.name())), ( "merge_blocking_run".into(), Json::Bool(r.provenance.profile == Profile::Merge), ), ( "note".into(), Json::s(if r.provenance.profile == Profile::Merge { "merge profile: floors enforced, all obligations acceptance-blocking" } else { "advisory profile: NOT acceptance; floors not enforced (fast/tiny)" }), ), ("required_reports".into(), Json::Arr(REQUIRED_REPORTS.iter().map(|s| Json::s(*s)).collect())), ("missing_reports".into(), Json::Arr(missing.iter().map(|s| Json::s(s.clone())).collect())), ("obligations".into(), Json::Arr(json_items)), ]), ); all_ok } fn status(v: bool) -> &'static str { if v { "PASS" } else { "FAIL" } } fn write_markdown(dir: &Path, r: &CiResults) { let p = &r.provenance; let mut s = String::new(); s.push_str("# Magicka VM — Phase 0/1 CI Report\n\n"); s.push_str(&format!("Overall: **{}**\n\n", status(r.passed()))); s.push_str(&format!("Profile: **{}**", p.profile.name())); if p.profile == Profile::Fast { s.push_str(" (representative slice — NOT merge-blocking)"); } s.push_str("\n\n"); s.push_str(&format!( "Scale: executions={}, mutants={}, replay={}, collapse_samples={}\n\n", r.scale.executions, r.scale.mutants, r.scale.replay_cases, r.scale.collapse_samples )); s.push_str("## Provenance\n\n"); s.push_str(&format!("- Execution Merkle root: `{:016x}` over {} leaves\n", p.execution_merkle_root.0, p.merkle_leaf_count)); s.push_str(&format!( "- Reference engine id: `{:016x}`; runtime-under-test engine id: `{:016x}`; agree: **{}**\n", p.reference_engine_id.0, p.rut_engine_id.0, p.engines_agree )); s.push_str(&format!( "- Actual executions: {} (merge floor {}), min perturbations/exec: {} (floor {})\n", p.actual_executions, p.floor.executions, p.min_perturbations_per_exec, p.floor.perturbations_per_exec )); s.push_str(&format!("- Collapse measures real trace structure ({} features/trace)\n\n", p.collapse_feature_width)); s.push_str("| Report | Status | Key metrics |\n|---|---|---|\n"); s.push_str(&format!( "| runtime_equivalence | {} | {}/{} matched, independent impls, engines agree={} |\n", status(r.equivalence.failures.is_empty()), r.equivalence.matched, r.equivalence.total, p.engines_agree )); s.push_str(&format!( "| causal_rank/trace | {} | rank med={} p95={}, edges med={}, touched med={} |\n", status(r.trace.failures.is_empty()), r.trace.median_rank, r.trace.p95_causal_rank, r.trace.median_causal_edges, r.trace.median_touched )); s.push_str(&format!( "| domain_participation | {} | min_merge_loss={:.3} |\n", status(r.domain.failures.is_empty()), r.domain.min_merge_loss )); s.push_str(&format!( "| metamorphic_response | {} | trace={:.3} delta={:.3} future={:.3} |\n", status(r.metamorphic.failures.is_empty()), r.metamorphic.altered_trace, r.metamorphic.altered_delta, r.metamorphic.altered_future )); s.push_str(&format!( "| compression_resistance | {} | 1f={:.3} 2f={:.3} 4f={:.3} single={:.3} pair={:.3} info_loss={:.3} |\n", status(r.collapse.failures.is_empty()), r.collapse.best_1factor, r.collapse.best_2factor, r.collapse.best_4factor, r.collapse.max_single_domain, r.collapse.max_pair, r.collapse.min_info_loss )); s.push_str(&format!( "| mutation_survivor | {} | killed {}/{} by named gate |\n", status(r.mutation.passed()), r.mutation.killed, r.mutation.total )); s.push_str(&format!( "| contract | {} | {}/{} cases |\n", status(r.contract.failures.is_empty()), r.contract.passed, r.contract.total )); s.push_str(&format!( "| replay | {} | {}/{} deterministic (committed corpus), drift={} |\n", status(r.replay.failures.is_empty()), r.replay.deterministic, r.replay.total, r.replay.drift )); s.push_str(&format!( "| coverage | {} | exec={}, perturb={}, rejected={} |\n", status(r.coverage.failures.is_empty()), r.coverage.executions, r.coverage.perturbations, r.coverage.generated_rejected )); s.push_str(&format!( "| provenance | {} | merkle over {} leaves, floor enforced |\n", status(r.provenance.failures.is_empty()), r.provenance.merkle_leaf_count )); s.push_str("\n## Failures\n\n"); let mut any = false; for (name, f) in r.all_failures() { for msg in f { any = true; s.push_str(&format!("- **{}**: {}\n", name, msg)); } } for (id, reason) in &r.mutation.survivors { any = true; s.push_str(&format!("- **mutation_survivor**: mutant {} survived ({})\n", id, reason)); } if !any { s.push_str("None. The fake universe failed to collapse. ✅\n"); } let path = dir.join("ci_summary.md"); fs::write(path, s).expect("write markdown"); } fn main() { let scale = Scale::from_env(); let out_dir = std::env::var("MAGICKA_OUT").unwrap_or_else(|_| "ci_out".to_string()); let dir = Path::new(&out_dir); fs::create_dir_all(dir).expect("create out dir"); eprintln!( "running CI: profile={} executions={} mutants={} replay={} collapse_samples={}", scale.profile.name(), scale.executions, scale.mutants, scale.replay_cases, scale.collapse_samples ); for v in &scale.override_violations { eprintln!(" override rejected: {v}"); } let merge = scale.profile == Profile::Merge; let start = Instant::now(); // Stream full-trace evidence for 100% of base executions straight to disk. let ev_dir = dir.join("evidence"); fs::create_dir_all(&ev_dir).expect("create evidence dir"); let traces_path = ev_dir.join("traces.tsv"); let mut traces_w = std::io::BufWriter::new(fs::File::create(&traces_path).expect("create traces")); traces_w .write_all(b"# full-trace evidence (100% of base executions): headertracedelta\n") .expect("write traces header"); let results = run_all_to(scale, Some(&mut traces_w)); traces_w.flush().expect("flush traces"); drop(traces_w); let elapsed = start.elapsed(); build_reports(dir, &results); write_evidence(dir, &results); write_markdown(dir, &results); let compliance_ok = build_compliance_report(dir, &results); println!("\n=== Magicka VM CI ({:?}) ===", elapsed); for (name, f) in results.all_failures() { println!(" {:<24} {}", name, status(f.is_empty())); } println!( " {:<24} {} ({} killed / {} mutants{})", "mutation_survivor", status(results.mutation.passed()), results.mutation.killed, results.mutation.total, if results.mutation.survivors.is_empty() { String::new() } else { format!(", {} survivors", results.mutation.survivors.len()) } ); println!(" {:<24} {}", "compliance", status(compliance_ok)); println!( " provenance: merkle={:016x} over {} leaves, engines_agree={}, comparisons={}", results.provenance.execution_merkle_root.0, results.provenance.merkle_leaf_count, results.provenance.engines_agree, results.provenance.total_comparisons, ); println!("reports written to {}/", out_dir); let gates_pass = results.passed() && compliance_ok; if !merge { // The fast/tiny profiles are advisory only — they may never stand in for // the merge-blocking acceptance run (compliance: no default-profile // substitution). Report status but make clear this is not acceptance. println!( "\nADVISORY ({} profile): {} — NOT a merge-blocking acceptance run. \ Acceptance requires MAGICKA_PROFILE=merge (full gates).", results.provenance.profile.name(), status(gates_pass) ); // A failing advisory run still fails the PR check; a passing one is green // but explicitly non-acceptance. std::process::exit(if gates_pass { 0 } else { 1 }); } if gates_pass { println!("\nACCEPTANCE: PASS — full merge gates satisfied with persisted, independently reproducible evidence."); std::process::exit(0); } else { println!("\nACCEPTANCE: FAIL"); std::process::exit(1); } }