linus-dandClaude Opus 4.8
bea076df43
Removes the deferral. Each sampled execution's FULL trace (every graph edge, count, weight) and delta are persisted as evidence; the independent attestor reconstructs them, recomputes the canonical trace hash and delta hash, re-derives the replay-record leaf, and confirms it is among the retained Merkle leaves the root is built from. The behavior-fingerprint hash is re-derived from features (not trusted), and f64 divergence is stored bit-exact for an identical hash. - trace_model: ExecutionTrace::serialize/deserialize (round-trips canonical_hash; total on garbage). Test: full_trace_serialize_roundtrips_canonical_hash. - world_model: WorldDelta::serialize/deserialize (round-trips hash). - ci_reports: retains TRACE_EVIDENCE_SAMPLE full traces; writes evidence/traces.tsv. - attestation: depends on trace_model/world_model; reconstructs each trace, recomputes the leaf, requires it to match the claimed leaf AND be a retained leaf. Negative control: tampered_trace_breaks_attestation (corrupting the full trace, leaving the claimed leaf, fails attestation). - merge-gates: requires evidence/traces.tsv; the separate attest step verifies it. End-to-end (fast profile): 256/256 full traces reconstructed and re-derived to retained leaves; recomputed root matches the claim over 6600 leaves. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
694 lines
28 KiB
Rust
694 lines
28 KiB
Rust
//! The `ci` binary: runs the full adversarial framework against the reference
|
|
//! and the independently-implemented runtime under test, writes the required
|
|
//! reports (JSON + markdown) including a provenance report binding the numbers
|
|
//! to executed work, and exits nonzero if any gate fails.
|
|
|
|
use ci_reports::json::Json;
|
|
use ci_reports::{run_all, CiResults, Profile, Scale};
|
|
use std::fs;
|
|
use std::io::Write;
|
|
use std::path::Path;
|
|
use std::time::Instant;
|
|
|
|
fn arr_f(vals: &[f64]) -> Json {
|
|
Json::Arr(vals.iter().map(|&v| Json::Num(v)).collect())
|
|
}
|
|
fn fails(v: &[String]) -> Json {
|
|
Json::Arr(v.iter().map(|s| Json::s(s.clone())).collect())
|
|
}
|
|
fn pass_field(v: &[String]) -> Json {
|
|
Json::Bool(v.is_empty())
|
|
}
|
|
fn hex(h: world_model::Hash) -> Json {
|
|
Json::s(format!("{:016x}", h.0))
|
|
}
|
|
|
|
fn write_report(dir: &Path, name: &str, j: &Json) {
|
|
let path = dir.join(format!("{name}.json"));
|
|
let mut f = fs::File::create(&path).expect("create report");
|
|
f.write_all(j.to_pretty().as_bytes()).expect("write report");
|
|
}
|
|
|
|
/// Write the raw evidence the independent `attest` binary verifies (findings
|
|
/// 2, 3): every Merkle leaf, plus the producer's claims. The attestor recomputes
|
|
/// the root from these leaves and checks it against the claimed root, in a
|
|
/// separate process that never reads the compliance report.
|
|
fn write_evidence(dir: &Path, r: &CiResults) {
|
|
let ev = dir.join("evidence");
|
|
fs::create_dir_all(&ev).expect("create evidence dir");
|
|
|
|
let mut leaves = String::from("leaf\n");
|
|
for h in &r.merkle_leaves {
|
|
leaves.push_str(&format!("{:016x}\n", h.0));
|
|
}
|
|
fs::write(ev.join("leaves.tsv"), leaves).expect("write leaves");
|
|
|
|
let claims = format!(
|
|
"# evidence claims for independent attestation\n\
|
|
root\t{:016x}\n\
|
|
leaf_count\t{}\n\
|
|
total_comparisons\t{}\n\
|
|
reference_engine_id\t{:016x}\n\
|
|
rut_engine_id\t{:016x}\n\
|
|
engines_agree\t{}\n",
|
|
r.provenance.execution_merkle_root.0,
|
|
r.provenance.merkle_leaf_count,
|
|
r.provenance.total_comparisons,
|
|
r.provenance.reference_engine_id.0,
|
|
r.provenance.rut_engine_id.0,
|
|
r.provenance.engines_agree,
|
|
);
|
|
fs::write(ev.join("claims.tsv"), claims).expect("write claims");
|
|
|
|
// Full-trace evidence (finding 4): each line is
|
|
// ws ps cs prs future leaf <TAB> <full trace> <TAB> <full delta>
|
|
// The attestor reconstructs the trace + delta, recomputes the canonical
|
|
// trace hash and the replay-record leaf, and checks the leaf is among the
|
|
// retained leaves. This is the full trace, not a summary.
|
|
let mut traces = String::from("# full-trace evidence: header<TAB>trace<TAB>delta\n");
|
|
for ev_rec in &r.trace_evidence {
|
|
let rr = &ev_rec.replay;
|
|
traces.push_str(&format!(
|
|
"{:016x} {:016x} {:016x} {:016x} {:016x} {:016x}\t{}\t{}\n",
|
|
rr.world_seed,
|
|
rr.program_seed,
|
|
rr.contract_seed,
|
|
rr.perturbation_seed,
|
|
rr.future_hash.0,
|
|
rr.hash().0,
|
|
ev_rec.trace.serialize(),
|
|
ev_rec.delta.serialize(),
|
|
));
|
|
}
|
|
fs::write(ev.join("traces.tsv"), traces).expect("write traces");
|
|
}
|
|
|
|
fn build_reports(dir: &Path, r: &CiResults) {
|
|
// 1. domain_participation_report
|
|
write_report(
|
|
dir,
|
|
"domain_participation_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.domain.failures)),
|
|
("appears_fraction".into(), arr_f(&r.domain.appears)),
|
|
("influences_fraction".into(), arr_f(&r.domain.influences)),
|
|
("mutated_fraction".into(), arr_f(&r.domain.mutated)),
|
|
("removal_diversity_loss".into(), arr_f(&r.domain.removal_loss)),
|
|
("min_merge_loss".into(), Json::Num(r.domain.min_merge_loss)),
|
|
(
|
|
"read_only_domains".into(),
|
|
Json::Arr(r.domain.read_only.iter().map(|&d| Json::Int(d as i64)).collect()),
|
|
),
|
|
(
|
|
"write_only_domains".into(),
|
|
Json::Arr(r.domain.write_only.iter().map(|&d| Json::Int(d as i64)).collect()),
|
|
),
|
|
("failures".into(), fails(&r.domain.failures)),
|
|
]),
|
|
);
|
|
|
|
// 2. causal_rank_report (trace gates)
|
|
write_report(
|
|
dir,
|
|
"causal_rank_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.trace.failures)),
|
|
("median_causal_rank".into(), Json::Num(r.trace.median_rank)),
|
|
("p95_causal_rank".into(), Json::Num(r.trace.p95_causal_rank)),
|
|
("median_causal_edges".into(), Json::Num(r.trace.median_causal_edges)),
|
|
("median_touched_domains".into(), Json::Num(r.trace.median_touched)),
|
|
("p95_touched_domains".into(), Json::Num(r.trace.p95_touched)),
|
|
("fp_collision_rate".into(), Json::Num(r.trace.fp_collision_rate)),
|
|
("largest_cluster".into(), Json::Num(r.trace.largest_cluster)),
|
|
("failures".into(), fails(&r.trace.failures)),
|
|
]),
|
|
);
|
|
|
|
// 3. compression_resistance_report
|
|
let attack_json: Vec<Json> = r
|
|
.collapse
|
|
.reports
|
|
.iter()
|
|
.map(|rep| {
|
|
Json::Obj(vec![
|
|
("attack".into(), Json::s(rep.attack.clone())),
|
|
("reconstructs".into(), Json::Num(rep.predicts)),
|
|
("info_loss".into(), Json::Num(rep.info_loss)),
|
|
("detail".into(), Json::s(rep.detail.clone())),
|
|
])
|
|
})
|
|
.collect();
|
|
write_report(
|
|
dir,
|
|
"compression_resistance_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.collapse.failures)),
|
|
("measures".into(), Json::s("real serialized trace structure")),
|
|
("best_1factor".into(), Json::Num(r.collapse.best_1factor)),
|
|
("best_2factor".into(), Json::Num(r.collapse.best_2factor)),
|
|
("best_4factor".into(), Json::Num(r.collapse.best_4factor)),
|
|
("max_single_domain".into(), Json::Num(r.collapse.max_single_domain)),
|
|
("max_pair".into(), Json::Num(r.collapse.max_pair)),
|
|
("min_info_loss".into(), Json::Num(r.collapse.min_info_loss)),
|
|
("attacks".into(), Json::Arr(attack_json)),
|
|
("failures".into(), fails(&r.collapse.failures)),
|
|
]),
|
|
);
|
|
|
|
// 4. metamorphic_response_report
|
|
write_report(
|
|
dir,
|
|
"metamorphic_response_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.metamorphic.failures)),
|
|
("perturbations".into(), Json::Int(r.metamorphic.total as i64)),
|
|
("altered_trace".into(), Json::Num(r.metamorphic.altered_trace)),
|
|
("altered_delta".into(), Json::Num(r.metamorphic.altered_delta)),
|
|
("altered_future".into(), Json::Num(r.metamorphic.altered_future)),
|
|
("consumed_perturbation_trace_violations".into(), Json::Num(r.metamorphic.expectation_violations)),
|
|
("legitimately_neutral".into(), Json::Num(r.metamorphic.explained_neutral)),
|
|
("consumed_perturbations".into(), Json::Int(r.metamorphic.consumed as i64)),
|
|
("failures".into(), fails(&r.metamorphic.failures)),
|
|
]),
|
|
);
|
|
|
|
// 4b. causal_explanation_report (finding 10: intervention-confirmed edges)
|
|
write_report(
|
|
dir,
|
|
"causal_explanation_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.causal_explanation.failures)),
|
|
("edges_tested".into(), Json::Int(r.causal_explanation.edges_tested as i64)),
|
|
("edges_confirmed".into(), Json::Int(r.causal_explanation.edges_confirmed as i64)),
|
|
("confirmed_fraction".into(), Json::Num(r.causal_explanation.confirmed_fraction)),
|
|
("method".into(), Json::s("ablate recorded causal source lane; require destination delta to change")),
|
|
("failures".into(), fails(&r.causal_explanation.failures)),
|
|
]),
|
|
);
|
|
|
|
// 5. mutation_survivor_report
|
|
let survivors: Vec<Json> = r
|
|
.mutation
|
|
.survivors
|
|
.iter()
|
|
.map(|(id, reason)| {
|
|
Json::Obj(vec![
|
|
("id".into(), Json::Int(*id as i64)),
|
|
("reason".into(), Json::s(reason.clone())),
|
|
])
|
|
})
|
|
.collect();
|
|
write_report(
|
|
dir,
|
|
"mutation_survivor_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), Json::Bool(r.mutation.passed())),
|
|
("killed_by".into(), Json::s("named acceptance gate")),
|
|
("total_mutants".into(), Json::Int(r.mutation.total as i64)),
|
|
("killed".into(), Json::Int(r.mutation.killed as i64)),
|
|
("survivors".into(), Json::Arr(survivors)),
|
|
]),
|
|
);
|
|
|
|
// 6. runtime_equivalence_report
|
|
write_report(
|
|
dir,
|
|
"runtime_equivalence_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.equivalence.failures)),
|
|
("independent_implementations".into(), Json::Bool(r.equivalence.independent)),
|
|
("reference_engine_id".into(), hex(r.provenance.reference_engine_id)),
|
|
("rut_engine_id".into(), hex(r.provenance.rut_engine_id)),
|
|
("engines_agree".into(), Json::Bool(r.provenance.engines_agree)),
|
|
("total".into(), Json::Int(r.equivalence.total as i64)),
|
|
("matched".into(), Json::Int(r.equivalence.matched as i64)),
|
|
("failures".into(), fails(&r.equivalence.failures)),
|
|
]),
|
|
);
|
|
|
|
// 7. replay_report
|
|
write_report(
|
|
dir,
|
|
"replay_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.replay.failures)),
|
|
("loaded_from_committed_corpus".into(), Json::Bool(r.replay.loaded_from_disk)),
|
|
("total".into(), Json::Int(r.replay.total as i64)),
|
|
("deterministic".into(), Json::Int(r.replay.deterministic as i64)),
|
|
("drift".into(), Json::Int(r.replay.drift as i64)),
|
|
("retained_failures_present".into(), Json::Bool(r.replay.retained_present)),
|
|
("retained_failures_total".into(), Json::Int(r.replay.retained_total as i64)),
|
|
("retained_failures_regressions".into(), Json::Int(r.replay.retained_regressions as i64)),
|
|
("failures".into(), fails(&r.replay.failures)),
|
|
]),
|
|
);
|
|
|
|
// 8. coverage_report
|
|
write_report(
|
|
dir,
|
|
"coverage_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&r.coverage.failures)),
|
|
("generated_cases".into(), Json::Int(r.coverage.generated_cases as i64)),
|
|
("generated_rejected".into(), Json::Int(r.coverage.generated_rejected as i64)),
|
|
("contract_rejected".into(), Json::Int(r.coverage.contract_rejected as i64)),
|
|
("executions".into(), Json::Int(r.coverage.executions as i64)),
|
|
("perturbations".into(), Json::Int(r.coverage.perturbations as i64)),
|
|
("contracts_passed".into(), Json::Int(r.contract.passed as i64)),
|
|
("contracts_total".into(), Json::Int(r.contract.total as i64)),
|
|
("failures".into(), fails(&r.coverage.failures)),
|
|
]),
|
|
);
|
|
|
|
// 9. provenance_report — binds the run to executed work.
|
|
let p = &r.provenance;
|
|
write_report(
|
|
dir,
|
|
"provenance_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), pass_field(&p.failures)),
|
|
("profile".into(), Json::s(p.profile.name())),
|
|
(
|
|
"merge_floor".into(),
|
|
Json::Obj(vec![
|
|
("worlds".into(), Json::Int(p.floor.worlds as i64)),
|
|
("programs".into(), Json::Int(p.floor.programs as i64)),
|
|
("executions".into(), Json::Int(p.floor.executions as i64)),
|
|
("perturbations_per_exec".into(), Json::Int(p.floor.perturbations_per_exec as i64)),
|
|
("mutants".into(), Json::Int(p.floor.mutants as i64)),
|
|
("replay_cases".into(), Json::Int(p.floor.replay_cases as i64)),
|
|
]),
|
|
),
|
|
("worlds_generated".into(), Json::Int(p.worlds_generated as i64)),
|
|
("programs_generated".into(), Json::Int(p.programs_generated as i64)),
|
|
("actual_executions".into(), Json::Int(p.actual_executions as i64)),
|
|
("min_perturbations_per_exec".into(), Json::Int(p.min_perturbations_per_exec as i64)),
|
|
("total_comparisons".into(), Json::Int(p.total_comparisons as i64)),
|
|
("actual_mutants".into(), Json::Int(p.actual_mutants as i64)),
|
|
("replay_total".into(), Json::Int(p.replay_total as i64)),
|
|
("reference_engine_id".into(), hex(p.reference_engine_id)),
|
|
("rut_engine_id".into(), hex(p.rut_engine_id)),
|
|
("engines_agree".into(), Json::Bool(p.engines_agree)),
|
|
("execution_merkle_root".into(), hex(p.execution_merkle_root)),
|
|
("merkle_leaf_count".into(), Json::Int(p.merkle_leaf_count as i64)),
|
|
("collapse_feature_width".into(), Json::Int(p.collapse_feature_width as i64)),
|
|
("failures".into(), fails(&p.failures)),
|
|
]),
|
|
);
|
|
}
|
|
|
|
/// The reports the spec requires every CI run to produce. A missing or empty
|
|
/// artifact is itself an acceptance failure (compliance rule 4).
|
|
const REQUIRED_REPORTS: [&str; 10] = [
|
|
"domain_participation_report",
|
|
"causal_rank_report",
|
|
"causal_explanation_report",
|
|
"compression_resistance_report",
|
|
"metamorphic_response_report",
|
|
"mutation_survivor_report",
|
|
"runtime_equivalence_report",
|
|
"replay_report",
|
|
"coverage_report",
|
|
"provenance_report",
|
|
];
|
|
|
|
fn report_present(dir: &Path, name: &str) -> bool {
|
|
fs::metadata(dir.join(format!("{name}.json")))
|
|
.map(|m| m.len() > 0)
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// One acceptance obligation: a measured artifact, a provenance chain, a
|
|
/// merge-blocking enforcement point, and a failure if the artifact/provenance is
|
|
/// absent. This is the compliance model made machine-checkable.
|
|
struct Obligation {
|
|
requirement: &'static str,
|
|
artifact: &'static str,
|
|
floor: i64,
|
|
actual: i64,
|
|
gate_pass: bool,
|
|
}
|
|
|
|
fn obligations(r: &CiResults) -> Vec<Obligation> {
|
|
let p = &r.provenance;
|
|
let merge = p.profile == Profile::Merge;
|
|
let f = |v: usize| v as i64;
|
|
let floor = |v: usize| if merge { v as i64 } else { 0 };
|
|
vec![
|
|
Obligation {
|
|
requirement: "generated worlds >= 50,000",
|
|
artifact: "provenance_report",
|
|
floor: floor(p.floor.worlds),
|
|
actual: f(p.worlds_generated),
|
|
gate_pass: !merge || p.worlds_generated >= p.floor.worlds,
|
|
},
|
|
Obligation {
|
|
requirement: "generated programs >= 250,000",
|
|
artifact: "provenance_report",
|
|
floor: floor(p.floor.programs),
|
|
actual: f(p.programs_generated),
|
|
gate_pass: !merge || p.programs_generated >= p.floor.programs,
|
|
},
|
|
Obligation {
|
|
requirement: "executions >= 1,000,000",
|
|
artifact: "provenance_report",
|
|
floor: floor(p.floor.executions),
|
|
actual: f(p.actual_executions),
|
|
gate_pass: !merge || p.actual_executions >= p.floor.executions,
|
|
},
|
|
Obligation {
|
|
requirement: "perturbations per execution >= 10",
|
|
artifact: "metamorphic_response_report",
|
|
floor: floor(p.floor.perturbations_per_exec),
|
|
actual: f(p.min_perturbations_per_exec),
|
|
gate_pass: !merge || p.min_perturbations_per_exec >= p.floor.perturbations_per_exec,
|
|
},
|
|
Obligation {
|
|
requirement: "reference/runtime comparison = 100% of executions",
|
|
artifact: "runtime_equivalence_report",
|
|
floor: floor(p.floor.executions * (1 + p.floor.perturbations_per_exec)),
|
|
actual: f(p.total_comparisons),
|
|
gate_pass: r.equivalence.failures.is_empty(),
|
|
},
|
|
Obligation {
|
|
requirement: "semantic mutants >= 500, 0 survivors, killed by named gate",
|
|
artifact: "mutation_survivor_report",
|
|
floor: floor(p.floor.mutants),
|
|
actual: f(r.mutation.total),
|
|
gate_pass: r.mutation.passed() && (!merge || r.mutation.total >= p.floor.mutants),
|
|
},
|
|
Obligation {
|
|
requirement: "replay corpus >= 10,000, persisted, 0 drift",
|
|
artifact: "replay_report",
|
|
floor: floor(p.floor.replay_cases),
|
|
actual: f(r.replay.total),
|
|
gate_pass: r.replay.failures.is_empty() && r.replay.loaded_from_disk,
|
|
},
|
|
Obligation {
|
|
requirement: "collapse attacks fail to simplify (real trace info)",
|
|
artifact: "compression_resistance_report",
|
|
floor: 0,
|
|
actual: 0,
|
|
gate_pass: r.collapse.failures.is_empty(),
|
|
},
|
|
Obligation {
|
|
requirement: "trace gates (causal edges/rank/touched/fingerprints)",
|
|
artifact: "causal_rank_report",
|
|
floor: 0,
|
|
actual: 0,
|
|
gate_pass: r.trace.failures.is_empty(),
|
|
},
|
|
Obligation {
|
|
requirement: "domain participation (no decorative/read-only/write-only)",
|
|
artifact: "domain_participation_report",
|
|
floor: 0,
|
|
actual: 0,
|
|
gate_pass: r.domain.failures.is_empty(),
|
|
},
|
|
Obligation {
|
|
requirement: "metamorphic response thresholds",
|
|
artifact: "metamorphic_response_report",
|
|
floor: 0,
|
|
actual: 0,
|
|
gate_pass: r.metamorphic.failures.is_empty(),
|
|
},
|
|
Obligation {
|
|
requirement: "recorded causal edges are intervention-confirmed (not counted)",
|
|
artifact: "causal_explanation_report",
|
|
floor: 0,
|
|
actual: f(r.causal_explanation.edges_confirmed),
|
|
gate_pass: r.causal_explanation.failures.is_empty(),
|
|
},
|
|
Obligation {
|
|
requirement: "every admitted case satisfies its contract",
|
|
artifact: "coverage_report",
|
|
floor: 0,
|
|
actual: 0,
|
|
gate_pass: r.contract.failures.is_empty() && r.coverage.failures.is_empty(),
|
|
},
|
|
Obligation {
|
|
requirement: "provenance binds reports to executed work",
|
|
artifact: "provenance_report",
|
|
floor: 0,
|
|
actual: 0,
|
|
gate_pass: r.provenance.failures.is_empty(),
|
|
},
|
|
]
|
|
}
|
|
|
|
/// Write the compliance report and return whether the compliance model holds:
|
|
/// every obligation's artifact is present and every obligation passes.
|
|
fn build_compliance_report(dir: &Path, r: &CiResults) -> bool {
|
|
let items = obligations(r);
|
|
let mut all_ok = true;
|
|
let mut json_items = Vec::new();
|
|
for ob in &items {
|
|
let present = report_present(dir, ob.artifact);
|
|
let pass = present && ob.gate_pass;
|
|
if !pass {
|
|
all_ok = false;
|
|
}
|
|
json_items.push(Json::Obj(vec![
|
|
("requirement".into(), Json::s(ob.requirement)),
|
|
("measured_artifact".into(), Json::s(format!("{}.json", ob.artifact))),
|
|
("artifact_present".into(), Json::Bool(present)),
|
|
("provenance".into(), Json::s("provenance_report.json (merkle root + engine ids)")),
|
|
("merge_blocking".into(), Json::Bool(true)),
|
|
("floor".into(), Json::Int(ob.floor)),
|
|
("actual".into(), Json::Int(ob.actual)),
|
|
("pass".into(), Json::Bool(pass)),
|
|
]));
|
|
}
|
|
// Every required report must exist and be non-empty.
|
|
let mut missing = Vec::new();
|
|
for rep in REQUIRED_REPORTS {
|
|
if !report_present(dir, rep) {
|
|
missing.push(rep.to_string());
|
|
all_ok = false;
|
|
}
|
|
}
|
|
write_report(
|
|
dir,
|
|
"compliance_report",
|
|
&Json::Obj(vec![
|
|
("pass".into(), Json::Bool(all_ok)),
|
|
("profile".into(), Json::s(r.provenance.profile.name())),
|
|
(
|
|
"merge_blocking_run".into(),
|
|
Json::Bool(r.provenance.profile == Profile::Merge),
|
|
),
|
|
(
|
|
"note".into(),
|
|
Json::s(if r.provenance.profile == Profile::Merge {
|
|
"merge profile: floors enforced, all obligations acceptance-blocking"
|
|
} else {
|
|
"advisory profile: NOT acceptance; floors not enforced (fast/tiny)"
|
|
}),
|
|
),
|
|
("required_reports".into(), Json::Arr(REQUIRED_REPORTS.iter().map(|s| Json::s(*s)).collect())),
|
|
("missing_reports".into(), Json::Arr(missing.iter().map(|s| Json::s(s.clone())).collect())),
|
|
("obligations".into(), Json::Arr(json_items)),
|
|
]),
|
|
);
|
|
all_ok
|
|
}
|
|
|
|
fn status(v: bool) -> &'static str {
|
|
if v {
|
|
"PASS"
|
|
} else {
|
|
"FAIL"
|
|
}
|
|
}
|
|
|
|
fn write_markdown(dir: &Path, r: &CiResults) {
|
|
let p = &r.provenance;
|
|
let mut s = String::new();
|
|
s.push_str("# Magicka VM — Phase 0/1 CI Report\n\n");
|
|
s.push_str(&format!("Overall: **{}**\n\n", status(r.passed())));
|
|
s.push_str(&format!("Profile: **{}**", p.profile.name()));
|
|
if p.profile == Profile::Fast {
|
|
s.push_str(" (representative slice — NOT merge-blocking)");
|
|
}
|
|
s.push_str("\n\n");
|
|
s.push_str(&format!(
|
|
"Scale: executions={}, mutants={}, replay={}, collapse_samples={}\n\n",
|
|
r.scale.executions, r.scale.mutants, r.scale.replay_cases, r.scale.collapse_samples
|
|
));
|
|
s.push_str("## Provenance\n\n");
|
|
s.push_str(&format!("- Execution Merkle root: `{:016x}` over {} leaves\n", p.execution_merkle_root.0, p.merkle_leaf_count));
|
|
s.push_str(&format!(
|
|
"- Reference engine id: `{:016x}`; runtime-under-test engine id: `{:016x}`; agree: **{}**\n",
|
|
p.reference_engine_id.0, p.rut_engine_id.0, p.engines_agree
|
|
));
|
|
s.push_str(&format!(
|
|
"- Actual executions: {} (merge floor {}), min perturbations/exec: {} (floor {})\n",
|
|
p.actual_executions, p.floor.executions, p.min_perturbations_per_exec, p.floor.perturbations_per_exec
|
|
));
|
|
s.push_str(&format!("- Collapse measures real trace structure ({} features/trace)\n\n", p.collapse_feature_width));
|
|
|
|
s.push_str("| Report | Status | Key metrics |\n|---|---|---|\n");
|
|
s.push_str(&format!(
|
|
"| runtime_equivalence | {} | {}/{} matched, independent impls, engines agree={} |\n",
|
|
status(r.equivalence.failures.is_empty()),
|
|
r.equivalence.matched,
|
|
r.equivalence.total,
|
|
p.engines_agree
|
|
));
|
|
s.push_str(&format!(
|
|
"| causal_rank/trace | {} | rank med={} p95={}, edges med={}, touched med={} |\n",
|
|
status(r.trace.failures.is_empty()),
|
|
r.trace.median_rank,
|
|
r.trace.p95_causal_rank,
|
|
r.trace.median_causal_edges,
|
|
r.trace.median_touched
|
|
));
|
|
s.push_str(&format!(
|
|
"| domain_participation | {} | min_merge_loss={:.3} |\n",
|
|
status(r.domain.failures.is_empty()),
|
|
r.domain.min_merge_loss
|
|
));
|
|
s.push_str(&format!(
|
|
"| metamorphic_response | {} | trace={:.3} delta={:.3} future={:.3} |\n",
|
|
status(r.metamorphic.failures.is_empty()),
|
|
r.metamorphic.altered_trace,
|
|
r.metamorphic.altered_delta,
|
|
r.metamorphic.altered_future
|
|
));
|
|
s.push_str(&format!(
|
|
"| compression_resistance | {} | 1f={:.3} 2f={:.3} 4f={:.3} single={:.3} pair={:.3} info_loss={:.3} |\n",
|
|
status(r.collapse.failures.is_empty()),
|
|
r.collapse.best_1factor,
|
|
r.collapse.best_2factor,
|
|
r.collapse.best_4factor,
|
|
r.collapse.max_single_domain,
|
|
r.collapse.max_pair,
|
|
r.collapse.min_info_loss
|
|
));
|
|
s.push_str(&format!(
|
|
"| mutation_survivor | {} | killed {}/{} by named gate |\n",
|
|
status(r.mutation.passed()),
|
|
r.mutation.killed,
|
|
r.mutation.total
|
|
));
|
|
s.push_str(&format!(
|
|
"| contract | {} | {}/{} cases |\n",
|
|
status(r.contract.failures.is_empty()),
|
|
r.contract.passed,
|
|
r.contract.total
|
|
));
|
|
s.push_str(&format!(
|
|
"| replay | {} | {}/{} deterministic (committed corpus), drift={} |\n",
|
|
status(r.replay.failures.is_empty()),
|
|
r.replay.deterministic,
|
|
r.replay.total,
|
|
r.replay.drift
|
|
));
|
|
s.push_str(&format!(
|
|
"| coverage | {} | exec={}, perturb={}, rejected={} |\n",
|
|
status(r.coverage.failures.is_empty()),
|
|
r.coverage.executions,
|
|
r.coverage.perturbations,
|
|
r.coverage.generated_rejected
|
|
));
|
|
s.push_str(&format!(
|
|
"| provenance | {} | merkle over {} leaves, floor enforced |\n",
|
|
status(r.provenance.failures.is_empty()),
|
|
r.provenance.merkle_leaf_count
|
|
));
|
|
|
|
s.push_str("\n## Failures\n\n");
|
|
let mut any = false;
|
|
for (name, f) in r.all_failures() {
|
|
for msg in f {
|
|
any = true;
|
|
s.push_str(&format!("- **{}**: {}\n", name, msg));
|
|
}
|
|
}
|
|
for (id, reason) in &r.mutation.survivors {
|
|
any = true;
|
|
s.push_str(&format!("- **mutation_survivor**: mutant {} survived ({})\n", id, reason));
|
|
}
|
|
if !any {
|
|
s.push_str("None. The fake universe failed to collapse. ✅\n");
|
|
}
|
|
|
|
let path = dir.join("ci_summary.md");
|
|
fs::write(path, s).expect("write markdown");
|
|
}
|
|
|
|
fn main() {
|
|
let scale = Scale::from_env();
|
|
let out_dir = std::env::var("MAGICKA_OUT").unwrap_or_else(|_| "ci_out".to_string());
|
|
let dir = Path::new(&out_dir);
|
|
fs::create_dir_all(dir).expect("create out dir");
|
|
|
|
eprintln!(
|
|
"running CI: profile={} executions={} mutants={} replay={} collapse_samples={}",
|
|
scale.profile.name(), scale.executions, scale.mutants, scale.replay_cases, scale.collapse_samples
|
|
);
|
|
for v in &scale.override_violations {
|
|
eprintln!(" override rejected: {v}");
|
|
}
|
|
let merge = scale.profile == Profile::Merge;
|
|
let start = Instant::now();
|
|
let results = run_all(scale);
|
|
let elapsed = start.elapsed();
|
|
|
|
build_reports(dir, &results);
|
|
write_evidence(dir, &results);
|
|
write_markdown(dir, &results);
|
|
let compliance_ok = build_compliance_report(dir, &results);
|
|
|
|
println!("\n=== Magicka VM CI ({:?}) ===", elapsed);
|
|
for (name, f) in results.all_failures() {
|
|
println!(" {:<24} {}", name, status(f.is_empty()));
|
|
}
|
|
println!(
|
|
" {:<24} {} ({} killed / {} mutants{})",
|
|
"mutation_survivor",
|
|
status(results.mutation.passed()),
|
|
results.mutation.killed,
|
|
results.mutation.total,
|
|
if results.mutation.survivors.is_empty() {
|
|
String::new()
|
|
} else {
|
|
format!(", {} survivors", results.mutation.survivors.len())
|
|
}
|
|
);
|
|
println!(" {:<24} {}", "compliance", status(compliance_ok));
|
|
println!(
|
|
" provenance: merkle={:016x} over {} leaves, engines_agree={}, comparisons={}",
|
|
results.provenance.execution_merkle_root.0,
|
|
results.provenance.merkle_leaf_count,
|
|
results.provenance.engines_agree,
|
|
results.provenance.total_comparisons,
|
|
);
|
|
println!("reports written to {}/", out_dir);
|
|
|
|
let gates_pass = results.passed() && compliance_ok;
|
|
|
|
if !merge {
|
|
// The fast/tiny profiles are advisory only — they may never stand in for
|
|
// the merge-blocking acceptance run (compliance: no default-profile
|
|
// substitution). Report status but make clear this is not acceptance.
|
|
println!(
|
|
"\nADVISORY ({} profile): {} — NOT a merge-blocking acceptance run. \
|
|
Acceptance requires MAGICKA_PROFILE=merge (full gates).",
|
|
results.provenance.profile.name(),
|
|
status(gates_pass)
|
|
);
|
|
// A failing advisory run still fails the PR check; a passing one is green
|
|
// but explicitly non-acceptance.
|
|
std::process::exit(if gates_pass { 0 } else { 1 });
|
|
}
|
|
|
|
if gates_pass {
|
|
println!("\nACCEPTANCE: PASS — full merge gates satisfied with persisted, independently reproducible evidence.");
|
|
std::process::exit(0);
|
|
} else {
|
|
println!("\nACCEPTANCE: FAIL");
|
|
std::process::exit(1);
|
|
}
|
|
}
|