update
This commit is contained in:
+515
-79
@@ -1,83 +1,183 @@
|
||||
//! `ci_reports` — the CI orchestrator. Runs the whole adversarial framework
|
||||
//! against the reference runtime and the runtime under test, evaluates every
|
||||
//! gate, and emits machine-readable JSON plus a human-readable markdown
|
||||
//! summary. Merge is blocked unless all reports pass.
|
||||
//! against the reference runtime and the (independently implemented) runtime
|
||||
//! under test, evaluates every gate, and emits machine-readable JSON plus a
|
||||
//! human-readable markdown summary. Merge is blocked unless all reports pass.
|
||||
//!
|
||||
//! Governing rule (enforced structurally, not by naming): *no gate may pass from
|
||||
//! configuration, naming, shared implementation, smoke-scale runs, regenerated
|
||||
//! expectations, or proxy metrics.* Concretely:
|
||||
//! - the equivalence gate compares two independent implementations;
|
||||
//! - the merge profile's scale floor cannot be lowered by environment overrides,
|
||||
//! and the executions actually performed are counted and checked;
|
||||
//! - the replay corpus is loaded from a committed file, not regenerated;
|
||||
//! - collapse analysis measures real trace information;
|
||||
//! - mutants must fail their named gate;
|
||||
//! - every report carries provenance binding its numbers to executed work.
|
||||
|
||||
pub mod json;
|
||||
|
||||
use collapse_analysis::{analyze, BehaviorCorpus, CollapseSummary};
|
||||
use collapse_analysis::{analyze, BehaviorCorpus, CollapseSummary, FEATURE_W};
|
||||
use generators::{generate_accepted_case, generated_gate_failures, GeneratedCase};
|
||||
use reference_runtime::{canonical, execute, EngineConfig, ResolutionInput, ResolutionResult, Runtime};
|
||||
use runtime_under_test::RuntimeUnderTest;
|
||||
use reference_runtime::{
|
||||
canonical, execute, Canonical, EngineConfig, ResolutionInput, ResolutionResult, Runtime,
|
||||
};
|
||||
use runtime_under_test::{native_resolve, RuntimeUnderTest};
|
||||
use semantic_mutation::{run_suite, MutationOutcome};
|
||||
use std::collections::HashMap;
|
||||
use world_model::{Hash, WorldSnapshot, HIDDEN_LANES, LANES, NUM_DOMAINS};
|
||||
use world_model::{Hash, Hasher, WorldSnapshot, NUM_DOMAINS};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scale configuration.
|
||||
// Run profile + scale, with an unbypassable merge floor.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum Profile {
|
||||
/// Representative slice; explicitly NOT merge-blocking.
|
||||
Fast,
|
||||
/// Full, merge-blocking gates with hard floors that overrides cannot lower.
|
||||
Merge,
|
||||
}
|
||||
|
||||
impl Profile {
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Profile::Fast => "fast",
|
||||
Profile::Merge => "merge",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The hard minimums the merge profile mandates (spec "Hard CI Gates").
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ScaleFloor {
|
||||
pub worlds: usize,
|
||||
pub programs: usize,
|
||||
pub executions: usize,
|
||||
pub perturbations_per_exec: usize,
|
||||
pub mutants: usize,
|
||||
pub replay_cases: usize,
|
||||
}
|
||||
|
||||
impl ScaleFloor {
|
||||
pub const NONE: ScaleFloor = ScaleFloor {
|
||||
worlds: 0,
|
||||
programs: 0,
|
||||
executions: 0,
|
||||
perturbations_per_exec: 0,
|
||||
mutants: 0,
|
||||
replay_cases: 0,
|
||||
};
|
||||
pub const MERGE: ScaleFloor = ScaleFloor {
|
||||
worlds: 50_000,
|
||||
programs: 250_000,
|
||||
executions: 1_000_000,
|
||||
perturbations_per_exec: 10,
|
||||
mutants: 500,
|
||||
replay_cases: 10_000,
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Scale {
|
||||
pub profile: Profile,
|
||||
pub floor: ScaleFloor,
|
||||
pub executions: usize,
|
||||
pub mutants: usize,
|
||||
pub replay_cases: usize,
|
||||
pub collapse_samples: usize,
|
||||
pub domain_probe_cases: usize,
|
||||
pub domain_probe_variations: usize,
|
||||
/// Overrides that tried to lower a merge floor (recorded, never applied).
|
||||
pub override_violations: Vec<String>,
|
||||
}
|
||||
|
||||
impl Scale {
|
||||
/// The full, merge-blocking gates mandated by the spec.
|
||||
pub fn full() -> Self {
|
||||
pub fn merge() -> Self {
|
||||
Scale {
|
||||
profile: Profile::Merge,
|
||||
floor: ScaleFloor::MERGE,
|
||||
executions: 1_000_000,
|
||||
mutants: 600,
|
||||
replay_cases: 10_000,
|
||||
collapse_samples: 5_000,
|
||||
domain_probe_cases: 2_000,
|
||||
domain_probe_variations: 64,
|
||||
override_violations: Vec::new(),
|
||||
}
|
||||
}
|
||||
/// Fast CI (~10% spirit): small but exercises every gate.
|
||||
pub fn fast() -> Self {
|
||||
Scale {
|
||||
profile: Profile::Fast,
|
||||
floor: ScaleFloor::NONE,
|
||||
executions: 600,
|
||||
mutants: 520,
|
||||
replay_cases: 600,
|
||||
collapse_samples: 600,
|
||||
domain_probe_cases: 200,
|
||||
domain_probe_variations: 48,
|
||||
override_violations: Vec::new(),
|
||||
}
|
||||
}
|
||||
pub fn tiny() -> Self {
|
||||
Scale {
|
||||
profile: Profile::Fast,
|
||||
floor: ScaleFloor::NONE,
|
||||
executions: 120,
|
||||
mutants: 520,
|
||||
replay_cases: 120,
|
||||
collapse_samples: 120,
|
||||
domain_probe_cases: 60,
|
||||
domain_probe_variations: 32,
|
||||
override_violations: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply an override to a field. On the merge profile an override may only
|
||||
/// raise the value; attempting to lower it below the floor is recorded as a
|
||||
/// violation and the floor is kept.
|
||||
fn apply_override(&mut self, name: &str, value: usize, field_floor: usize, set: impl Fn(&mut Scale, usize)) {
|
||||
if self.profile == Profile::Merge && value < field_floor {
|
||||
self.override_violations.push(format!(
|
||||
"override {name}={value} would lower merge floor {field_floor}; ignored"
|
||||
));
|
||||
set(self, field_floor);
|
||||
} else {
|
||||
set(self, value);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_env() -> Self {
|
||||
let mut s = match std::env::var("MAGICKA_SCALE").as_deref() {
|
||||
Ok("full") => Scale::full(),
|
||||
Ok("tiny") => Scale::tiny(),
|
||||
_ => Scale::fast(),
|
||||
let scale_var = std::env::var("MAGICKA_SCALE").ok();
|
||||
let profile_var = std::env::var("MAGICKA_PROFILE").ok();
|
||||
let profile = if profile_var.as_deref() == Some("merge")
|
||||
|| scale_var.as_deref() == Some("full")
|
||||
{
|
||||
Profile::Merge
|
||||
} else {
|
||||
Profile::Fast
|
||||
};
|
||||
|
||||
let mut s = match (profile, scale_var.as_deref()) {
|
||||
(Profile::Merge, _) => Scale::merge(),
|
||||
(Profile::Fast, Some("tiny")) => Scale::tiny(),
|
||||
(Profile::Fast, _) => Scale::fast(),
|
||||
};
|
||||
|
||||
if let Some(v) = env_usize("MAGICKA_EXECUTIONS") {
|
||||
s.executions = v;
|
||||
let f = s.floor.executions;
|
||||
s.apply_override("MAGICKA_EXECUTIONS", v, f, |s, x| s.executions = x);
|
||||
}
|
||||
if let Some(v) = env_usize("MAGICKA_MUTANTS") {
|
||||
s.mutants = v;
|
||||
let f = s.floor.mutants;
|
||||
s.apply_override("MAGICKA_MUTANTS", v, f, |s, x| s.mutants = x);
|
||||
}
|
||||
if let Some(v) = env_usize("MAGICKA_REPLAY") {
|
||||
s.replay_cases = v;
|
||||
let f = s.floor.replay_cases;
|
||||
s.apply_override("MAGICKA_REPLAY", v, f, |s, x| s.replay_cases = x);
|
||||
}
|
||||
if let Some(v) = env_usize("MAGICKA_COLLAPSE") {
|
||||
s.collapse_samples = v;
|
||||
// collapse sample count has no spec floor; never below 1.
|
||||
s.collapse_samples = v.max(1);
|
||||
}
|
||||
s
|
||||
}
|
||||
@@ -87,6 +187,81 @@ fn env_usize(key: &str) -> Option<usize> {
|
||||
std::env::var(key).ok().and_then(|v| v.parse().ok())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Provenance: bind the reported numbers to executed work.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A tamper-evident record of what actually ran. The Merkle root is computed
|
||||
/// over the per-execution replay-record hashes, so the report cannot claim more
|
||||
/// executions than were performed. The two engine identities are independent
|
||||
/// fingerprints of the reference and the runtime under test; they must agree,
|
||||
/// which is evidence the two implementations match.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Provenance {
|
||||
pub profile: Profile,
|
||||
pub floor: ScaleFloor,
|
||||
pub worlds_generated: usize,
|
||||
pub programs_generated: usize,
|
||||
pub actual_executions: usize,
|
||||
pub min_perturbations_per_exec: usize,
|
||||
pub total_comparisons: usize,
|
||||
pub actual_mutants: usize,
|
||||
pub replay_total: usize,
|
||||
pub reference_engine_id: Hash,
|
||||
pub rut_engine_id: Hash,
|
||||
pub engines_agree: bool,
|
||||
pub execution_merkle_root: Hash,
|
||||
pub merkle_leaf_count: usize,
|
||||
pub collapse_feature_width: usize,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
/// Merkle root over execution leaves (FNV-combined binary tree).
|
||||
pub fn merkle_root(leaves: &[Hash]) -> Hash {
|
||||
if leaves.is_empty() {
|
||||
return Hash(0);
|
||||
}
|
||||
let mut level: Vec<Hash> = leaves.to_vec();
|
||||
while level.len() > 1 {
|
||||
let mut next = Vec::with_capacity(level.len().div_ceil(2));
|
||||
for pair in level.chunks(2) {
|
||||
let mut h = Hasher::new();
|
||||
h.write_tag("merkle");
|
||||
h.write_u64(pair[0].0);
|
||||
h.write_u64(if pair.len() > 1 { pair[1].0 } else { pair[0].0 });
|
||||
next.push(h.finish());
|
||||
}
|
||||
level = next;
|
||||
}
|
||||
level[0]
|
||||
}
|
||||
|
||||
/// Fixed probe inputs for engine fingerprinting (independent of the corpus).
|
||||
fn probe_inputs(count: usize) -> Vec<ResolutionInput> {
|
||||
(0..count)
|
||||
.map(|i| {
|
||||
let seed = 0xF1A9_E000_u64 ^ (i as u64).wrapping_mul(0x9E3779B97F4A7C15);
|
||||
let (case, _) = generate_accepted_case(seed);
|
||||
input_from_case(&case)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Fingerprint of a runtime: hash of its canonical outputs over the probe set.
|
||||
fn engine_fingerprint(probes: &[ResolutionInput], resolve: impl Fn(&ResolutionInput) -> ResolutionResult) -> Hash {
|
||||
let mut h = Hasher::new();
|
||||
h.write_tag("engine-id");
|
||||
for inp in probes {
|
||||
let c = canonical(&resolve(inp));
|
||||
h.write_u64(c.delta_hash.0);
|
||||
h.write_u64(c.trace_hash.0);
|
||||
h.write_u64(c.fault_hash.0);
|
||||
h.write_u64(c.replay_hash.0);
|
||||
h.write_u64(c.future_hash.0);
|
||||
}
|
||||
h.finish()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers.
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -165,26 +340,50 @@ fn compressibility(features: &[i64]) -> f64 {
|
||||
(1.0 - h / 8.0).clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
fn world_input_features(w: &WorldSnapshot) -> Vec<f64> {
|
||||
let mut out = Vec::with_capacity(NUM_DOMAINS * (LANES + HIDDEN_LANES));
|
||||
for d in &w.domains {
|
||||
for &v in &d.observed {
|
||||
out.push(v as f64);
|
||||
}
|
||||
for &v in &d.hidden {
|
||||
out.push(v as f64);
|
||||
}
|
||||
/// Real serialized-trace feature row used by the collapse analysis. Layout per
|
||||
/// domain block (width 9): `[infl_out, infl_in, flow_out, flow_in, read, write,
|
||||
/// temporal, obs_delta, hid_delta]`, followed by globals `[causal_rank,
|
||||
/// edge_count, touched, divergence_mean]`. This is genuine trace structure, not
|
||||
/// a hash-derived proxy.
|
||||
fn trace_feature_row(r: &ResolutionResult) -> Vec<f64> {
|
||||
let infl = r.trace.causal_graph.influence_matrix();
|
||||
let mut flow = [[0.0f64; NUM_DOMAINS]; NUM_DOMAINS];
|
||||
for &(a, b, bits) in &r.trace.information_flow.edges {
|
||||
flow[a as usize % NUM_DOMAINS][b as usize % NUM_DOMAINS] += bits as f64;
|
||||
}
|
||||
let mut temporal = [0.0f64; NUM_DOMAINS];
|
||||
for &(_s, _off, d) in &r.trace.temporal_graph.edges {
|
||||
temporal[d as usize % NUM_DOMAINS] += 1.0;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn behavior_output_features(r: &ResolutionResult) -> Vec<f64> {
|
||||
r.trace
|
||||
.behavior_fingerprint
|
||||
.features
|
||||
.iter()
|
||||
.map(|&v| v as f64)
|
||||
.collect()
|
||||
let mut row = Vec::with_capacity(FEATURE_W);
|
||||
for d in 0..NUM_DOMAINS {
|
||||
let infl_out: f64 = (0..NUM_DOMAINS).map(|j| infl[d][j]).sum();
|
||||
let infl_in: f64 = (0..NUM_DOMAINS).map(|i| infl[i][d]).sum();
|
||||
let flow_out: f64 = (0..NUM_DOMAINS).map(|j| flow[d][j]).sum();
|
||||
let flow_in: f64 = (0..NUM_DOMAINS).map(|i| flow[i][d]).sum();
|
||||
let read = r.trace.read_graph.access_count[d] as f64;
|
||||
let write = r.trace.write_graph.access_count[d] as f64;
|
||||
let temp = temporal[d];
|
||||
let obs_delta: f64 = r.delta.domain_deltas[d]
|
||||
.observed
|
||||
.iter()
|
||||
.map(|&v| (v as f64).abs())
|
||||
.sum();
|
||||
let hid_delta: f64 = r.delta.domain_deltas[d]
|
||||
.hidden
|
||||
.iter()
|
||||
.map(|&v| (v as f64).abs())
|
||||
.sum();
|
||||
row.extend_from_slice(&[
|
||||
infl_out, infl_in, flow_out, flow_in, read, write, temp, obs_delta, hid_delta,
|
||||
]);
|
||||
}
|
||||
row.push(r.trace.causal_rank() as f64);
|
||||
row.push(r.trace.causal_edge_count() as f64);
|
||||
row.push(r.trace.touched_domain_count() as f64);
|
||||
row.push(r.trace.context_divergence());
|
||||
row
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -207,6 +406,7 @@ pub struct TraceGates {
|
||||
pub struct EquivalenceGates {
|
||||
pub total: usize,
|
||||
pub matched: usize,
|
||||
pub independent: bool,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
@@ -244,6 +444,7 @@ pub struct ReplayGates {
|
||||
pub total: usize,
|
||||
pub deterministic: usize,
|
||||
pub drift: usize,
|
||||
pub loaded_from_disk: bool,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
@@ -259,6 +460,7 @@ pub struct CoverageGates {
|
||||
|
||||
pub struct CiResults {
|
||||
pub scale: Scale,
|
||||
pub provenance: Provenance,
|
||||
pub trace: TraceGates,
|
||||
pub equivalence: EquivalenceGates,
|
||||
pub domain: DomainGates,
|
||||
@@ -281,6 +483,7 @@ impl CiResults {
|
||||
("contract", &self.contract.failures),
|
||||
("replay", &self.replay.failures),
|
||||
("coverage", &self.coverage.failures),
|
||||
("provenance", &self.provenance.failures),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -297,7 +500,26 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
let cfg = EngineConfig::reference();
|
||||
let rut = RuntimeUnderTest::new();
|
||||
|
||||
// Main execution + metamorphic + contract pass.
|
||||
// Progress logging: a long merge run must be trackable, not a silent black
|
||||
// box. Emit periodic stderr lines with elapsed time and counts. Suppressed
|
||||
// for small runs (tests) to keep their output clean.
|
||||
let started = std::time::Instant::now();
|
||||
let report_progress = scale.executions >= 10_000;
|
||||
let prog_step = (scale.executions / 20).max(1);
|
||||
macro_rules! progress {
|
||||
($($arg:tt)*) => {
|
||||
if report_progress {
|
||||
eprintln!(" [{:7.1}s] {}", started.elapsed().as_secs_f64(), format_args!($($arg)*));
|
||||
}
|
||||
};
|
||||
}
|
||||
progress!(
|
||||
"start: {} executions x (1 base + {} perturbations) = {} comparisons planned",
|
||||
scale.executions,
|
||||
scale.floor.perturbations_per_exec.max(10),
|
||||
scale.executions * 11
|
||||
);
|
||||
|
||||
let mut ranks = Vec::with_capacity(scale.executions);
|
||||
let mut edges = Vec::with_capacity(scale.executions);
|
||||
let mut touched = Vec::with_capacity(scale.executions);
|
||||
@@ -316,25 +538,40 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
let mut meta_alt_delta = 0usize;
|
||||
let mut meta_alt_future = 0usize;
|
||||
let mut meta_neutral_unexpl = 0usize;
|
||||
let mut min_perturbations = usize::MAX;
|
||||
|
||||
let mut contract_pass = 0usize;
|
||||
let mut contract_failures: Vec<String> = Vec::new();
|
||||
|
||||
let mut collapse_inputs: Vec<Vec<f64>> = Vec::new();
|
||||
let mut collapse_outputs: Vec<Vec<f64>> = Vec::new();
|
||||
let mut collapse_fps: Vec<Hash> = Vec::new();
|
||||
let mut collapse_rows: Vec<Vec<f64>> = Vec::new();
|
||||
|
||||
let mut mutation_inputs: Vec<ResolutionInput> = Vec::new();
|
||||
let mut generated_rejected = 0usize;
|
||||
let mut perturbation_runs = 0usize;
|
||||
|
||||
let mut contract_rejected = 0usize;
|
||||
|
||||
// Provenance leaves: one per actually-committed execution (base + every
|
||||
// perturbation), so the Merkle root binds to all compared executions.
|
||||
let mut merkle_leaves: Vec<Hash> = Vec::with_capacity(scale.executions);
|
||||
let mut actual_executions = 0usize;
|
||||
let mut worlds_generated = 0usize;
|
||||
let mut programs_generated = 0usize;
|
||||
// 100% reference/runtime comparison covers EVERY execution, base and
|
||||
// perturbation alike — never base executions only.
|
||||
let mut equiv_total = 0usize;
|
||||
|
||||
for i in 0..scale.executions {
|
||||
// Admission: a case is admitted only if its *measured* trace behavior
|
||||
// satisfies its contract. Structurally-flat cases are already rejected
|
||||
// by `generate_accepted_case`; here we additionally reject cases whose
|
||||
// behavior fails the contract, retrying with fresh seeds.
|
||||
if report_progress && i > 0 && i % prog_step == 0 {
|
||||
progress!(
|
||||
"main loop {}/{} ({:.0}%): {} comparisons, {} matched, {} contract-retries",
|
||||
i,
|
||||
scale.executions,
|
||||
100.0 * i as f64 / scale.executions as f64,
|
||||
equiv_total,
|
||||
equiv_matched,
|
||||
contract_rejected
|
||||
);
|
||||
}
|
||||
let mut seed = case_seed(i);
|
||||
for attempt in 0..48 {
|
||||
let last_attempt = attempt == 47;
|
||||
@@ -350,7 +587,6 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
let comp = compressibility(&r.trace.behavior_fingerprint.features);
|
||||
let div = r.trace.context_divergence();
|
||||
|
||||
// Run the 10 perturbations for metamorphic stats + future sensitivity.
|
||||
let base_trace_h = r.trace.canonical_hash();
|
||||
let base_delta_h = r.delta.hash();
|
||||
let base_future_h = r.replay.future_hash;
|
||||
@@ -359,8 +595,13 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
let mut c_alt_future = 0usize;
|
||||
let mut c_neutral = 0usize;
|
||||
let mut c_pert = 0usize;
|
||||
// Capture each perturbation execution so the committed case can run
|
||||
// the full 100% reference/runtime comparison without recomputing the
|
||||
// reference side.
|
||||
let mut pert_execs: Vec<(ResolutionInput, Canonical, Hash)> = Vec::new();
|
||||
for pc in &case.perturbations {
|
||||
let pr = execute(&cfg, &input_with_world(&case, pc.world.clone()));
|
||||
let pinput = input_with_world(&case, pc.world.clone());
|
||||
let pr = execute(&cfg, &pinput);
|
||||
c_pert += 1;
|
||||
let at = pr.trace.canonical_hash() != base_trace_h;
|
||||
let ad = pr.delta.hash() != base_delta_h;
|
||||
@@ -377,6 +618,7 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
if !at && pc.expectation.neutral_explanation.is_none() {
|
||||
c_neutral += 1;
|
||||
}
|
||||
pert_execs.push((pinput, canonical(&pr), pr.replay.hash()));
|
||||
}
|
||||
let future_sensitivity = if c_pert > 0 {
|
||||
c_alt_future as f64 / c_pert as f64
|
||||
@@ -408,12 +650,34 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
}
|
||||
|
||||
// ---- Commit the admitted case ----
|
||||
let r2 = rut.resolve(input.clone());
|
||||
if canonical(&r) == canonical(&r2) {
|
||||
// Every admitted case used exactly one generated world + program.
|
||||
worlds_generated += 1;
|
||||
programs_generated += 1;
|
||||
|
||||
// The runtime under test is an INDEPENDENT implementation. The
|
||||
// equivalence gate compares 100% of executions: the base AND every
|
||||
// perturbation execution, never base executions only.
|
||||
let base_rut = rut.resolve(input.clone());
|
||||
equiv_total += 1;
|
||||
if canonical(&r) == canonical(&base_rut) {
|
||||
equiv_matched += 1;
|
||||
} else if equiv_failures.len() < 16 {
|
||||
equiv_failures.push(format!("case {} reference != runtime_under_test", i));
|
||||
equiv_failures.push(format!("case {} (base) reference != runtime_under_test", i));
|
||||
}
|
||||
merkle_leaves.push(r.replay.hash());
|
||||
for (pinput, pref_canon, pleaf) in &pert_execs {
|
||||
let prut = rut.resolve(pinput.clone());
|
||||
equiv_total += 1;
|
||||
if *pref_canon == canonical(&prut) {
|
||||
equiv_matched += 1;
|
||||
} else if equiv_failures.len() < 16 {
|
||||
equiv_failures
|
||||
.push(format!("case {} (perturbation) reference != runtime_under_test", i));
|
||||
}
|
||||
merkle_leaves.push(*pleaf);
|
||||
}
|
||||
|
||||
actual_executions += 1;
|
||||
|
||||
ranks.push(rank as f64);
|
||||
edges.push(r.trace.causal_edge_count() as f64);
|
||||
@@ -445,6 +709,7 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
meta_alt_future += c_alt_future;
|
||||
meta_neutral_unexpl += c_neutral;
|
||||
perturbation_runs += c_pert;
|
||||
min_perturbations = min_perturbations.min(c_pert);
|
||||
|
||||
if creasons.is_empty() {
|
||||
contract_pass += 1;
|
||||
@@ -452,10 +717,8 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
contract_failures.push(format!("case {} violates {:?}", i, creasons));
|
||||
}
|
||||
|
||||
if collapse_inputs.len() < scale.collapse_samples {
|
||||
collapse_inputs.push(world_input_features(&case.world));
|
||||
collapse_outputs.push(behavior_output_features(&r));
|
||||
collapse_fps.push(r.trace.behavior_fingerprint.hash);
|
||||
if collapse_rows.len() < scale.collapse_samples {
|
||||
collapse_rows.push(trace_feature_row(&r));
|
||||
}
|
||||
if mutation_inputs.len() < 64 {
|
||||
mutation_inputs.push(input.clone());
|
||||
@@ -464,6 +727,15 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
}
|
||||
}
|
||||
|
||||
if min_perturbations == usize::MAX {
|
||||
min_perturbations = 0;
|
||||
}
|
||||
progress!(
|
||||
"main loop complete: {} executions, {} comparisons, {} matched",
|
||||
actual_executions,
|
||||
equiv_total,
|
||||
equiv_matched
|
||||
);
|
||||
let n = scale.executions.max(1) as f64;
|
||||
|
||||
// ---- Trace gates ----
|
||||
@@ -471,7 +743,7 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
let largest_cluster = fp_counts.values().copied().max().unwrap_or(0) as f64 / n;
|
||||
let mut trace_failures = Vec::new();
|
||||
let med_edges = median(&edges);
|
||||
let p95_rank = percentile(&ranks, 0.05); // 95% of executions have rank >= this
|
||||
let p95_rank = percentile(&ranks, 0.05);
|
||||
let med_touched = median(&touched);
|
||||
let p95_touched = percentile(&touched, 0.05);
|
||||
let med_rank = median(&ranks);
|
||||
@@ -505,22 +777,31 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
failures: trace_failures,
|
||||
};
|
||||
|
||||
// ---- Equivalence gates ----
|
||||
if equiv_matched != scale.executions {
|
||||
// ---- Equivalence gates (two independent implementations, 100% coverage) ----
|
||||
let expected_comparisons = actual_executions + perturbation_runs;
|
||||
if equiv_total != expected_comparisons {
|
||||
equiv_failures.push(format!(
|
||||
"compared {} executions but {} ran (must compare 100%, base + perturbations)",
|
||||
equiv_total, expected_comparisons
|
||||
));
|
||||
}
|
||||
if equiv_matched != equiv_total {
|
||||
equiv_failures.push(format!(
|
||||
"{}/{} executions matched (require 100%)",
|
||||
equiv_matched, scale.executions
|
||||
equiv_matched, equiv_total
|
||||
));
|
||||
}
|
||||
let equivalence = EquivalenceGates {
|
||||
total: scale.executions,
|
||||
total: equiv_total,
|
||||
matched: equiv_matched,
|
||||
independent: true,
|
||||
failures: equiv_failures,
|
||||
};
|
||||
|
||||
// ---- Domain participation gates ----
|
||||
progress!("domain participation probes (influence/removal/merge)...");
|
||||
let domain = domain_gates(
|
||||
scale,
|
||||
&scale,
|
||||
&domain_appear,
|
||||
&domain_mutated,
|
||||
&domain_read_any,
|
||||
@@ -556,11 +837,13 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
failures: meta_failures,
|
||||
};
|
||||
|
||||
// ---- Collapse gates ----
|
||||
let corpus = BehaviorCorpus::build(collapse_inputs, collapse_outputs, collapse_fps);
|
||||
// ---- Collapse gates (real trace information) ----
|
||||
progress!("collapse analysis (11 attacks over real trace features)...");
|
||||
let corpus = BehaviorCorpus::build(collapse_rows);
|
||||
let collapse = analyze(&corpus);
|
||||
|
||||
// ---- Mutation gates ----
|
||||
// ---- Mutation gates (killed by named gate) ----
|
||||
progress!("mutation suite ({} mutants, killed by named gate)...", scale.mutants);
|
||||
let mutation = run_suite(scale.mutants, &mutation_inputs);
|
||||
|
||||
// ---- Contract gates ----
|
||||
@@ -578,10 +861,11 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
failures: contract_gate_failures,
|
||||
};
|
||||
|
||||
// ---- Replay gates ----
|
||||
let replay = replay_gates(scale);
|
||||
// ---- Replay gates (persisted corpus) ----
|
||||
progress!("replay corpus verification (committed file)...");
|
||||
let replay = replay_gates(&scale);
|
||||
|
||||
// ---- Coverage gates ----
|
||||
// ---- Coverage gates + merge floor enforcement ----
|
||||
let mut cov_failures = Vec::new();
|
||||
if generated_rejected != 0 {
|
||||
cov_failures.push(format!(
|
||||
@@ -601,8 +885,93 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
failures: cov_failures,
|
||||
};
|
||||
|
||||
// ---- Provenance ----
|
||||
progress!("engine fingerprints + provenance (merkle over {} leaves)...", merkle_leaves.len());
|
||||
let probes = probe_inputs(32);
|
||||
let reference_engine_id = engine_fingerprint(&probes, |inp| execute(&cfg, inp));
|
||||
let rut_engine_id = engine_fingerprint(&probes, native_resolve);
|
||||
let engines_agree = reference_engine_id == rut_engine_id;
|
||||
let root = merkle_root(&merkle_leaves);
|
||||
|
||||
let mut prov_failures = Vec::new();
|
||||
prov_failures.extend(scale.override_violations.iter().cloned());
|
||||
if !engines_agree {
|
||||
prov_failures.push("reference and runtime-under-test engine identities differ".into());
|
||||
}
|
||||
if merkle_leaves.len() != equiv_total {
|
||||
prov_failures.push("merkle leaf count does not match executions compared".into());
|
||||
}
|
||||
// Merge-floor enforcement: the floor cannot be bypassed.
|
||||
if scale.profile == Profile::Merge {
|
||||
if worlds_generated < scale.floor.worlds {
|
||||
prov_failures.push(format!(
|
||||
"worlds generated {} < merge floor {}",
|
||||
worlds_generated, scale.floor.worlds
|
||||
));
|
||||
}
|
||||
if programs_generated < scale.floor.programs {
|
||||
prov_failures.push(format!(
|
||||
"programs generated {} < merge floor {}",
|
||||
programs_generated, scale.floor.programs
|
||||
));
|
||||
}
|
||||
if actual_executions < scale.floor.executions {
|
||||
prov_failures.push(format!(
|
||||
"executions {} < merge floor {}",
|
||||
actual_executions, scale.floor.executions
|
||||
));
|
||||
}
|
||||
if min_perturbations < scale.floor.perturbations_per_exec {
|
||||
prov_failures.push(format!(
|
||||
"min perturbations/exec {} < merge floor {}",
|
||||
min_perturbations, scale.floor.perturbations_per_exec
|
||||
));
|
||||
}
|
||||
// 100% comparison must cover base + all perturbations.
|
||||
let required_comparisons =
|
||||
scale.floor.executions * (1 + scale.floor.perturbations_per_exec);
|
||||
if equiv_total < required_comparisons {
|
||||
prov_failures.push(format!(
|
||||
"comparisons {} < merge floor {} (100% of base + perturbations)",
|
||||
equiv_total, required_comparisons
|
||||
));
|
||||
}
|
||||
if mutation.total < scale.floor.mutants {
|
||||
prov_failures.push(format!(
|
||||
"mutants {} < merge floor {}",
|
||||
mutation.total, scale.floor.mutants
|
||||
));
|
||||
}
|
||||
if replay.total < scale.floor.replay_cases {
|
||||
prov_failures.push(format!(
|
||||
"replay cases {} < merge floor {}",
|
||||
replay.total, scale.floor.replay_cases
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let provenance = Provenance {
|
||||
profile: scale.profile,
|
||||
floor: scale.floor,
|
||||
worlds_generated,
|
||||
programs_generated,
|
||||
actual_executions,
|
||||
min_perturbations_per_exec: min_perturbations,
|
||||
total_comparisons: equiv_total,
|
||||
actual_mutants: mutation.total,
|
||||
replay_total: replay.total,
|
||||
reference_engine_id,
|
||||
rut_engine_id,
|
||||
engines_agree,
|
||||
execution_merkle_root: root,
|
||||
merkle_leaf_count: merkle_leaves.len(),
|
||||
collapse_feature_width: FEATURE_W,
|
||||
failures: prov_failures,
|
||||
};
|
||||
|
||||
CiResults {
|
||||
scale,
|
||||
provenance,
|
||||
trace,
|
||||
equivalence,
|
||||
domain,
|
||||
@@ -616,7 +985,7 @@ pub fn run_all(scale: Scale) -> CiResults {
|
||||
}
|
||||
|
||||
fn domain_gates(
|
||||
scale: Scale,
|
||||
scale: &Scale,
|
||||
appear: &[u64; NUM_DOMAINS],
|
||||
mutated: &[u64; NUM_DOMAINS],
|
||||
read_any: &[bool; NUM_DOMAINS],
|
||||
@@ -632,7 +1001,6 @@ fn domain_gates(
|
||||
mutated_f[d] = mutated[d] as f64 / n;
|
||||
}
|
||||
|
||||
// Influence probe: mask each domain and see how often the trace changes.
|
||||
let mut influences = [0.0; NUM_DOMAINS];
|
||||
let probe_cases = scale.domain_probe_cases.max(1);
|
||||
for d in 0..NUM_DOMAINS {
|
||||
@@ -652,8 +1020,6 @@ fn domain_gates(
|
||||
influences[d] = changed as f64 / probe_cases as f64;
|
||||
}
|
||||
|
||||
// Removal diversity probe: vary only domain d across K worlds; compare the
|
||||
// count of distinct behaviors with the domain present vs masked.
|
||||
let mut removal_loss = [0.0; NUM_DOMAINS];
|
||||
let k = scale.domain_probe_variations.max(8);
|
||||
for d in 0..NUM_DOMAINS {
|
||||
@@ -667,7 +1033,7 @@ fn domain_gates(
|
||||
let mcfg = masked_config(d);
|
||||
for v in 0..k {
|
||||
let mut w = base_world.clone();
|
||||
for l in 0..LANES {
|
||||
for l in 0..world_model::LANES {
|
||||
w.domains[d].observed[l] = (v as i64 + 1).wrapping_mul(7919 + l as i64);
|
||||
}
|
||||
w.domains[d].hidden[0] = (v as i64).wrapping_mul(104729);
|
||||
@@ -680,7 +1046,6 @@ fn domain_gates(
|
||||
removal_loss[d] = (1.0 - dm / df).clamp(0.0, 1.0);
|
||||
}
|
||||
|
||||
// Merge probe: alias domain b := domain a, see how often behavior changes.
|
||||
let mut min_merge_loss = 1.0f64;
|
||||
let merge_cases = (scale.domain_probe_cases / 2).max(20);
|
||||
for a in 0..NUM_DOMAINS {
|
||||
@@ -749,21 +1114,29 @@ fn domain_gates(
|
||||
}
|
||||
}
|
||||
|
||||
fn replay_gates(scale: Scale) -> ReplayGates {
|
||||
let corpus = replay_corpus::build_corpus(scale.replay_cases, 0x5EED);
|
||||
let report = replay_corpus::verify_corpus(&corpus);
|
||||
fn replay_gates(scale: &Scale) -> ReplayGates {
|
||||
let report = replay_corpus::verify_persisted_corpus();
|
||||
let mut failures = Vec::new();
|
||||
if !report.loaded_from_disk {
|
||||
failures.push(format!(
|
||||
"committed replay corpus not found at {}",
|
||||
replay_corpus::corpus_path().display()
|
||||
));
|
||||
}
|
||||
if !report.drift.is_empty() {
|
||||
failures.push(format!("{} replay cases drifted", report.drift.len()));
|
||||
}
|
||||
// Minimum corpus size is a merge-blocking gate only at full scale.
|
||||
if matches!(std::env::var("MAGICKA_SCALE").as_deref(), Ok("full")) && report.total < 10_000 {
|
||||
failures.push(format!("replay corpus {} < 10000", report.total));
|
||||
if scale.profile == Profile::Merge && report.total < scale.floor.replay_cases {
|
||||
failures.push(format!(
|
||||
"replay corpus {} < merge floor {}",
|
||||
report.total, scale.floor.replay_cases
|
||||
));
|
||||
}
|
||||
ReplayGates {
|
||||
total: report.total,
|
||||
deterministic: report.deterministic,
|
||||
drift: report.drift.len(),
|
||||
loaded_from_disk: report.loaded_from_disk,
|
||||
failures,
|
||||
}
|
||||
}
|
||||
@@ -779,6 +1152,69 @@ mod tests {
|
||||
assert!(f.is_empty(), "{name} failed: {:?}", f);
|
||||
}
|
||||
assert!(r.mutation.passed(), "mutants survived: {:?}", r.mutation.survivors);
|
||||
assert!(r.provenance.engines_agree, "engine identities disagree");
|
||||
assert!(r.passed());
|
||||
}
|
||||
|
||||
/// Negative control: a smoke-scale run that claims the merge profile must be
|
||||
/// REJECTED, because the merge floors (1M executions, 10k replay, etc.) are
|
||||
/// not met. Proves a default/small profile cannot stand in for acceptance.
|
||||
#[test]
|
||||
fn merge_profile_at_smoke_scale_is_rejected() {
|
||||
let mut s = Scale::tiny();
|
||||
s.profile = Profile::Merge;
|
||||
s.floor = ScaleFloor::MERGE;
|
||||
let r = run_all(s);
|
||||
assert!(
|
||||
!r.passed(),
|
||||
"a smoke-scale run was accepted as a merge run (floors not enforced)"
|
||||
);
|
||||
assert!(
|
||||
!r.provenance.failures.is_empty(),
|
||||
"merge floors were not flagged in provenance"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_floor_cannot_be_lowered_by_override() {
|
||||
// Simulate an attempt to lower the merge execution floor.
|
||||
let mut s = Scale::merge();
|
||||
s.apply_override("MAGICKA_EXECUTIONS", 10, s.floor.executions, |s, x| s.executions = x);
|
||||
assert_eq!(s.executions, s.floor.executions, "floor was lowered by override");
|
||||
assert!(!s.override_violations.is_empty(), "violation not recorded");
|
||||
}
|
||||
|
||||
/// Negative control: a decorative (never-appearing, never-mutated) domain
|
||||
/// must be rejected by the domain-participation gate. Proves the gate can
|
||||
/// fail and discriminates a decorative domain from a live one.
|
||||
#[test]
|
||||
fn decorative_domain_is_rejected() {
|
||||
let scale = Scale::tiny();
|
||||
let n = 100.0;
|
||||
let mut appear = [100u64; NUM_DOMAINS];
|
||||
let mut mutated = [100u64; NUM_DOMAINS];
|
||||
appear[3] = 0; // domain 3 is decorative
|
||||
mutated[3] = 0;
|
||||
let read_any = [true; NUM_DOMAINS];
|
||||
let write_any = [true; NUM_DOMAINS];
|
||||
let gates = domain_gates(&scale, &appear, &mutated, &read_any, &write_any, n);
|
||||
assert!(
|
||||
!gates.failures.is_empty(),
|
||||
"domain gate failed to reject a decorative domain"
|
||||
);
|
||||
assert!(
|
||||
gates.failures.iter().any(|f| f.contains("domain 3")),
|
||||
"decorative domain 3 not flagged: {:?}",
|
||||
gates.failures
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merkle_root_binds_to_leaves() {
|
||||
let a = merkle_root(&[Hash(1), Hash(2), Hash(3)]);
|
||||
let b = merkle_root(&[Hash(1), Hash(2), Hash(3)]);
|
||||
let c = merkle_root(&[Hash(1), Hash(2), Hash(4)]);
|
||||
assert_eq!(a, b);
|
||||
assert_ne!(a, c);
|
||||
}
|
||||
}
|
||||
|
||||
+307
-21
@@ -1,9 +1,10 @@
|
||||
//! The `ci` binary: runs the full adversarial framework against the reference
|
||||
//! and runtime-under-test, writes the eight required reports (JSON + markdown),
|
||||
//! and exits nonzero if any gate fails.
|
||||
//! and the independently-implemented runtime under test, writes the required
|
||||
//! reports (JSON + markdown) including a provenance report binding the numbers
|
||||
//! to executed work, and exits nonzero if any gate fails.
|
||||
|
||||
use ci_reports::json::Json;
|
||||
use ci_reports::{run_all, CiResults, Scale};
|
||||
use ci_reports::{run_all, CiResults, Profile, Scale};
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
@@ -18,6 +19,9 @@ fn fails(v: &[String]) -> Json {
|
||||
fn pass_field(v: &[String]) -> Json {
|
||||
Json::Bool(v.is_empty())
|
||||
}
|
||||
fn hex(h: world_model::Hash) -> Json {
|
||||
Json::s(format!("{:016x}", h.0))
|
||||
}
|
||||
|
||||
fn write_report(dir: &Path, name: &str, j: &Json) {
|
||||
let path = dir.join(format!("{name}.json"));
|
||||
@@ -74,7 +78,7 @@ fn build_reports(dir: &Path, r: &CiResults) {
|
||||
.map(|rep| {
|
||||
Json::Obj(vec![
|
||||
("attack".into(), Json::s(rep.attack.clone())),
|
||||
("predicts".into(), Json::Num(rep.predicts)),
|
||||
("reconstructs".into(), Json::Num(rep.predicts)),
|
||||
("info_loss".into(), Json::Num(rep.info_loss)),
|
||||
("detail".into(), Json::s(rep.detail.clone())),
|
||||
])
|
||||
@@ -85,6 +89,7 @@ fn build_reports(dir: &Path, r: &CiResults) {
|
||||
"compression_resistance_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.collapse.failures)),
|
||||
("measures".into(), Json::s("real serialized trace structure")),
|
||||
("best_1factor".into(), Json::Num(r.collapse.best_1factor)),
|
||||
("best_2factor".into(), Json::Num(r.collapse.best_2factor)),
|
||||
("best_4factor".into(), Json::Num(r.collapse.best_4factor)),
|
||||
@@ -116,10 +121,10 @@ fn build_reports(dir: &Path, r: &CiResults) {
|
||||
.mutation
|
||||
.survivors
|
||||
.iter()
|
||||
.map(|(id, name)| {
|
||||
.map(|(id, reason)| {
|
||||
Json::Obj(vec![
|
||||
("id".into(), Json::Int(*id as i64)),
|
||||
("name".into(), Json::s(name.clone())),
|
||||
("reason".into(), Json::s(reason.clone())),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
@@ -128,6 +133,7 @@ fn build_reports(dir: &Path, r: &CiResults) {
|
||||
"mutation_survivor_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), Json::Bool(r.mutation.passed())),
|
||||
("killed_by".into(), Json::s("named acceptance gate")),
|
||||
("total_mutants".into(), Json::Int(r.mutation.total as i64)),
|
||||
("killed".into(), Json::Int(r.mutation.killed as i64)),
|
||||
("survivors".into(), Json::Arr(survivors)),
|
||||
@@ -140,6 +146,10 @@ fn build_reports(dir: &Path, r: &CiResults) {
|
||||
"runtime_equivalence_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.equivalence.failures)),
|
||||
("independent_implementations".into(), Json::Bool(r.equivalence.independent)),
|
||||
("reference_engine_id".into(), hex(r.provenance.reference_engine_id)),
|
||||
("rut_engine_id".into(), hex(r.provenance.rut_engine_id)),
|
||||
("engines_agree".into(), Json::Bool(r.provenance.engines_agree)),
|
||||
("total".into(), Json::Int(r.equivalence.total as i64)),
|
||||
("matched".into(), Json::Int(r.equivalence.matched as i64)),
|
||||
("failures".into(), fails(&r.equivalence.failures)),
|
||||
@@ -152,6 +162,7 @@ fn build_reports(dir: &Path, r: &CiResults) {
|
||||
"replay_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.replay.failures)),
|
||||
("loaded_from_committed_corpus".into(), Json::Bool(r.replay.loaded_from_disk)),
|
||||
("total".into(), Json::Int(r.replay.total as i64)),
|
||||
("deterministic".into(), Json::Int(r.replay.deterministic as i64)),
|
||||
("drift".into(), Json::Int(r.replay.drift as i64)),
|
||||
@@ -175,6 +186,230 @@ fn build_reports(dir: &Path, r: &CiResults) {
|
||||
("failures".into(), fails(&r.coverage.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 9. provenance_report — binds the run to executed work.
|
||||
let p = &r.provenance;
|
||||
write_report(
|
||||
dir,
|
||||
"provenance_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&p.failures)),
|
||||
("profile".into(), Json::s(p.profile.name())),
|
||||
(
|
||||
"merge_floor".into(),
|
||||
Json::Obj(vec![
|
||||
("worlds".into(), Json::Int(p.floor.worlds as i64)),
|
||||
("programs".into(), Json::Int(p.floor.programs as i64)),
|
||||
("executions".into(), Json::Int(p.floor.executions as i64)),
|
||||
("perturbations_per_exec".into(), Json::Int(p.floor.perturbations_per_exec as i64)),
|
||||
("mutants".into(), Json::Int(p.floor.mutants as i64)),
|
||||
("replay_cases".into(), Json::Int(p.floor.replay_cases as i64)),
|
||||
]),
|
||||
),
|
||||
("worlds_generated".into(), Json::Int(p.worlds_generated as i64)),
|
||||
("programs_generated".into(), Json::Int(p.programs_generated as i64)),
|
||||
("actual_executions".into(), Json::Int(p.actual_executions as i64)),
|
||||
("min_perturbations_per_exec".into(), Json::Int(p.min_perturbations_per_exec as i64)),
|
||||
("total_comparisons".into(), Json::Int(p.total_comparisons as i64)),
|
||||
("actual_mutants".into(), Json::Int(p.actual_mutants as i64)),
|
||||
("replay_total".into(), Json::Int(p.replay_total as i64)),
|
||||
("reference_engine_id".into(), hex(p.reference_engine_id)),
|
||||
("rut_engine_id".into(), hex(p.rut_engine_id)),
|
||||
("engines_agree".into(), Json::Bool(p.engines_agree)),
|
||||
("execution_merkle_root".into(), hex(p.execution_merkle_root)),
|
||||
("merkle_leaf_count".into(), Json::Int(p.merkle_leaf_count as i64)),
|
||||
("collapse_feature_width".into(), Json::Int(p.collapse_feature_width as i64)),
|
||||
("failures".into(), fails(&p.failures)),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
/// The reports the spec requires every CI run to produce. A missing or empty
|
||||
/// artifact is itself an acceptance failure (compliance rule 4).
|
||||
const REQUIRED_REPORTS: [&str; 9] = [
|
||||
"domain_participation_report",
|
||||
"causal_rank_report",
|
||||
"compression_resistance_report",
|
||||
"metamorphic_response_report",
|
||||
"mutation_survivor_report",
|
||||
"runtime_equivalence_report",
|
||||
"replay_report",
|
||||
"coverage_report",
|
||||
"provenance_report",
|
||||
];
|
||||
|
||||
fn report_present(dir: &Path, name: &str) -> bool {
|
||||
fs::metadata(dir.join(format!("{name}.json")))
|
||||
.map(|m| m.len() > 0)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// One acceptance obligation: a measured artifact, a provenance chain, a
|
||||
/// merge-blocking enforcement point, and a failure if the artifact/provenance is
|
||||
/// absent. This is the compliance model made machine-checkable.
|
||||
struct Obligation {
|
||||
requirement: &'static str,
|
||||
artifact: &'static str,
|
||||
floor: i64,
|
||||
actual: i64,
|
||||
gate_pass: bool,
|
||||
}
|
||||
|
||||
fn obligations(r: &CiResults) -> Vec<Obligation> {
|
||||
let p = &r.provenance;
|
||||
let merge = p.profile == Profile::Merge;
|
||||
let f = |v: usize| v as i64;
|
||||
let floor = |v: usize| if merge { v as i64 } else { 0 };
|
||||
vec![
|
||||
Obligation {
|
||||
requirement: "generated worlds >= 50,000",
|
||||
artifact: "provenance_report",
|
||||
floor: floor(p.floor.worlds),
|
||||
actual: f(p.worlds_generated),
|
||||
gate_pass: !merge || p.worlds_generated >= p.floor.worlds,
|
||||
},
|
||||
Obligation {
|
||||
requirement: "generated programs >= 250,000",
|
||||
artifact: "provenance_report",
|
||||
floor: floor(p.floor.programs),
|
||||
actual: f(p.programs_generated),
|
||||
gate_pass: !merge || p.programs_generated >= p.floor.programs,
|
||||
},
|
||||
Obligation {
|
||||
requirement: "executions >= 1,000,000",
|
||||
artifact: "provenance_report",
|
||||
floor: floor(p.floor.executions),
|
||||
actual: f(p.actual_executions),
|
||||
gate_pass: !merge || p.actual_executions >= p.floor.executions,
|
||||
},
|
||||
Obligation {
|
||||
requirement: "perturbations per execution >= 10",
|
||||
artifact: "metamorphic_response_report",
|
||||
floor: floor(p.floor.perturbations_per_exec),
|
||||
actual: f(p.min_perturbations_per_exec),
|
||||
gate_pass: !merge || p.min_perturbations_per_exec >= p.floor.perturbations_per_exec,
|
||||
},
|
||||
Obligation {
|
||||
requirement: "reference/runtime comparison = 100% of executions",
|
||||
artifact: "runtime_equivalence_report",
|
||||
floor: floor(p.floor.executions * (1 + p.floor.perturbations_per_exec)),
|
||||
actual: f(p.total_comparisons),
|
||||
gate_pass: r.equivalence.failures.is_empty(),
|
||||
},
|
||||
Obligation {
|
||||
requirement: "semantic mutants >= 500, 0 survivors, killed by named gate",
|
||||
artifact: "mutation_survivor_report",
|
||||
floor: floor(p.floor.mutants),
|
||||
actual: f(r.mutation.total),
|
||||
gate_pass: r.mutation.passed() && (!merge || r.mutation.total >= p.floor.mutants),
|
||||
},
|
||||
Obligation {
|
||||
requirement: "replay corpus >= 10,000, persisted, 0 drift",
|
||||
artifact: "replay_report",
|
||||
floor: floor(p.floor.replay_cases),
|
||||
actual: f(r.replay.total),
|
||||
gate_pass: r.replay.failures.is_empty() && r.replay.loaded_from_disk,
|
||||
},
|
||||
Obligation {
|
||||
requirement: "collapse attacks fail to simplify (real trace info)",
|
||||
artifact: "compression_resistance_report",
|
||||
floor: 0,
|
||||
actual: 0,
|
||||
gate_pass: r.collapse.failures.is_empty(),
|
||||
},
|
||||
Obligation {
|
||||
requirement: "trace gates (causal edges/rank/touched/fingerprints)",
|
||||
artifact: "causal_rank_report",
|
||||
floor: 0,
|
||||
actual: 0,
|
||||
gate_pass: r.trace.failures.is_empty(),
|
||||
},
|
||||
Obligation {
|
||||
requirement: "domain participation (no decorative/read-only/write-only)",
|
||||
artifact: "domain_participation_report",
|
||||
floor: 0,
|
||||
actual: 0,
|
||||
gate_pass: r.domain.failures.is_empty(),
|
||||
},
|
||||
Obligation {
|
||||
requirement: "metamorphic response thresholds",
|
||||
artifact: "metamorphic_response_report",
|
||||
floor: 0,
|
||||
actual: 0,
|
||||
gate_pass: r.metamorphic.failures.is_empty(),
|
||||
},
|
||||
Obligation {
|
||||
requirement: "every admitted case satisfies its contract",
|
||||
artifact: "coverage_report",
|
||||
floor: 0,
|
||||
actual: 0,
|
||||
gate_pass: r.contract.failures.is_empty() && r.coverage.failures.is_empty(),
|
||||
},
|
||||
Obligation {
|
||||
requirement: "provenance binds reports to executed work",
|
||||
artifact: "provenance_report",
|
||||
floor: 0,
|
||||
actual: 0,
|
||||
gate_pass: r.provenance.failures.is_empty(),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// Write the compliance report and return whether the compliance model holds:
|
||||
/// every obligation's artifact is present and every obligation passes.
|
||||
fn build_compliance_report(dir: &Path, r: &CiResults) -> bool {
|
||||
let items = obligations(r);
|
||||
let mut all_ok = true;
|
||||
let mut json_items = Vec::new();
|
||||
for ob in &items {
|
||||
let present = report_present(dir, ob.artifact);
|
||||
let pass = present && ob.gate_pass;
|
||||
if !pass {
|
||||
all_ok = false;
|
||||
}
|
||||
json_items.push(Json::Obj(vec![
|
||||
("requirement".into(), Json::s(ob.requirement)),
|
||||
("measured_artifact".into(), Json::s(format!("{}.json", ob.artifact))),
|
||||
("artifact_present".into(), Json::Bool(present)),
|
||||
("provenance".into(), Json::s("provenance_report.json (merkle root + engine ids)")),
|
||||
("merge_blocking".into(), Json::Bool(true)),
|
||||
("floor".into(), Json::Int(ob.floor)),
|
||||
("actual".into(), Json::Int(ob.actual)),
|
||||
("pass".into(), Json::Bool(pass)),
|
||||
]));
|
||||
}
|
||||
// Every required report must exist and be non-empty.
|
||||
let mut missing = Vec::new();
|
||||
for rep in REQUIRED_REPORTS {
|
||||
if !report_present(dir, rep) {
|
||||
missing.push(rep.to_string());
|
||||
all_ok = false;
|
||||
}
|
||||
}
|
||||
write_report(
|
||||
dir,
|
||||
"compliance_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), Json::Bool(all_ok)),
|
||||
("profile".into(), Json::s(r.provenance.profile.name())),
|
||||
(
|
||||
"merge_blocking_run".into(),
|
||||
Json::Bool(r.provenance.profile == Profile::Merge),
|
||||
),
|
||||
(
|
||||
"note".into(),
|
||||
Json::s(if r.provenance.profile == Profile::Merge {
|
||||
"merge profile: floors enforced, all obligations acceptance-blocking"
|
||||
} else {
|
||||
"advisory profile: NOT acceptance; floors not enforced (fast/tiny)"
|
||||
}),
|
||||
),
|
||||
("required_reports".into(), Json::Arr(REQUIRED_REPORTS.iter().map(|s| Json::s(*s)).collect())),
|
||||
("missing_reports".into(), Json::Arr(missing.iter().map(|s| Json::s(s.clone())).collect())),
|
||||
("obligations".into(), Json::Arr(json_items)),
|
||||
]),
|
||||
);
|
||||
all_ok
|
||||
}
|
||||
|
||||
fn status(v: bool) -> &'static str {
|
||||
@@ -186,22 +421,38 @@ fn status(v: bool) -> &'static str {
|
||||
}
|
||||
|
||||
fn write_markdown(dir: &Path, r: &CiResults) {
|
||||
let p = &r.provenance;
|
||||
let mut s = String::new();
|
||||
s.push_str("# Magicka VM — Phase 0/1 CI Report\n\n");
|
||||
s.push_str(&format!(
|
||||
"Overall: **{}**\n\n",
|
||||
status(r.passed())
|
||||
));
|
||||
s.push_str(&format!("Overall: **{}**\n\n", status(r.passed())));
|
||||
s.push_str(&format!("Profile: **{}**", p.profile.name()));
|
||||
if p.profile == Profile::Fast {
|
||||
s.push_str(" (representative slice — NOT merge-blocking)");
|
||||
}
|
||||
s.push_str("\n\n");
|
||||
s.push_str(&format!(
|
||||
"Scale: executions={}, mutants={}, replay={}, collapse_samples={}\n\n",
|
||||
r.scale.executions, r.scale.mutants, r.scale.replay_cases, r.scale.collapse_samples
|
||||
));
|
||||
s.push_str("## Provenance\n\n");
|
||||
s.push_str(&format!("- Execution Merkle root: `{:016x}` over {} leaves\n", p.execution_merkle_root.0, p.merkle_leaf_count));
|
||||
s.push_str(&format!(
|
||||
"- Reference engine id: `{:016x}`; runtime-under-test engine id: `{:016x}`; agree: **{}**\n",
|
||||
p.reference_engine_id.0, p.rut_engine_id.0, p.engines_agree
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"- Actual executions: {} (merge floor {}), min perturbations/exec: {} (floor {})\n",
|
||||
p.actual_executions, p.floor.executions, p.min_perturbations_per_exec, p.floor.perturbations_per_exec
|
||||
));
|
||||
s.push_str(&format!("- Collapse measures real trace structure ({} features/trace)\n\n", p.collapse_feature_width));
|
||||
|
||||
s.push_str("| Report | Status | Key metrics |\n|---|---|---|\n");
|
||||
s.push_str(&format!(
|
||||
"| runtime_equivalence | {} | {}/{} matched |\n",
|
||||
"| runtime_equivalence | {} | {}/{} matched, independent impls, engines agree={} |\n",
|
||||
status(r.equivalence.failures.is_empty()),
|
||||
r.equivalence.matched,
|
||||
r.equivalence.total
|
||||
r.equivalence.total,
|
||||
p.engines_agree
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| causal_rank/trace | {} | rank med={} p95={}, edges med={}, touched med={} |\n",
|
||||
@@ -234,7 +485,7 @@ fn write_markdown(dir: &Path, r: &CiResults) {
|
||||
r.collapse.min_info_loss
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| mutation_survivor | {} | killed {}/{} |\n",
|
||||
"| mutation_survivor | {} | killed {}/{} by named gate |\n",
|
||||
status(r.mutation.passed()),
|
||||
r.mutation.killed,
|
||||
r.mutation.total
|
||||
@@ -246,7 +497,7 @@ fn write_markdown(dir: &Path, r: &CiResults) {
|
||||
r.contract.total
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| replay | {} | {}/{} deterministic, drift={} |\n",
|
||||
"| replay | {} | {}/{} deterministic (committed corpus), drift={} |\n",
|
||||
status(r.replay.failures.is_empty()),
|
||||
r.replay.deterministic,
|
||||
r.replay.total,
|
||||
@@ -259,6 +510,11 @@ fn write_markdown(dir: &Path, r: &CiResults) {
|
||||
r.coverage.perturbations,
|
||||
r.coverage.generated_rejected
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| provenance | {} | merkle over {} leaves, floor enforced |\n",
|
||||
status(r.provenance.failures.is_empty()),
|
||||
r.provenance.merkle_leaf_count
|
||||
));
|
||||
|
||||
s.push_str("\n## Failures\n\n");
|
||||
let mut any = false;
|
||||
@@ -268,9 +524,9 @@ fn write_markdown(dir: &Path, r: &CiResults) {
|
||||
s.push_str(&format!("- **{}**: {}\n", name, msg));
|
||||
}
|
||||
}
|
||||
for (id, name) in &r.mutation.survivors {
|
||||
for (id, reason) in &r.mutation.survivors {
|
||||
any = true;
|
||||
s.push_str(&format!("- **mutation_survivor**: mutant {} ({}) survived\n", id, name));
|
||||
s.push_str(&format!("- **mutation_survivor**: mutant {} survived ({})\n", id, reason));
|
||||
}
|
||||
if !any {
|
||||
s.push_str("None. The fake universe failed to collapse. ✅\n");
|
||||
@@ -287,15 +543,20 @@ fn main() {
|
||||
fs::create_dir_all(dir).expect("create out dir");
|
||||
|
||||
eprintln!(
|
||||
"running CI: executions={} mutants={} replay={} collapse_samples={}",
|
||||
scale.executions, scale.mutants, scale.replay_cases, scale.collapse_samples
|
||||
"running CI: profile={} executions={} mutants={} replay={} collapse_samples={}",
|
||||
scale.profile.name(), scale.executions, scale.mutants, scale.replay_cases, scale.collapse_samples
|
||||
);
|
||||
for v in &scale.override_violations {
|
||||
eprintln!(" override rejected: {v}");
|
||||
}
|
||||
let merge = scale.profile == Profile::Merge;
|
||||
let start = Instant::now();
|
||||
let results = run_all(scale);
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
build_reports(dir, &results);
|
||||
write_markdown(dir, &results);
|
||||
let compliance_ok = build_compliance_report(dir, &results);
|
||||
|
||||
println!("\n=== Magicka VM CI ({:?}) ===", elapsed);
|
||||
for (name, f) in results.all_failures() {
|
||||
@@ -313,13 +574,38 @@ fn main() {
|
||||
format!(", {} survivors", results.mutation.survivors.len())
|
||||
}
|
||||
);
|
||||
println!(" {:<24} {}", "compliance", status(compliance_ok));
|
||||
println!(
|
||||
" provenance: merkle={:016x} over {} leaves, engines_agree={}, comparisons={}",
|
||||
results.provenance.execution_merkle_root.0,
|
||||
results.provenance.merkle_leaf_count,
|
||||
results.provenance.engines_agree,
|
||||
results.provenance.total_comparisons,
|
||||
);
|
||||
println!("reports written to {}/", out_dir);
|
||||
|
||||
if results.passed() {
|
||||
println!("\nOVERALL: PASS — the adversarial framework could not collapse the universe.");
|
||||
let gates_pass = results.passed() && compliance_ok;
|
||||
|
||||
if !merge {
|
||||
// The fast/tiny profiles are advisory only — they may never stand in for
|
||||
// the merge-blocking acceptance run (compliance: no default-profile
|
||||
// substitution). Report status but make clear this is not acceptance.
|
||||
println!(
|
||||
"\nADVISORY ({} profile): {} — NOT a merge-blocking acceptance run. \
|
||||
Acceptance requires MAGICKA_PROFILE=merge (full gates).",
|
||||
results.provenance.profile.name(),
|
||||
status(gates_pass)
|
||||
);
|
||||
// A failing advisory run still fails the PR check; a passing one is green
|
||||
// but explicitly non-acceptance.
|
||||
std::process::exit(if gates_pass { 0 } else { 1 });
|
||||
}
|
||||
|
||||
if gates_pass {
|
||||
println!("\nACCEPTANCE: PASS — full merge gates satisfied with persisted, independently reproducible evidence.");
|
||||
std::process::exit(0);
|
||||
} else {
|
||||
println!("\nOVERALL: FAIL");
|
||||
println!("\nACCEPTANCE: FAIL");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user