changes claude never committed
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "ci_reports"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
world_model = { path = "../world_model" }
|
||||
rune_ir = { path = "../rune_ir" }
|
||||
trace_model = { path = "../trace_model" }
|
||||
generators = { path = "../generators" }
|
||||
reference_runtime = { path = "../reference_runtime" }
|
||||
runtime_under_test = { path = "../runtime_under_test" }
|
||||
collapse_analysis = { path = "../collapse_analysis" }
|
||||
semantic_mutation = { path = "../semantic_mutation" }
|
||||
replay_corpus = { path = "../replay_corpus" }
|
||||
|
||||
[lib]
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "ci"
|
||||
path = "src/main.rs"
|
||||
@@ -0,0 +1,94 @@
|
||||
//! Minimal hand-rolled JSON value + pretty printer (no external crates).
|
||||
|
||||
pub enum Json {
|
||||
Null,
|
||||
Bool(bool),
|
||||
Int(i64),
|
||||
Num(f64),
|
||||
Str(String),
|
||||
Arr(Vec<Json>),
|
||||
Obj(Vec<(String, Json)>),
|
||||
}
|
||||
|
||||
impl Json {
|
||||
pub fn s(v: impl Into<String>) -> Json {
|
||||
Json::Str(v.into())
|
||||
}
|
||||
pub fn to_pretty(&self) -> String {
|
||||
let mut out = String::new();
|
||||
self.write(&mut out, 0);
|
||||
out.push('\n');
|
||||
out
|
||||
}
|
||||
|
||||
fn write(&self, out: &mut String, indent: usize) {
|
||||
match self {
|
||||
Json::Null => out.push_str("null"),
|
||||
Json::Bool(b) => out.push_str(if *b { "true" } else { "false" }),
|
||||
Json::Int(i) => out.push_str(&i.to_string()),
|
||||
Json::Num(f) => {
|
||||
if f.is_finite() {
|
||||
out.push_str(&format!("{:.6}", f));
|
||||
} else {
|
||||
out.push_str("null");
|
||||
}
|
||||
}
|
||||
Json::Str(s) => {
|
||||
out.push('"');
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'"' => out.push_str("\\\""),
|
||||
'\\' => out.push_str("\\\\"),
|
||||
'\n' => out.push_str("\\n"),
|
||||
'\t' => out.push_str("\\t"),
|
||||
_ => out.push(c),
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
Json::Arr(items) => {
|
||||
if items.is_empty() {
|
||||
out.push_str("[]");
|
||||
return;
|
||||
}
|
||||
out.push_str("[\n");
|
||||
for (i, it) in items.iter().enumerate() {
|
||||
pad(out, indent + 1);
|
||||
it.write(out, indent + 1);
|
||||
if i + 1 < items.len() {
|
||||
out.push(',');
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
pad(out, indent);
|
||||
out.push(']');
|
||||
}
|
||||
Json::Obj(fields) => {
|
||||
if fields.is_empty() {
|
||||
out.push_str("{}");
|
||||
return;
|
||||
}
|
||||
out.push_str("{\n");
|
||||
for (i, (k, v)) in fields.iter().enumerate() {
|
||||
pad(out, indent + 1);
|
||||
out.push('"');
|
||||
out.push_str(k);
|
||||
out.push_str("\": ");
|
||||
v.write(out, indent + 1);
|
||||
if i + 1 < fields.len() {
|
||||
out.push(',');
|
||||
}
|
||||
out.push('\n');
|
||||
}
|
||||
pad(out, indent);
|
||||
out.push('}');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn pad(out: &mut String, indent: usize) {
|
||||
for _ in 0..indent {
|
||||
out.push_str(" ");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,784 @@
|
||||
//! `ci_reports` — the CI orchestrator. Runs the whole adversarial framework
|
||||
//! against the reference runtime and the runtime under test, evaluates every
|
||||
//! gate, and emits machine-readable JSON plus a human-readable markdown
|
||||
//! summary. Merge is blocked unless all reports pass.
|
||||
|
||||
pub mod json;
|
||||
|
||||
use collapse_analysis::{analyze, BehaviorCorpus, CollapseSummary};
|
||||
use generators::{generate_accepted_case, generated_gate_failures, GeneratedCase};
|
||||
use reference_runtime::{canonical, execute, EngineConfig, ResolutionInput, ResolutionResult, Runtime};
|
||||
use runtime_under_test::RuntimeUnderTest;
|
||||
use semantic_mutation::{run_suite, MutationOutcome};
|
||||
use std::collections::HashMap;
|
||||
use world_model::{Hash, WorldSnapshot, HIDDEN_LANES, LANES, NUM_DOMAINS};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scale configuration.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Scale {
|
||||
pub executions: usize,
|
||||
pub mutants: usize,
|
||||
pub replay_cases: usize,
|
||||
pub collapse_samples: usize,
|
||||
pub domain_probe_cases: usize,
|
||||
pub domain_probe_variations: usize,
|
||||
}
|
||||
|
||||
impl Scale {
|
||||
/// The full, merge-blocking gates mandated by the spec.
|
||||
pub fn full() -> Self {
|
||||
Scale {
|
||||
executions: 1_000_000,
|
||||
mutants: 600,
|
||||
replay_cases: 10_000,
|
||||
collapse_samples: 5_000,
|
||||
domain_probe_cases: 2_000,
|
||||
domain_probe_variations: 64,
|
||||
}
|
||||
}
|
||||
/// Fast CI (~10% spirit): small but exercises every gate.
|
||||
pub fn fast() -> Self {
|
||||
Scale {
|
||||
executions: 600,
|
||||
mutants: 520,
|
||||
replay_cases: 600,
|
||||
collapse_samples: 600,
|
||||
domain_probe_cases: 200,
|
||||
domain_probe_variations: 48,
|
||||
}
|
||||
}
|
||||
pub fn tiny() -> Self {
|
||||
Scale {
|
||||
executions: 120,
|
||||
mutants: 520,
|
||||
replay_cases: 120,
|
||||
collapse_samples: 120,
|
||||
domain_probe_cases: 60,
|
||||
domain_probe_variations: 32,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_env() -> Self {
|
||||
let mut s = match std::env::var("MAGICKA_SCALE").as_deref() {
|
||||
Ok("full") => Scale::full(),
|
||||
Ok("tiny") => Scale::tiny(),
|
||||
_ => Scale::fast(),
|
||||
};
|
||||
if let Some(v) = env_usize("MAGICKA_EXECUTIONS") {
|
||||
s.executions = v;
|
||||
}
|
||||
if let Some(v) = env_usize("MAGICKA_MUTANTS") {
|
||||
s.mutants = v;
|
||||
}
|
||||
if let Some(v) = env_usize("MAGICKA_REPLAY") {
|
||||
s.replay_cases = v;
|
||||
}
|
||||
if let Some(v) = env_usize("MAGICKA_COLLAPSE") {
|
||||
s.collapse_samples = v;
|
||||
}
|
||||
s
|
||||
}
|
||||
}
|
||||
|
||||
fn env_usize(key: &str) -> Option<usize> {
|
||||
std::env::var(key).ok().and_then(|v| v.parse().ok())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn case_seed(i: usize) -> u64 {
|
||||
0xC0FFEE_u64 ^ (i as u64).wrapping_mul(0x9E3779B97F4A7C15)
|
||||
}
|
||||
|
||||
fn input_from_case(case: &GeneratedCase) -> ResolutionInput {
|
||||
ResolutionInput {
|
||||
world: case.world.clone(),
|
||||
program: case.program.clone(),
|
||||
contexts: case.contexts.clone(),
|
||||
contract_seed: case.contract_seed,
|
||||
perturbation_seed: case.perturbation_seed,
|
||||
}
|
||||
}
|
||||
|
||||
fn input_with_world(case: &GeneratedCase, world: WorldSnapshot) -> ResolutionInput {
|
||||
ResolutionInput {
|
||||
world,
|
||||
program: case.program.clone(),
|
||||
contexts: case.contexts.clone(),
|
||||
contract_seed: case.contract_seed,
|
||||
perturbation_seed: case.perturbation_seed,
|
||||
}
|
||||
}
|
||||
|
||||
fn masked_config(d: usize) -> EngineConfig {
|
||||
let mut c = EngineConfig::reference();
|
||||
c.domain_mask[d] = false;
|
||||
c
|
||||
}
|
||||
|
||||
fn median(v: &[f64]) -> f64 {
|
||||
if v.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut s = v.to_vec();
|
||||
s.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
s[s.len() / 2]
|
||||
}
|
||||
|
||||
fn percentile(v: &[f64], p: f64) -> f64 {
|
||||
if v.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let mut s = v.to_vec();
|
||||
s.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let idx = ((s.len() as f64 - 1.0) * p).round() as usize;
|
||||
s[idx.min(s.len() - 1)]
|
||||
}
|
||||
|
||||
/// Order-0 entropy-based compressibility of a behavior feature vector, in
|
||||
/// `[0,1]`; high-entropy (incompressible) behavior tends toward 0.
|
||||
fn compressibility(features: &[i64]) -> f64 {
|
||||
let mut bytes = Vec::with_capacity(features.len() * 8);
|
||||
for &f in features {
|
||||
bytes.extend_from_slice(&f.to_le_bytes());
|
||||
}
|
||||
if bytes.is_empty() {
|
||||
return 1.0;
|
||||
}
|
||||
let mut counts = [0u32; 256];
|
||||
for &b in &bytes {
|
||||
counts[b as usize] += 1;
|
||||
}
|
||||
let n = bytes.len() as f64;
|
||||
let mut h = 0.0;
|
||||
for &c in &counts {
|
||||
if c > 0 {
|
||||
let p = c as f64 / n;
|
||||
h -= p * p.log2();
|
||||
}
|
||||
}
|
||||
(1.0 - h / 8.0).clamp(0.0, 1.0)
|
||||
}
|
||||
|
||||
fn world_input_features(w: &WorldSnapshot) -> Vec<f64> {
|
||||
let mut out = Vec::with_capacity(NUM_DOMAINS * (LANES + HIDDEN_LANES));
|
||||
for d in &w.domains {
|
||||
for &v in &d.observed {
|
||||
out.push(v as f64);
|
||||
}
|
||||
for &v in &d.hidden {
|
||||
out.push(v as f64);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn behavior_output_features(r: &ResolutionResult) -> Vec<f64> {
|
||||
r.trace
|
||||
.behavior_fingerprint
|
||||
.features
|
||||
.iter()
|
||||
.map(|&v| v as f64)
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result aggregates.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct TraceGates {
|
||||
pub median_causal_edges: f64,
|
||||
pub p95_causal_rank: f64,
|
||||
pub median_touched: f64,
|
||||
pub p95_touched: f64,
|
||||
pub median_rank: f64,
|
||||
pub fp_collision_rate: f64,
|
||||
pub largest_cluster: f64,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct EquivalenceGates {
|
||||
pub total: usize,
|
||||
pub matched: usize,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DomainGates {
|
||||
pub appears: [f64; NUM_DOMAINS],
|
||||
pub influences: [f64; NUM_DOMAINS],
|
||||
pub mutated: [f64; NUM_DOMAINS],
|
||||
pub removal_loss: [f64; NUM_DOMAINS],
|
||||
pub min_merge_loss: f64,
|
||||
pub read_only: Vec<usize>,
|
||||
pub write_only: Vec<usize>,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MetamorphicGates {
|
||||
pub total: usize,
|
||||
pub altered_trace: f64,
|
||||
pub altered_delta: f64,
|
||||
pub altered_future: f64,
|
||||
pub neutral_unexplained: f64,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ContractGates {
|
||||
pub total: usize,
|
||||
pub passed: usize,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ReplayGates {
|
||||
pub total: usize,
|
||||
pub deterministic: usize,
|
||||
pub drift: usize,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CoverageGates {
|
||||
pub generated_cases: usize,
|
||||
pub generated_rejected: usize,
|
||||
pub contract_rejected: usize,
|
||||
pub executions: usize,
|
||||
pub perturbations: usize,
|
||||
pub failures: Vec<String>,
|
||||
}
|
||||
|
||||
pub struct CiResults {
|
||||
pub scale: Scale,
|
||||
pub trace: TraceGates,
|
||||
pub equivalence: EquivalenceGates,
|
||||
pub domain: DomainGates,
|
||||
pub metamorphic: MetamorphicGates,
|
||||
pub collapse: CollapseSummary,
|
||||
pub mutation: MutationOutcome,
|
||||
pub contract: ContractGates,
|
||||
pub replay: ReplayGates,
|
||||
pub coverage: CoverageGates,
|
||||
}
|
||||
|
||||
impl CiResults {
|
||||
pub fn all_failures(&self) -> Vec<(&'static str, &Vec<String>)> {
|
||||
vec![
|
||||
("causal_rank/trace", &self.trace.failures),
|
||||
("runtime_equivalence", &self.equivalence.failures),
|
||||
("domain_participation", &self.domain.failures),
|
||||
("metamorphic_response", &self.metamorphic.failures),
|
||||
("compression_resistance", &self.collapse.failures),
|
||||
("contract", &self.contract.failures),
|
||||
("replay", &self.replay.failures),
|
||||
("coverage", &self.coverage.failures),
|
||||
]
|
||||
}
|
||||
|
||||
pub fn passed(&self) -> bool {
|
||||
self.all_failures().iter().all(|(_, f)| f.is_empty()) && self.mutation.passed()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The pipeline.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub fn run_all(scale: Scale) -> CiResults {
|
||||
let cfg = EngineConfig::reference();
|
||||
let rut = RuntimeUnderTest::new();
|
||||
|
||||
// Main execution + metamorphic + contract pass.
|
||||
let mut ranks = Vec::with_capacity(scale.executions);
|
||||
let mut edges = Vec::with_capacity(scale.executions);
|
||||
let mut touched = Vec::with_capacity(scale.executions);
|
||||
let mut fp_counts: HashMap<Hash, u32> = HashMap::new();
|
||||
|
||||
let mut equiv_matched = 0usize;
|
||||
let mut equiv_failures = Vec::new();
|
||||
|
||||
let mut domain_appear = [0u64; NUM_DOMAINS];
|
||||
let mut domain_mutated = [0u64; NUM_DOMAINS];
|
||||
let mut domain_read_any = [false; NUM_DOMAINS];
|
||||
let mut domain_write_any = [false; NUM_DOMAINS];
|
||||
|
||||
let mut meta_total = 0usize;
|
||||
let mut meta_alt_trace = 0usize;
|
||||
let mut meta_alt_delta = 0usize;
|
||||
let mut meta_alt_future = 0usize;
|
||||
let mut meta_neutral_unexpl = 0usize;
|
||||
|
||||
let mut contract_pass = 0usize;
|
||||
let mut contract_failures: Vec<String> = Vec::new();
|
||||
|
||||
let mut collapse_inputs: Vec<Vec<f64>> = Vec::new();
|
||||
let mut collapse_outputs: Vec<Vec<f64>> = Vec::new();
|
||||
let mut collapse_fps: Vec<Hash> = Vec::new();
|
||||
|
||||
let mut mutation_inputs: Vec<ResolutionInput> = Vec::new();
|
||||
let mut generated_rejected = 0usize;
|
||||
let mut perturbation_runs = 0usize;
|
||||
|
||||
let mut contract_rejected = 0usize;
|
||||
|
||||
for i in 0..scale.executions {
|
||||
// Admission: a case is admitted only if its *measured* trace behavior
|
||||
// satisfies its contract. Structurally-flat cases are already rejected
|
||||
// by `generate_accepted_case`; here we additionally reject cases whose
|
||||
// behavior fails the contract, retrying with fresh seeds.
|
||||
let mut seed = case_seed(i);
|
||||
for attempt in 0..48 {
|
||||
let last_attempt = attempt == 47;
|
||||
let (case, _) = generate_accepted_case(seed);
|
||||
if !generated_gate_failures(&case.estimate()).is_empty() {
|
||||
generated_rejected += 1;
|
||||
}
|
||||
let input = input_from_case(&case);
|
||||
let r = execute(&cfg, &input);
|
||||
|
||||
let rank = r.trace.causal_rank();
|
||||
let touched_cnt = r.trace.touched_domain_count();
|
||||
let comp = compressibility(&r.trace.behavior_fingerprint.features);
|
||||
let div = r.trace.context_divergence();
|
||||
|
||||
// Run the 10 perturbations for metamorphic stats + future sensitivity.
|
||||
let base_trace_h = r.trace.canonical_hash();
|
||||
let base_delta_h = r.delta.hash();
|
||||
let base_future_h = r.replay.future_hash;
|
||||
let mut c_alt_trace = 0usize;
|
||||
let mut c_alt_delta = 0usize;
|
||||
let mut c_alt_future = 0usize;
|
||||
let mut c_neutral = 0usize;
|
||||
let mut c_pert = 0usize;
|
||||
for pc in &case.perturbations {
|
||||
let pr = execute(&cfg, &input_with_world(&case, pc.world.clone()));
|
||||
c_pert += 1;
|
||||
let at = pr.trace.canonical_hash() != base_trace_h;
|
||||
let ad = pr.delta.hash() != base_delta_h;
|
||||
let af = pr.replay.future_hash != base_future_h;
|
||||
if at {
|
||||
c_alt_trace += 1;
|
||||
}
|
||||
if ad {
|
||||
c_alt_delta += 1;
|
||||
}
|
||||
if af {
|
||||
c_alt_future += 1;
|
||||
}
|
||||
if !at && pc.expectation.neutral_explanation.is_none() {
|
||||
c_neutral += 1;
|
||||
}
|
||||
}
|
||||
let future_sensitivity = if c_pert > 0 {
|
||||
c_alt_future as f64 / c_pert as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let mut creasons = Vec::new();
|
||||
if rank < case.contract.min_causal_rank {
|
||||
creasons.push("causal_rank");
|
||||
}
|
||||
if touched_cnt < case.contract.min_domain_participation {
|
||||
creasons.push("domain_participation");
|
||||
}
|
||||
if future_sensitivity < case.contract.min_future_sensitivity {
|
||||
creasons.push("future_sensitivity");
|
||||
}
|
||||
if div < case.contract.min_context_divergence {
|
||||
creasons.push("context_divergence");
|
||||
}
|
||||
if comp > case.contract.max_compressibility {
|
||||
creasons.push("compressibility");
|
||||
}
|
||||
|
||||
if !creasons.is_empty() && !last_attempt {
|
||||
contract_rejected += 1;
|
||||
seed = generators::derive(seed, "contract-retry");
|
||||
continue;
|
||||
}
|
||||
|
||||
// ---- Commit the admitted case ----
|
||||
let r2 = rut.resolve(input.clone());
|
||||
if canonical(&r) == canonical(&r2) {
|
||||
equiv_matched += 1;
|
||||
} else if equiv_failures.len() < 16 {
|
||||
equiv_failures.push(format!("case {} reference != runtime_under_test", i));
|
||||
}
|
||||
|
||||
ranks.push(rank as f64);
|
||||
edges.push(r.trace.causal_edge_count() as f64);
|
||||
touched.push(touched_cnt as f64);
|
||||
*fp_counts.entry(r.trace.behavior_fingerprint.hash).or_insert(0) += 1;
|
||||
|
||||
for d in 0..NUM_DOMAINS {
|
||||
let read = r.trace.read_graph.access_count[d] > 0;
|
||||
let write = r.trace.write_graph.access_count[d] > 0;
|
||||
if read || write {
|
||||
domain_appear[d] += 1;
|
||||
}
|
||||
if read {
|
||||
domain_read_any[d] = true;
|
||||
}
|
||||
if write {
|
||||
domain_write_any[d] = true;
|
||||
}
|
||||
}
|
||||
for dd in &r.delta.domain_deltas {
|
||||
if !dd.is_zero() {
|
||||
domain_mutated[dd.domain.0 as usize] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
meta_total += c_pert;
|
||||
meta_alt_trace += c_alt_trace;
|
||||
meta_alt_delta += c_alt_delta;
|
||||
meta_alt_future += c_alt_future;
|
||||
meta_neutral_unexpl += c_neutral;
|
||||
perturbation_runs += c_pert;
|
||||
|
||||
if creasons.is_empty() {
|
||||
contract_pass += 1;
|
||||
} else if contract_failures.len() < 16 {
|
||||
contract_failures.push(format!("case {} violates {:?}", i, creasons));
|
||||
}
|
||||
|
||||
if collapse_inputs.len() < scale.collapse_samples {
|
||||
collapse_inputs.push(world_input_features(&case.world));
|
||||
collapse_outputs.push(behavior_output_features(&r));
|
||||
collapse_fps.push(r.trace.behavior_fingerprint.hash);
|
||||
}
|
||||
if mutation_inputs.len() < 64 {
|
||||
mutation_inputs.push(input.clone());
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let n = scale.executions.max(1) as f64;
|
||||
|
||||
// ---- Trace gates ----
|
||||
let collisions = scale.executions - fp_counts.len();
|
||||
let largest_cluster = fp_counts.values().copied().max().unwrap_or(0) as f64 / n;
|
||||
let mut trace_failures = Vec::new();
|
||||
let med_edges = median(&edges);
|
||||
let p95_rank = percentile(&ranks, 0.05); // 95% of executions have rank >= this
|
||||
let med_touched = median(&touched);
|
||||
let p95_touched = percentile(&touched, 0.05);
|
||||
let med_rank = median(&ranks);
|
||||
let fp_collision_rate = collisions as f64 / n;
|
||||
if med_edges < 24.0 {
|
||||
trace_failures.push(format!("median causal edges {} < 24", med_edges));
|
||||
}
|
||||
if p95_rank < 6.0 {
|
||||
trace_failures.push(format!("95% causal rank {} < 6", p95_rank));
|
||||
}
|
||||
if med_touched < 4.0 {
|
||||
trace_failures.push(format!("median touched {} < 4", med_touched));
|
||||
}
|
||||
if p95_touched < 3.0 {
|
||||
trace_failures.push(format!("95% touched {} < 3", p95_touched));
|
||||
}
|
||||
if fp_collision_rate >= 0.05 {
|
||||
trace_failures.push(format!("fp collision rate {:.4} >= 0.05", fp_collision_rate));
|
||||
}
|
||||
if largest_cluster >= 0.02 {
|
||||
trace_failures.push(format!("largest cluster {:.4} >= 0.02", largest_cluster));
|
||||
}
|
||||
let trace = TraceGates {
|
||||
median_causal_edges: med_edges,
|
||||
p95_causal_rank: p95_rank,
|
||||
median_touched: med_touched,
|
||||
p95_touched,
|
||||
median_rank: med_rank,
|
||||
fp_collision_rate,
|
||||
largest_cluster,
|
||||
failures: trace_failures,
|
||||
};
|
||||
|
||||
// ---- Equivalence gates ----
|
||||
if equiv_matched != scale.executions {
|
||||
equiv_failures.push(format!(
|
||||
"{}/{} executions matched (require 100%)",
|
||||
equiv_matched, scale.executions
|
||||
));
|
||||
}
|
||||
let equivalence = EquivalenceGates {
|
||||
total: scale.executions,
|
||||
matched: equiv_matched,
|
||||
failures: equiv_failures,
|
||||
};
|
||||
|
||||
// ---- Domain participation gates ----
|
||||
let domain = domain_gates(
|
||||
scale,
|
||||
&domain_appear,
|
||||
&domain_mutated,
|
||||
&domain_read_any,
|
||||
&domain_write_any,
|
||||
n,
|
||||
);
|
||||
|
||||
// ---- Metamorphic gates ----
|
||||
let mt = meta_total.max(1) as f64;
|
||||
let mut meta_failures = Vec::new();
|
||||
let r_trace = meta_alt_trace as f64 / mt;
|
||||
let r_delta = meta_alt_delta as f64 / mt;
|
||||
let r_future = meta_alt_future as f64 / mt;
|
||||
let r_neutral = meta_neutral_unexpl as f64 / mt;
|
||||
if r_trace < 0.90 {
|
||||
meta_failures.push(format!("altered trace {:.3} < 0.90", r_trace));
|
||||
}
|
||||
if r_delta < 0.75 {
|
||||
meta_failures.push(format!("altered delta {:.3} < 0.75", r_delta));
|
||||
}
|
||||
if r_future < 0.50 {
|
||||
meta_failures.push(format!("altered future {:.3} < 0.50", r_future));
|
||||
}
|
||||
if r_neutral > 0.05 {
|
||||
meta_failures.push(format!("unexplained neutral {:.3} > 0.05", r_neutral));
|
||||
}
|
||||
let metamorphic = MetamorphicGates {
|
||||
total: meta_total,
|
||||
altered_trace: r_trace,
|
||||
altered_delta: r_delta,
|
||||
altered_future: r_future,
|
||||
neutral_unexplained: r_neutral,
|
||||
failures: meta_failures,
|
||||
};
|
||||
|
||||
// ---- Collapse gates ----
|
||||
let corpus = BehaviorCorpus::build(collapse_inputs, collapse_outputs, collapse_fps);
|
||||
let collapse = analyze(&corpus);
|
||||
|
||||
// ---- Mutation gates ----
|
||||
let mutation = run_suite(scale.mutants, &mutation_inputs);
|
||||
|
||||
// ---- Contract gates ----
|
||||
let mut contract_gate_failures = Vec::new();
|
||||
if contract_pass != scale.executions {
|
||||
contract_gate_failures.push(format!(
|
||||
"{}/{} cases satisfy their contract",
|
||||
contract_pass, scale.executions
|
||||
));
|
||||
contract_gate_failures.extend(contract_failures);
|
||||
}
|
||||
let contract = ContractGates {
|
||||
total: scale.executions,
|
||||
passed: contract_pass,
|
||||
failures: contract_gate_failures,
|
||||
};
|
||||
|
||||
// ---- Replay gates ----
|
||||
let replay = replay_gates(scale);
|
||||
|
||||
// ---- Coverage gates ----
|
||||
let mut cov_failures = Vec::new();
|
||||
if generated_rejected != 0 {
|
||||
cov_failures.push(format!(
|
||||
"{} admitted cases failed generated gates",
|
||||
generated_rejected
|
||||
));
|
||||
}
|
||||
if scale.executions == 0 {
|
||||
cov_failures.push("no executions".into());
|
||||
}
|
||||
let coverage = CoverageGates {
|
||||
generated_cases: scale.executions,
|
||||
generated_rejected,
|
||||
contract_rejected,
|
||||
executions: scale.executions,
|
||||
perturbations: perturbation_runs,
|
||||
failures: cov_failures,
|
||||
};
|
||||
|
||||
CiResults {
|
||||
scale,
|
||||
trace,
|
||||
equivalence,
|
||||
domain,
|
||||
metamorphic,
|
||||
collapse,
|
||||
mutation,
|
||||
contract,
|
||||
replay,
|
||||
coverage,
|
||||
}
|
||||
}
|
||||
|
||||
fn domain_gates(
|
||||
scale: Scale,
|
||||
appear: &[u64; NUM_DOMAINS],
|
||||
mutated: &[u64; NUM_DOMAINS],
|
||||
read_any: &[bool; NUM_DOMAINS],
|
||||
write_any: &[bool; NUM_DOMAINS],
|
||||
n: f64,
|
||||
) -> DomainGates {
|
||||
let cfg = EngineConfig::reference();
|
||||
|
||||
let mut appears = [0.0; NUM_DOMAINS];
|
||||
let mut mutated_f = [0.0; NUM_DOMAINS];
|
||||
for d in 0..NUM_DOMAINS {
|
||||
appears[d] = appear[d] as f64 / n;
|
||||
mutated_f[d] = mutated[d] as f64 / n;
|
||||
}
|
||||
|
||||
// Influence probe: mask each domain and see how often the trace changes.
|
||||
let mut influences = [0.0; NUM_DOMAINS];
|
||||
let probe_cases = scale.domain_probe_cases.max(1);
|
||||
for d in 0..NUM_DOMAINS {
|
||||
let masked = masked_config(d);
|
||||
let mut changed = 0usize;
|
||||
for i in 0..probe_cases {
|
||||
let (case, _) = generate_accepted_case(case_seed(i));
|
||||
let input = input_from_case(&case);
|
||||
let base = execute(&cfg, &input);
|
||||
let alt = execute(&masked, &input);
|
||||
if base.trace.canonical_hash() != alt.trace.canonical_hash()
|
||||
|| base.delta.hash() != alt.delta.hash()
|
||||
{
|
||||
changed += 1;
|
||||
}
|
||||
}
|
||||
influences[d] = changed as f64 / probe_cases as f64;
|
||||
}
|
||||
|
||||
// Removal diversity probe: vary only domain d across K worlds; compare the
|
||||
// count of distinct behaviors with the domain present vs masked.
|
||||
let mut removal_loss = [0.0; NUM_DOMAINS];
|
||||
let k = scale.domain_probe_variations.max(8);
|
||||
for d in 0..NUM_DOMAINS {
|
||||
let base_world = generators::generate_world(0xBEEF ^ d as u64);
|
||||
let case = {
|
||||
let (c, _) = generate_accepted_case(case_seed(d + 1));
|
||||
c
|
||||
};
|
||||
let mut full = std::collections::HashSet::new();
|
||||
let mut masked = std::collections::HashSet::new();
|
||||
let mcfg = masked_config(d);
|
||||
for v in 0..k {
|
||||
let mut w = base_world.clone();
|
||||
for l in 0..LANES {
|
||||
w.domains[d].observed[l] = (v as i64 + 1).wrapping_mul(7919 + l as i64);
|
||||
}
|
||||
w.domains[d].hidden[0] = (v as i64).wrapping_mul(104729);
|
||||
let input = input_with_world(&case, w);
|
||||
full.insert(execute(&cfg, &input).trace.behavior_fingerprint.hash);
|
||||
masked.insert(execute(&mcfg, &input).trace.behavior_fingerprint.hash);
|
||||
}
|
||||
let df = full.len().max(1) as f64;
|
||||
let dm = masked.len() as f64;
|
||||
removal_loss[d] = (1.0 - dm / df).clamp(0.0, 1.0);
|
||||
}
|
||||
|
||||
// Merge probe: alias domain b := domain a, see how often behavior changes.
|
||||
let mut min_merge_loss = 1.0f64;
|
||||
let merge_cases = (scale.domain_probe_cases / 2).max(20);
|
||||
for a in 0..NUM_DOMAINS {
|
||||
for b in (a + 1)..NUM_DOMAINS {
|
||||
let mut changed = 0usize;
|
||||
for i in 0..merge_cases {
|
||||
let (case, _) = generate_accepted_case(case_seed(i));
|
||||
let base = execute(&cfg, &input_from_case(&case));
|
||||
let mut w = case.world.clone();
|
||||
w.domains[b].observed = w.domains[a].observed;
|
||||
w.domains[b].hidden = w.domains[a].hidden;
|
||||
let merged = execute(&cfg, &input_with_world(&case, w));
|
||||
if merged.trace.behavior_fingerprint.hash != base.trace.behavior_fingerprint.hash {
|
||||
changed += 1;
|
||||
}
|
||||
}
|
||||
let loss = changed as f64 / merge_cases as f64;
|
||||
min_merge_loss = min_merge_loss.min(loss);
|
||||
}
|
||||
}
|
||||
|
||||
let read_only: Vec<usize> = (0..NUM_DOMAINS)
|
||||
.filter(|&d| read_any[d] && !write_any[d])
|
||||
.collect();
|
||||
let write_only: Vec<usize> = (0..NUM_DOMAINS)
|
||||
.filter(|&d| write_any[d] && !read_any[d])
|
||||
.collect();
|
||||
|
||||
let mut failures = Vec::new();
|
||||
for d in 0..NUM_DOMAINS {
|
||||
if appears[d] < 0.35 {
|
||||
failures.push(format!("domain {} appears {:.3} < 0.35", d, appears[d]));
|
||||
}
|
||||
if influences[d] < 0.20 {
|
||||
failures.push(format!("domain {} influences {:.3} < 0.20", d, influences[d]));
|
||||
}
|
||||
if mutated_f[d] < 0.20 {
|
||||
failures.push(format!("domain {} mutated {:.3} < 0.20", d, mutated_f[d]));
|
||||
}
|
||||
if removal_loss[d] < 0.10 {
|
||||
failures.push(format!(
|
||||
"domain {} removal diversity loss {:.3} < 0.10",
|
||||
d, removal_loss[d]
|
||||
));
|
||||
}
|
||||
}
|
||||
if min_merge_loss < 0.08 {
|
||||
failures.push(format!("min merge loss {:.3} < 0.08", min_merge_loss));
|
||||
}
|
||||
if !read_only.is_empty() {
|
||||
failures.push(format!("read-only domains: {:?}", read_only));
|
||||
}
|
||||
if !write_only.is_empty() {
|
||||
failures.push(format!("write-only domains: {:?}", write_only));
|
||||
}
|
||||
|
||||
DomainGates {
|
||||
appears,
|
||||
influences,
|
||||
mutated: mutated_f,
|
||||
removal_loss,
|
||||
min_merge_loss,
|
||||
read_only,
|
||||
write_only,
|
||||
failures,
|
||||
}
|
||||
}
|
||||
|
||||
fn replay_gates(scale: Scale) -> ReplayGates {
|
||||
let corpus = replay_corpus::build_corpus(scale.replay_cases, 0x5EED);
|
||||
let report = replay_corpus::verify_corpus(&corpus);
|
||||
let mut failures = Vec::new();
|
||||
if !report.drift.is_empty() {
|
||||
failures.push(format!("{} replay cases drifted", report.drift.len()));
|
||||
}
|
||||
// Minimum corpus size is a merge-blocking gate only at full scale.
|
||||
if matches!(std::env::var("MAGICKA_SCALE").as_deref(), Ok("full")) && report.total < 10_000 {
|
||||
failures.push(format!("replay corpus {} < 10000", report.total));
|
||||
}
|
||||
ReplayGates {
|
||||
total: report.total,
|
||||
deterministic: report.deterministic,
|
||||
drift: report.drift.len(),
|
||||
failures,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tiny_ci_passes_all_gates() {
|
||||
let r = run_all(Scale::tiny());
|
||||
for (name, f) in r.all_failures() {
|
||||
assert!(f.is_empty(), "{name} failed: {:?}", f);
|
||||
}
|
||||
assert!(r.mutation.passed(), "mutants survived: {:?}", r.mutation.survivors);
|
||||
assert!(r.passed());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
//! The `ci` binary: runs the full adversarial framework against the reference
|
||||
//! and runtime-under-test, writes the eight required reports (JSON + markdown),
|
||||
//! and exits nonzero if any gate fails.
|
||||
|
||||
use ci_reports::json::Json;
|
||||
use ci_reports::{run_all, CiResults, Scale};
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
fn arr_f(vals: &[f64]) -> Json {
|
||||
Json::Arr(vals.iter().map(|&v| Json::Num(v)).collect())
|
||||
}
|
||||
fn fails(v: &[String]) -> Json {
|
||||
Json::Arr(v.iter().map(|s| Json::s(s.clone())).collect())
|
||||
}
|
||||
fn pass_field(v: &[String]) -> Json {
|
||||
Json::Bool(v.is_empty())
|
||||
}
|
||||
|
||||
fn write_report(dir: &Path, name: &str, j: &Json) {
|
||||
let path = dir.join(format!("{name}.json"));
|
||||
let mut f = fs::File::create(&path).expect("create report");
|
||||
f.write_all(j.to_pretty().as_bytes()).expect("write report");
|
||||
}
|
||||
|
||||
fn build_reports(dir: &Path, r: &CiResults) {
|
||||
// 1. domain_participation_report
|
||||
write_report(
|
||||
dir,
|
||||
"domain_participation_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.domain.failures)),
|
||||
("appears_fraction".into(), arr_f(&r.domain.appears)),
|
||||
("influences_fraction".into(), arr_f(&r.domain.influences)),
|
||||
("mutated_fraction".into(), arr_f(&r.domain.mutated)),
|
||||
("removal_diversity_loss".into(), arr_f(&r.domain.removal_loss)),
|
||||
("min_merge_loss".into(), Json::Num(r.domain.min_merge_loss)),
|
||||
(
|
||||
"read_only_domains".into(),
|
||||
Json::Arr(r.domain.read_only.iter().map(|&d| Json::Int(d as i64)).collect()),
|
||||
),
|
||||
(
|
||||
"write_only_domains".into(),
|
||||
Json::Arr(r.domain.write_only.iter().map(|&d| Json::Int(d as i64)).collect()),
|
||||
),
|
||||
("failures".into(), fails(&r.domain.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 2. causal_rank_report (trace gates)
|
||||
write_report(
|
||||
dir,
|
||||
"causal_rank_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.trace.failures)),
|
||||
("median_causal_rank".into(), Json::Num(r.trace.median_rank)),
|
||||
("p95_causal_rank".into(), Json::Num(r.trace.p95_causal_rank)),
|
||||
("median_causal_edges".into(), Json::Num(r.trace.median_causal_edges)),
|
||||
("median_touched_domains".into(), Json::Num(r.trace.median_touched)),
|
||||
("p95_touched_domains".into(), Json::Num(r.trace.p95_touched)),
|
||||
("fp_collision_rate".into(), Json::Num(r.trace.fp_collision_rate)),
|
||||
("largest_cluster".into(), Json::Num(r.trace.largest_cluster)),
|
||||
("failures".into(), fails(&r.trace.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 3. compression_resistance_report
|
||||
let attack_json: Vec<Json> = r
|
||||
.collapse
|
||||
.reports
|
||||
.iter()
|
||||
.map(|rep| {
|
||||
Json::Obj(vec![
|
||||
("attack".into(), Json::s(rep.attack.clone())),
|
||||
("predicts".into(), Json::Num(rep.predicts)),
|
||||
("info_loss".into(), Json::Num(rep.info_loss)),
|
||||
("detail".into(), Json::s(rep.detail.clone())),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
write_report(
|
||||
dir,
|
||||
"compression_resistance_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.collapse.failures)),
|
||||
("best_1factor".into(), Json::Num(r.collapse.best_1factor)),
|
||||
("best_2factor".into(), Json::Num(r.collapse.best_2factor)),
|
||||
("best_4factor".into(), Json::Num(r.collapse.best_4factor)),
|
||||
("max_single_domain".into(), Json::Num(r.collapse.max_single_domain)),
|
||||
("max_pair".into(), Json::Num(r.collapse.max_pair)),
|
||||
("min_info_loss".into(), Json::Num(r.collapse.min_info_loss)),
|
||||
("attacks".into(), Json::Arr(attack_json)),
|
||||
("failures".into(), fails(&r.collapse.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 4. metamorphic_response_report
|
||||
write_report(
|
||||
dir,
|
||||
"metamorphic_response_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.metamorphic.failures)),
|
||||
("perturbations".into(), Json::Int(r.metamorphic.total as i64)),
|
||||
("altered_trace".into(), Json::Num(r.metamorphic.altered_trace)),
|
||||
("altered_delta".into(), Json::Num(r.metamorphic.altered_delta)),
|
||||
("altered_future".into(), Json::Num(r.metamorphic.altered_future)),
|
||||
("neutral_unexplained".into(), Json::Num(r.metamorphic.neutral_unexplained)),
|
||||
("failures".into(), fails(&r.metamorphic.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 5. mutation_survivor_report
|
||||
let survivors: Vec<Json> = r
|
||||
.mutation
|
||||
.survivors
|
||||
.iter()
|
||||
.map(|(id, name)| {
|
||||
Json::Obj(vec![
|
||||
("id".into(), Json::Int(*id as i64)),
|
||||
("name".into(), Json::s(name.clone())),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
write_report(
|
||||
dir,
|
||||
"mutation_survivor_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), Json::Bool(r.mutation.passed())),
|
||||
("total_mutants".into(), Json::Int(r.mutation.total as i64)),
|
||||
("killed".into(), Json::Int(r.mutation.killed as i64)),
|
||||
("survivors".into(), Json::Arr(survivors)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 6. runtime_equivalence_report
|
||||
write_report(
|
||||
dir,
|
||||
"runtime_equivalence_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.equivalence.failures)),
|
||||
("total".into(), Json::Int(r.equivalence.total as i64)),
|
||||
("matched".into(), Json::Int(r.equivalence.matched as i64)),
|
||||
("failures".into(), fails(&r.equivalence.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 7. replay_report
|
||||
write_report(
|
||||
dir,
|
||||
"replay_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.replay.failures)),
|
||||
("total".into(), Json::Int(r.replay.total as i64)),
|
||||
("deterministic".into(), Json::Int(r.replay.deterministic as i64)),
|
||||
("drift".into(), Json::Int(r.replay.drift as i64)),
|
||||
("failures".into(), fails(&r.replay.failures)),
|
||||
]),
|
||||
);
|
||||
|
||||
// 8. coverage_report
|
||||
write_report(
|
||||
dir,
|
||||
"coverage_report",
|
||||
&Json::Obj(vec![
|
||||
("pass".into(), pass_field(&r.coverage.failures)),
|
||||
("generated_cases".into(), Json::Int(r.coverage.generated_cases as i64)),
|
||||
("generated_rejected".into(), Json::Int(r.coverage.generated_rejected as i64)),
|
||||
("contract_rejected".into(), Json::Int(r.coverage.contract_rejected as i64)),
|
||||
("executions".into(), Json::Int(r.coverage.executions as i64)),
|
||||
("perturbations".into(), Json::Int(r.coverage.perturbations as i64)),
|
||||
("contracts_passed".into(), Json::Int(r.contract.passed as i64)),
|
||||
("contracts_total".into(), Json::Int(r.contract.total as i64)),
|
||||
("failures".into(), fails(&r.coverage.failures)),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
fn status(v: bool) -> &'static str {
|
||||
if v {
|
||||
"PASS"
|
||||
} else {
|
||||
"FAIL"
|
||||
}
|
||||
}
|
||||
|
||||
fn write_markdown(dir: &Path, r: &CiResults) {
|
||||
let mut s = String::new();
|
||||
s.push_str("# Magicka VM — Phase 0/1 CI Report\n\n");
|
||||
s.push_str(&format!(
|
||||
"Overall: **{}**\n\n",
|
||||
status(r.passed())
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"Scale: executions={}, mutants={}, replay={}, collapse_samples={}\n\n",
|
||||
r.scale.executions, r.scale.mutants, r.scale.replay_cases, r.scale.collapse_samples
|
||||
));
|
||||
s.push_str("| Report | Status | Key metrics |\n|---|---|---|\n");
|
||||
s.push_str(&format!(
|
||||
"| runtime_equivalence | {} | {}/{} matched |\n",
|
||||
status(r.equivalence.failures.is_empty()),
|
||||
r.equivalence.matched,
|
||||
r.equivalence.total
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| causal_rank/trace | {} | rank med={} p95={}, edges med={}, touched med={} |\n",
|
||||
status(r.trace.failures.is_empty()),
|
||||
r.trace.median_rank,
|
||||
r.trace.p95_causal_rank,
|
||||
r.trace.median_causal_edges,
|
||||
r.trace.median_touched
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| domain_participation | {} | min_merge_loss={:.3} |\n",
|
||||
status(r.domain.failures.is_empty()),
|
||||
r.domain.min_merge_loss
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| metamorphic_response | {} | trace={:.3} delta={:.3} future={:.3} |\n",
|
||||
status(r.metamorphic.failures.is_empty()),
|
||||
r.metamorphic.altered_trace,
|
||||
r.metamorphic.altered_delta,
|
||||
r.metamorphic.altered_future
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| compression_resistance | {} | 1f={:.3} 2f={:.3} 4f={:.3} single={:.3} pair={:.3} info_loss={:.3} |\n",
|
||||
status(r.collapse.failures.is_empty()),
|
||||
r.collapse.best_1factor,
|
||||
r.collapse.best_2factor,
|
||||
r.collapse.best_4factor,
|
||||
r.collapse.max_single_domain,
|
||||
r.collapse.max_pair,
|
||||
r.collapse.min_info_loss
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| mutation_survivor | {} | killed {}/{} |\n",
|
||||
status(r.mutation.passed()),
|
||||
r.mutation.killed,
|
||||
r.mutation.total
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| contract | {} | {}/{} cases |\n",
|
||||
status(r.contract.failures.is_empty()),
|
||||
r.contract.passed,
|
||||
r.contract.total
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| replay | {} | {}/{} deterministic, drift={} |\n",
|
||||
status(r.replay.failures.is_empty()),
|
||||
r.replay.deterministic,
|
||||
r.replay.total,
|
||||
r.replay.drift
|
||||
));
|
||||
s.push_str(&format!(
|
||||
"| coverage | {} | exec={}, perturb={}, rejected={} |\n",
|
||||
status(r.coverage.failures.is_empty()),
|
||||
r.coverage.executions,
|
||||
r.coverage.perturbations,
|
||||
r.coverage.generated_rejected
|
||||
));
|
||||
|
||||
s.push_str("\n## Failures\n\n");
|
||||
let mut any = false;
|
||||
for (name, f) in r.all_failures() {
|
||||
for msg in f {
|
||||
any = true;
|
||||
s.push_str(&format!("- **{}**: {}\n", name, msg));
|
||||
}
|
||||
}
|
||||
for (id, name) in &r.mutation.survivors {
|
||||
any = true;
|
||||
s.push_str(&format!("- **mutation_survivor**: mutant {} ({}) survived\n", id, name));
|
||||
}
|
||||
if !any {
|
||||
s.push_str("None. The fake universe failed to collapse. ✅\n");
|
||||
}
|
||||
|
||||
let path = dir.join("ci_summary.md");
|
||||
fs::write(path, s).expect("write markdown");
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let scale = Scale::from_env();
|
||||
let out_dir = std::env::var("MAGICKA_OUT").unwrap_or_else(|_| "ci_out".to_string());
|
||||
let dir = Path::new(&out_dir);
|
||||
fs::create_dir_all(dir).expect("create out dir");
|
||||
|
||||
eprintln!(
|
||||
"running CI: executions={} mutants={} replay={} collapse_samples={}",
|
||||
scale.executions, scale.mutants, scale.replay_cases, scale.collapse_samples
|
||||
);
|
||||
let start = Instant::now();
|
||||
let results = run_all(scale);
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
build_reports(dir, &results);
|
||||
write_markdown(dir, &results);
|
||||
|
||||
println!("\n=== Magicka VM CI ({:?}) ===", elapsed);
|
||||
for (name, f) in results.all_failures() {
|
||||
println!(" {:<24} {}", name, status(f.is_empty()));
|
||||
}
|
||||
println!(
|
||||
" {:<24} {} ({} killed / {} mutants{})",
|
||||
"mutation_survivor",
|
||||
status(results.mutation.passed()),
|
||||
results.mutation.killed,
|
||||
results.mutation.total,
|
||||
if results.mutation.survivors.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(", {} survivors", results.mutation.survivors.len())
|
||||
}
|
||||
);
|
||||
println!("reports written to {}/", out_dir);
|
||||
|
||||
if results.passed() {
|
||||
println!("\nOVERALL: PASS — the adversarial framework could not collapse the universe.");
|
||||
std::process::exit(0);
|
||||
} else {
|
||||
println!("\nOVERALL: FAIL");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user