update

crates/collapse_analysis/src/lib.rs

@@ -1,28 +1,41 @@
-//! `collapse_analysis` — the framework's compression attacks. Each attack
-//! tries to predict execution behavior with a *simpler* model. If any small
-//! model predicts above the configured thresholds, the universe has collapsed
-//! and CI must fail.
+//! `collapse_analysis` — the framework's compression attacks.
 //!
-//! The corpus is a set of (input, output) samples: `input` is the world ground
-//! truth (8 domains x (observed+hidden) lanes) and `output` is the behavior
-//! feature vector produced by the runtime. Inputs/outputs are standardized so
-//! that no single high-magnitude axis dominates the variance accounting.
+//! Each attack tries to reconstruct the *actual serialized trace* from a
+//! simpler (compressed) representation of it. If a small model reconstructs the
+//! trace above the configured thresholds, the universe has collapsed and CI must
+//! fail.
+//!
+//! The earlier version of this crate operated on a 22-element behavior
+//! *fingerprint* — a hash-derived proxy the avalanche engine guarantees is
+//! near-random — and defined information loss circularly as `1 - predicts`. That
+//! made every attack pass for free: the metric never touched real trace content.
+//!
+//! This version feeds the genuine trace structure (per-domain causal influence,
+//! information flow, access counts, temporal reach, and state deltas, plus the
+//! global rank/edge/divergence summaries) into the attacks, and defines
+//! information loss as the real unexplained-variance fraction of reconstructing
+//! the trace. The negative-control test builds a deliberately collapsible
+//! (single-factor) corpus and verifies the gate rejects it, demonstrating the
+//! gate discriminates a rich universe from a degenerate one.
 
 pub mod linalg;
 
 use linalg::{ols_r2, pca_scores, Mat};
-use world_model::{Hash, HIDDEN_LANES, LANES, NUM_DOMAINS};
+use world_model::NUM_DOMAINS;
 
-/// Input columns belonging to one domain (observed + hidden lanes).
-pub const DOMAIN_BLOCK: usize = LANES + HIDDEN_LANES;
+/// Width of one per-domain feature block in a trace feature row.
+/// `[infl_out, infl_in, flow_out, flow_in, read, write, temporal, obs_delta, hid_delta]`
+pub const BLOCK_W: usize = 9;
+/// Number of trailing global (non-domain) feature columns.
+/// `[causal_rank, edge_count, touched_count, divergence_mean]`
+pub const GLOBALS: usize = 4;
+/// Total trace feature width.
+pub const FEATURE_W: usize = NUM_DOMAINS * BLOCK_W + GLOBALS;
 
-/// A standardized behavior corpus.
+/// A standardized corpus of real trace feature rows.
 pub struct BehaviorCorpus {
-    /// `n x (NUM_DOMAINS*DOMAIN_BLOCK)` standardized input.
-    pub x: Mat,
-    /// `n x m` standardized output (behavior features).
-    pub y: Mat,
-    pub fingerprints: Vec<Hash>,
+    /// `n x FEATURE_W` standardized trace features.
+    pub traces: Mat,
 }
 
 fn standardize(rows: &[Vec<f64>]) -> Mat {
@@ -34,7 +47,6 @@ fn standardize(rows: &[Vec<f64>]) -> Mat {
             m.set(r, c, rows[r][c]);
         }
     }
-    // center + scale to unit std per column
     for c in 0..cols {
         let mut mean = 0.0;
         for r in 0..n {
@@ -57,47 +69,65 @@ fn standardize(rows: &[Vec<f64>]) -> Mat {
 }
 
 impl BehaviorCorpus {
-    pub fn build(inputs: Vec<Vec<f64>>, outputs: Vec<Vec<f64>>, fingerprints: Vec<Hash>) -> Self {
+    pub fn build(rows: Vec<Vec<f64>>) -> Self {
         BehaviorCorpus {
-            x: standardize(&inputs),
-            y: standardize(&outputs),
-            fingerprints,
+            traces: standardize(&rows),
         }
     }
 
     pub fn n(&self) -> usize {
-        self.x.rows
+        self.traces.rows
+    }
+
+    fn cols(&self) -> usize {
+        self.traces.cols
+    }
+
+    fn block_cols(d: usize) -> Vec<usize> {
+        (d * BLOCK_W..(d + 1) * BLOCK_W).collect()
     }
 
     fn select(&self, cols: &[usize]) -> Mat {
-        let mut m = Mat::zeros(self.x.rows, cols.len());
-        for r in 0..self.x.rows {
+        let mut m = Mat::zeros(self.traces.rows, cols.len());
+        for r in 0..self.traces.rows {
             for (j, &c) in cols.iter().enumerate() {
-                m.set(r, j, self.x.at(r, c));
+                m.set(r, j, self.traces.at(r, c));
             }
         }
         m
     }
 
-    fn domain_cols(domain: usize) -> Vec<usize> {
-        (domain * DOMAIN_BLOCK..(domain + 1) * DOMAIN_BLOCK).collect()
+    fn complement(&self, cols: &[usize]) -> Vec<usize> {
+        (0..self.cols()).filter(|c| !cols.contains(c)).collect()
     }
 
-    /// R² of predicting output from the top-`k` PCA factors of the full input.
-    pub fn predict_k_factor(&self, k: usize) -> f64 {
-        if self.n() == 0 {
+    /// R² of reconstructing the columns `target` from the columns `source`.
+    fn reconstruct(&self, source: &[usize], target: &[usize]) -> f64 {
+        if source.is_empty() || target.is_empty() {
             return 0.0;
         }
-        let scores = pca_scores(&self.x, k);
-        ols_r2(&scores, &self.y)
+        let x = self.select(source);
+        let y = self.select(target);
+        ols_r2(&x, &y)
     }
 
-    /// Best R² obtainable using just a single domain's input block.
+    /// Explained-variance fraction of the whole trace from its top-`k` PCA
+    /// factors (a genuine k-factor reconstruction quality).
+    pub fn predict_k_factor(&self, k: usize) -> f64 {
+        if self.n() == 0 || self.cols() == 0 {
+            return 0.0;
+        }
+        let scores = pca_scores(&self.traces, k);
+        ols_r2(&scores, &self.traces)
+    }
+
+    /// Best fraction of the *rest of the trace* explained by a single domain.
     pub fn max_single_domain(&self) -> (usize, f64) {
         let mut best = (0usize, 0.0);
         for d in 0..NUM_DOMAINS {
-            let sub = self.select(&Self::domain_cols(d));
-            let r2 = ols_r2(&sub, &self.y);
+            let src = Self::block_cols(d);
+            let tgt = self.complement(&src);
+            let r2 = self.reconstruct(&src, &tgt);
             if r2 > best.1 {
                 best = (d, r2);
             }
@@ -105,15 +135,15 @@ impl BehaviorCorpus {
         best
     }
 
-    /// Best R² obtainable using any pair of domain blocks.
+    /// Best fraction of the rest explained by any pair of domains.
     pub fn max_pair(&self) -> ((usize, usize), f64) {
         let mut best = ((0usize, 1usize), 0.0);
         for a in 0..NUM_DOMAINS {
             for b in (a + 1)..NUM_DOMAINS {
-                let mut cols = Self::domain_cols(a);
-                cols.extend(Self::domain_cols(b));
-                let sub = self.select(&cols);
-                let r2 = ols_r2(&sub, &self.y);
+                let mut src = Self::block_cols(a);
+                src.extend(Self::block_cols(b));
+                let tgt = self.complement(&src);
+                let r2 = self.reconstruct(&src, &tgt);
                 if r2 > best.1 {
                     best = ((a, b), r2);
                 }
@@ -121,13 +151,9 @@ impl BehaviorCorpus {
         }
         best
     }
-
-    fn full_r2(&self) -> f64 {
-        ols_r2(&self.x, &self.y)
-    }
 }
 
-/// A compressed model's predictive power and information loss.
+/// A compressed model's reconstruction power and (genuine) information loss.
 #[derive(Clone, Debug)]
 pub struct CompressedModel {
     pub predicts: f64,
@@ -139,6 +165,15 @@ pub struct CompressedModel {
 pub trait CollapseAttack {
     fn name(&self) -> &'static str;
     fn compress(&self, corpus: &BehaviorCorpus) -> CompressedModel;
+    /// Whether this attack produces a dimensionality-reduced representation of
+    /// the *entire* trace (a genuine compressed model), as opposed to an
+    /// ablation probe that reconstructs one facet from the rest. Only
+    /// whole-trace compressors define the "compressed model loses ≥ 35%
+    /// information" gate; ablation probes are reported for the record and feed
+    /// their own structural gates (domain participation, etc.).
+    fn whole_trace(&self) -> bool {
+        false
+    }
 }
 
 /// One attack's outcome.
@@ -150,8 +185,21 @@ pub struct CollapseReport {
     pub detail: String,
 }
 
+/// Build a compressed-model result. `info_loss` is the genuine unexplained
+/// fraction of the reconstructed trace variance.
+fn model(predicts: f64, detail: &str) -> CompressedModel {
+    CompressedModel {
+        predicts,
+        info_loss: (1.0 - predicts).clamp(0.0, 1.0),
+        detail: detail.to_string(),
+    }
+}
+
 macro_rules! attack {
     ($name:ident, $label:expr, $body:expr) => {
+        attack!($name, $label, false, $body);
+    };
+    ($name:ident, $label:expr, $whole:expr, $body:expr) => {
         pub struct $name;
         impl CollapseAttack for $name {
             fn name(&self) -> &'static str {
@@ -161,130 +209,129 @@ macro_rules! attack {
                 let f: fn(&BehaviorCorpus) -> CompressedModel = $body;
                 f(corpus)
             }
+            fn whole_trace(&self) -> bool {
+                $whole
+            }
         }
     };
 }
 
-fn model(predicts: f64, detail: &str) -> CompressedModel {
-    CompressedModel {
-        predicts,
-        info_loss: (1.0 - predicts).clamp(0.0, 1.0),
-        detail: detail.to_string(),
-    }
-}
-
 attack!(DomainRemoval, "domain_removal", |c| {
-    // Best prediction achievable while *removing* each domain in turn.
+    // Can the rest of the trace reconstruct each removed domain's own block?
     let mut best = 0.0;
     for d in 0..NUM_DOMAINS {
-        let cols: Vec<usize> = (0..NUM_DOMAINS)
-            .filter(|&x| x != d)
-            .flat_map(BehaviorCorpus::domain_cols)
-            .collect();
-        let sub = c.select(&cols);
-        best = f64::max(best, ols_r2(&sub, &c.y));
+        let tgt = BehaviorCorpus::block_cols(d);
+        let src = c.complement(&tgt);
+        best = f64::max(best, c.reconstruct(&src, &tgt));
     }
-    model(best, "predict with one domain removed")
+    model(best, "reconstruct a removed domain from the rest")
 });
 
 attack!(DomainMerging, "domain_merging", |c| {
-    // Merge each pair into a summed block; best prediction over pairs.
+    // Merge each pair (sum their blocks); can the merged sum reconstruct the two
+    // separate blocks? If so, distinguishing the domains is redundant.
     let mut best = 0.0;
     for a in 0..NUM_DOMAINS {
         for b in (a + 1)..NUM_DOMAINS {
-            let mut merged = Mat::zeros(c.x.rows, (NUM_DOMAINS - 1) * DOMAIN_BLOCK);
-            for r in 0..c.x.rows {
-                let mut out_col = 0;
-                for d in 0..NUM_DOMAINS {
-                    if d == b {
-                        continue;
-                    }
-                    for l in 0..DOMAIN_BLOCK {
-                        let mut v = c.x.at(r, d * DOMAIN_BLOCK + l);
-                        if d == a {
-                            v += c.x.at(r, b * DOMAIN_BLOCK + l);
-                        }
-                        merged.set(r, out_col, v);
-                        out_col += 1;
-                    }
+            let mut merged = Mat::zeros(c.traces.rows, BLOCK_W);
+            for r in 0..c.traces.rows {
+                for l in 0..BLOCK_W {
+                    let v = c.traces.at(r, a * BLOCK_W + l) + c.traces.at(r, b * BLOCK_W + l);
+                    merged.set(r, l, v);
                 }
             }
-            best = f64::max(best, ols_r2(&merged, &c.y));
+            let mut tgt_cols = BehaviorCorpus::block_cols(a);
+            tgt_cols.extend(BehaviorCorpus::block_cols(b));
+            let tgt = c.select(&tgt_cols);
+            best = f64::max(best, ols_r2(&merged, &tgt));
         }
     }
-    model(best, "predict with two domains merged")
+    model(best, "reconstruct two domains from their merged sum")
 });
 
-attack!(ConstantFolding, "constant_folding", |_c| {
-    // Folding the world to constants leaves no predictive features at all.
-    model(0.0, "world folded to constants")
+attack!(ConstantFolding, "constant_folding", true, |_c| {
+    // Folding the trace to constants reconstructs nothing.
+    model(0.0, "trace folded to constants")
 });
 
 attack!(CausalEdgeDeletion, "causal_edge_deletion", |c| {
-    // Keep only each domain's first observed lane (no cross-domain structure).
-    let cols: Vec<usize> = (0..NUM_DOMAINS).map(|d| d * DOMAIN_BLOCK).collect();
-    let sub = c.select(&cols);
-    model(ols_r2(&sub, &c.y), "diagonal-only features")
+    // Drop all cross-domain causal/flow columns; can local features
+    // (counts/deltas) reconstruct the deleted causal structure?
+    let mut deleted = Vec::new();
+    for d in 0..NUM_DOMAINS {
+        for l in 0..4 {
+            // infl_out, infl_in, flow_out, flow_in
+            deleted.push(d * BLOCK_W + l);
+        }
+    }
+    let kept = c.complement(&deleted);
+    model(c.reconstruct(&kept, &deleted), "reconstruct deleted causal edges from local features")
 });
 
-attack!(StateAliasing, "state_aliasing", |c| {
-    // Alias all domains into a single aggregate column.
-    let mut agg = Mat::zeros(c.x.rows, 1);
-    for r in 0..c.x.rows {
+attack!(StateAliasing, "state_aliasing", true, |c| {
+    // Alias the whole trace into one aggregate column; reconstruct the full trace.
+    let mut agg = Mat::zeros(c.traces.rows, 1);
+    for r in 0..c.traces.rows {
         let mut s = 0.0;
-        for col in 0..c.x.cols {
-            s += c.x.at(r, col);
+        for col in 0..c.traces.cols {
+            s += c.traces.at(r, col);
         }
         agg.set(r, 0, s);
     }
-    model(ols_r2(&agg, &c.y), "single aliased aggregate")
+    model(ols_r2(&agg, &c.traces), "reconstruct trace from a single aliased aggregate")
 });
 
-attack!(LatentFactorModeling, "latent_factor_modeling", |c| {
+attack!(LatentFactorModeling, "latent_factor_modeling", true, |c| {
     model(c.predict_k_factor(4), "top-4 latent factors")
 });
 
-attack!(BehaviorClustering, "behavior_clustering", |c| {
-    let recon = kmeans_reconstruct(&c.y, 4);
-    let scores = pca_scores(&c.x, 4);
-    model(ols_r2(&scores, &recon), "4-cluster behavior model")
+attack!(BehaviorClustering, "behavior_clustering", true, |c| {
+    let recon = kmeans_reconstruct(&c.traces, 4);
+    model(reconstruction_r2(&c.traces, &recon), "4-cluster behavior model")
 });
 
-attack!(SurrogatePrediction, "surrogate_prediction", |c| {
-    model(c.full_r2(), "full linear surrogate")
+attack!(SurrogatePrediction, "surrogate_prediction", true, |c| {
+    // A small (2-factor) linear surrogate of the whole trace.
+    model(c.predict_k_factor(2), "2-factor linear surrogate")
 });
 
 attack!(TemporalFlattening, "temporal_flattening", |c| {
-    // Drop hidden lanes (time-carrying state); predict from observed only.
-    let cols: Vec<usize> = (0..NUM_DOMAINS)
-        .flat_map(|d| (0..LANES).map(move |l| d * DOMAIN_BLOCK + l))
-        .collect();
-    let sub = c.select(&cols);
-    model(ols_r2(&sub, &c.y), "time-flattened (observed lanes only)")
+    // Drop temporal columns; can the rest reconstruct temporal reach?
+    let temporal: Vec<usize> = (0..NUM_DOMAINS).map(|d| d * BLOCK_W + 6).collect();
+    let kept = c.complement(&temporal);
+    model(c.reconstruct(&kept, &temporal), "reconstruct temporal reach from non-temporal features")
 });
 
 attack!(ObservationFlattening, "observation_flattening", |c| {
-    // Use only hidden lanes (collapse the observed surface).
-    let cols: Vec<usize> = (0..NUM_DOMAINS)
-        .flat_map(|d| (0..HIDDEN_LANES).map(move |l| d * DOMAIN_BLOCK + LANES + l))
-        .collect();
-    let sub = c.select(&cols);
-    model(ols_r2(&sub, &c.y), "observation-flattened (hidden lanes only)")
+    // Drop hidden-state delta columns; reconstruct them from the observed side.
+    let hidden: Vec<usize> = (0..NUM_DOMAINS).map(|d| d * BLOCK_W + 8).collect();
+    let kept = c.complement(&hidden);
+    model(c.reconstruct(&kept, &hidden), "reconstruct hidden deltas from observed features")
 });
 
 attack!(ExecutorIdentityErasure, "executor_identity_erasure", |c| {
-    // Predict all-but-last output dim (the divergence summary) from input.
-    if c.y.cols <= 1 {
-        return model(0.0, "no executor dim");
-    }
-    let mut y2 = Mat::zeros(c.y.rows, c.y.cols - 1);
-    for r in 0..c.y.rows {
-        for col in 0..c.y.cols - 1 {
-            y2.set(r, col, c.y.at(r, col));
+    // The divergence summary is the last global column; reconstruct it from the
+    // rest (erasing executor identity).
+    let div = vec![c.cols() - 1];
+    let kept = c.complement(&div);
+    model(c.reconstruct(&kept, &div), "reconstruct executor divergence from the rest")
+});
+
+fn reconstruction_r2(original: &Mat, recon: &Mat) -> f64 {
+    let mut ss_res = 0.0;
+    let mut ss_tot = 0.0;
+    for r in 0..original.rows {
+        for col in 0..original.cols {
+            let o = original.at(r, col);
+            ss_res += (o - recon.at(r, col)).powi(2);
+            ss_tot += o.powi(2);
         }
     }
-    model(ols_r2(&c.x, &y2), "executor identity erased")
-});
+    if ss_tot <= 1e-9 {
+        return 0.0;
+    }
+    (1.0 - ss_res / ss_tot).clamp(0.0, 1.0)
+}
 
 fn kmeans_reconstruct(y: &Mat, k: usize) -> Mat {
     let n = y.rows;
@@ -292,11 +339,11 @@ fn kmeans_reconstruct(y: &Mat, k: usize) -> Mat {
         return y.clone();
     }
     let k = k.min(n);
-    // deterministic init: spread initial centroids across the data
-    let mut centroids: Vec<Vec<f64>> = (0..k).map(|i| y.data[(i * n / k) * y.cols..(i * n / k) * y.cols + y.cols].to_vec()).collect();
+    let mut centroids: Vec<Vec<f64>> = (0..k)
+        .map(|i| y.data[(i * n / k) * y.cols..(i * n / k) * y.cols + y.cols].to_vec())
+        .collect();
     let mut assign = vec![0usize; n];
     for _ in 0..12 {
-        // assign
         for r in 0..n {
             let mut best = 0;
             let mut bestd = f64::MAX;
@@ -312,7 +359,6 @@ fn kmeans_reconstruct(y: &Mat, k: usize) -> Mat {
             }
             assign[r] = best;
         }
-        // update
         let mut sums = vec![vec![0.0; y.cols]; k];
         let mut counts = vec![0usize; k];
         for r in 0..n {
@@ -377,10 +423,16 @@ impl CollapseSummary {
 /// Run every attack and check all collapse gates.
 pub fn analyze(corpus: &BehaviorCorpus) -> CollapseSummary {
     let mut reports = Vec::new();
+    // `min_info_loss` is defined only over whole-trace compressors — the genuine
+    // "compressed model loses ≥ 35% information" gate. Ablation probes run and
+    // report, but reconstructing one facet from the rest is redundancy, not
+    // whole-trace compression, and is gated structurally elsewhere.
     let mut min_info_loss: f64 = 1.0;
     for atk in all_attacks() {
         let m = atk.compress(corpus);
-        min_info_loss = min_info_loss.min(m.info_loss);
+        if atk.whole_trace() {
+            min_info_loss = min_info_loss.min(m.info_loss);
+        }
         reports.push(CollapseReport {
             attack: atk.name().to_string(),
             predicts: m.predicts,
@@ -397,13 +449,13 @@ pub fn analyze(corpus: &BehaviorCorpus) -> CollapseSummary {
 
     let mut failures = Vec::new();
     if best_1 >= 0.40 {
-        failures.push(format!("1-factor predicts {:.3} >= 0.40", best_1));
+        failures.push(format!("1-factor reconstructs {:.3} >= 0.40", best_1));
     }
     if best_2 >= 0.55 {
-        failures.push(format!("2-factor predicts {:.3} >= 0.55", best_2));
+        failures.push(format!("2-factor reconstructs {:.3} >= 0.55", best_2));
     }
     if best_4 >= 0.70 {
-        failures.push(format!("4-factor predicts {:.3} >= 0.70", best_4));
+        failures.push(format!("4-factor reconstructs {:.3} >= 0.70", best_4));
     }
     if max_single > 0.30 {
         failures.push(format!("single domain explains {:.3} > 0.30", max_single));
@@ -430,41 +482,50 @@ pub fn analyze(corpus: &BehaviorCorpus) -> CollapseSummary {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use world_model::{Hasher, Rng};
+    use world_model::Rng;
+
+    fn random_trace_rows(n: usize, seed: u64) -> Vec<Vec<f64>> {
+        let mut rng = Rng::new(seed);
+        (0..n)
+            .map(|_| {
+                (0..FEATURE_W)
+                    .map(|_| rng.range_i64(-5000, 5000) as f64)
+                    .collect()
+            })
+            .collect()
+    }
 
     #[test]
-    fn random_behavior_resists_collapse() {
-        // Inputs random; outputs an avalanche hash of inputs -> no small linear
-        // model should predict them.
-        let mut rng = Rng::new(1);
-        let mut inputs = Vec::new();
-        let mut outputs = Vec::new();
-        let mut fps = Vec::new();
-        for _ in 0..400 {
-            let inp: Vec<f64> = (0..NUM_DOMAINS * DOMAIN_BLOCK)
-                .map(|_| rng.range_i64(-5000, 5000) as f64)
-                .collect();
-            let mut h = Hasher::new();
-            for &v in &inp {
-                h.write_i64(v as i64);
-            }
-            let base = h.finish().0;
-            let out: Vec<f64> = (0..12)
-                .map(|k| {
-                    let mut hh = Hasher::new();
-                    hh.write_u64(base);
-                    hh.write_u64(k);
-                    (hh.finish().0 as i64) as f64
-                })
-                .collect();
-            fps.push(world_model::Hash(base));
-            inputs.push(inp);
-            outputs.push(out);
-        }
-        let corpus = BehaviorCorpus::build(inputs, outputs, fps);
+    fn high_entropy_trace_resists_collapse() {
+        let corpus = BehaviorCorpus::build(random_trace_rows(400, 1));
         let summary = analyze(&corpus);
         assert!(summary.passed(), "collapse failures: {:?}", summary.failures);
-        assert!(summary.best_1factor < 0.40);
-        assert!(summary.min_info_loss >= 0.35);
+    }
+
+    /// Negative control: a single-factor (rank-1) corpus is genuinely
+    /// collapsible. Every feature is a fixed loading times one latent value plus
+    /// tiny noise, so a 1-factor model reconstructs almost everything. The gate
+    /// MUST reject it — proving the collapse analysis discriminates.
+    #[test]
+    fn single_factor_corpus_is_rejected() {
+        let mut rng = Rng::new(7);
+        let loadings: Vec<f64> = (0..FEATURE_W).map(|i| 1.0 + (i % 5) as f64).collect();
+        let rows: Vec<Vec<f64>> = (0..400)
+            .map(|_| {
+                let latent = rng.range_i64(-1000, 1000) as f64;
+                loadings
+                    .iter()
+                    .map(|&load| load * latent + rng.range_i64(-2, 2) as f64)
+                    .collect()
+            })
+            .collect();
+        let corpus = BehaviorCorpus::build(rows);
+        let summary = analyze(&corpus);
+        assert!(
+            !summary.passed(),
+            "collapse gate failed to reject a single-factor universe (1f={:.3}, info_loss={:.3})",
+            summary.best_1factor,
+            summary.min_info_loss
+        );
     }
 }