aaronAI/scripts/experiments/e16_analyze.py

"""
E1.6 analysis — correlate domain-purity ratings with cascade outcomes.
Applies pre-registered decision rules from E1.6 protocol.
"""
import json
from collections import defaultdict

RATINGS_PATH = "/home/aaron/aaronai/experiments/e16_purity_ratings.json"
COMPARISON_PATH = "/home/aaron/aaronai/experiments/e14_per_source_comparison.json"


def spearman(xs, ys):
    """Compute Spearman rank correlation."""
    n = len(xs)
    if n < 2:
        return None
    # Rank the values
    def rank(values):
        sorted_idx = sorted(range(len(values)), key=lambda i: values[i])
        ranks = [0] * len(values)
        i = 0
        while i < len(values):
            j = i
            while j + 1 < len(values) and values[sorted_idx[j+1]] == values[sorted_idx[i]]:
                j += 1
            avg_rank = (i + j) / 2 + 1
            for k in range(i, j + 1):
                ranks[sorted_idx[k]] = avg_rank
            i = j + 1
        return ranks
    rx = rank(xs)
    ry = rank(ys)
    mean_rx = sum(rx) / n
    mean_ry = sum(ry) / n
    num = sum((rx[i] - mean_rx) * (ry[i] - mean_ry) for i in range(n))
    den_x = (sum((rx[i] - mean_rx) ** 2 for i in range(n))) ** 0.5
    den_y = (sum((ry[i] - mean_ry) ** 2 for i in range(n))) ** 0.5
    if den_x == 0 or den_y == 0:
        return None
    return num / (den_x * den_y)


def main():
    with open(RATINGS_PATH) as f:
        ratings_data = json.load(f)
    with open(COMPARISON_PATH) as f:
        comparisons = json.load(f)

    ratings_by_name = {r['name']: r for r in ratings_data['ratings']}
    comp_by_name = {c['name']: c for c in comparisons}

    # Join ratings with cascade outcomes
    joined = []
    for name, rating in ratings_by_name.items():
        if name in comp_by_name:
            comp = comp_by_name[name]
            joined.append({
                'name': name,
                'binary': rating['binary'],
                'score': rating['score'],
                'note': rating.get('note'),
                'bucket': comp['bucket'],
                'delta_preds': comp['delta_preds'],
                'delta_edges': comp['delta_edges'],
                'prod_preds': comp['prod_preds'],
                'cascade_preds': comp['cascade_preds'],
            })

    print("=" * 100)
    print(f"E1.6 ANALYSIS — Domain Purity vs Cascade Outcome (n={len(joined)})")
    print("=" * 100)

    # Per-source detail with rating
    print()
    print(f"{'Bucket':<10} {'Source':<48} {'Domain':<8} {'Score':<6} {'Δpreds':<8} {'Δedges':<8}")
    print("-" * 100)
    for j in sorted(joined, key=lambda x: (x['binary'], -x['score'], x['bucket'], x['name'])):
        name_short = (j['name'][:45] + '..') if len(j['name']) > 48 else j['name']
        print(f"{j['bucket']:<10} {name_short:<48} {j['binary']:<8} {j['score']:<6} {j['delta_preds']:+d}     {j['delta_edges']:+d}")

    # PRIMARY TEST: binary purity vs cascade outcome distribution
    print()
    print("=" * 100)
    print("PRIMARY TEST: Binary purity vs cascade outcome distribution")
    print("=" * 100)

    def categorize_outcome(delta):
        if delta > 0:
            return 'positive'
        elif delta < 0:
            return 'negative'
        else:
            return 'flat'

    by_binary = defaultdict(lambda: {'positive': 0, 'flat': 0, 'negative': 0, 'total': 0})
    for j in joined:
        outcome = categorize_outcome(j['delta_preds'])
        by_binary[j['binary']][outcome] += 1
        by_binary[j['binary']]['total'] += 1

    print()
    print(f"{'Group':<15} {'n':<5} {'Positive':<12} {'Flat':<10} {'Negative':<12}")
    print("-" * 60)
    for binary in ['single', 'multi']:
        d = by_binary[binary]
        n = d['total']
        if n == 0:
            continue
        pos_pct = d['positive'] / n * 100
        flat_pct = d['flat'] / n * 100
        neg_pct = d['negative'] / n * 100
        print(f"{binary+'-domain':<15} {n:<5} {d['positive']} ({pos_pct:.0f}%)    {d['flat']} ({flat_pct:.0f}%)    {d['negative']} ({neg_pct:.0f}%)")

    # Compute the gap
    if by_binary['single']['total'] > 0 and by_binary['multi']['total'] > 0:
        single_pos_rate = by_binary['single']['positive'] / by_binary['single']['total'] * 100
        multi_pos_rate = by_binary['multi']['positive'] / by_binary['multi']['total'] * 100
        gap = single_pos_rate - multi_pos_rate
        print()
        print(f"Cascade-positive rate gap (single - multi): {gap:+.1f} percentage points")
        print()
        # Apply pre-registered decision rule
        if gap >= 20:
            verdict = "NARROWNESS HYPOTHESIS SUPPORTED"
            detail = f"Single-domain content is {gap:.0f}pp more likely to gain from cascade than multi-domain."
        elif gap <= -20:
            verdict = "REVERSE OF HYPOTHESIS"
            detail = f"Multi-domain content unexpectedly benefits more (counter to prediction)."
        elif abs(gap) < 10:
            verdict = "HYPOTHESIS NOT SUPPORTED"
            detail = "Domain purity does not appear to predict cascade outcome."
        else:
            verdict = "INCONCLUSIVE"
            detail = f"Gap of {gap:+.0f}pp is suggestive but below the pre-registered 20pp threshold."
        print(f"  Pre-registered decision rule: {verdict}")
        print(f"  {detail}")

    # SECONDARY TEST: Spearman correlation between purity score and predicate delta
    print()
    print("=" * 100)
    print("SECONDARY TEST: Spearman rank correlation (purity score vs predicate delta)")
    print("=" * 100)

    scores = [j['score'] for j in joined]
    deltas_pred = [j['delta_preds'] for j in joined]
    deltas_edge = [j['delta_edges'] for j in joined]

    rho_pred = spearman(scores, deltas_pred)
    rho_edge = spearman(scores, deltas_edge)

    print()
    print(f"  Spearman ρ (purity score vs Δpredicates): {rho_pred:.3f}")
    print(f"  Spearman ρ (purity score vs Δedges):      {rho_edge:.3f}")
    print()

    if rho_pred is not None:
        if rho_pred >= 0.4:
            v = "STRONG POSITIVE — narrowness hypothesis supported with monotonic relationship"
        elif rho_pred >= 0.2:
            v = "WEAK POSITIVE — consistent with hypothesis but not strong evidence"
        elif rho_pred <= -0.2:
            v = "NEGATIVE — refutes hypothesis"
        else:
            v = "NO CORRELATION — hypothesis not supported"
        print(f"  Predicate delta verdict: {v}")
    print()

    # TERTIARY TEST: within-bucket correlation
    print()
    print("=" * 100)
    print("TERTIARY TEST: Within-bucket correlation")
    print("=" * 100)

    by_bucket = defaultdict(list)
    for j in joined:
        by_bucket[j['bucket']].append(j)

    print()
    print(f"{'Bucket':<12} {'n':<5} {'Single':<10} {'Multi':<10} {'ρ (score vs Δpred)':<22}")
    print("-" * 75)
    for bucket in ['high', 'mid', 'low', 'document']:
        items = by_bucket.get(bucket, [])
        if not items:
            continue
        n = len(items)
        n_single = sum(1 for j in items if j['binary'] == 'single')
        n_multi = sum(1 for j in items if j['binary'] == 'multi')
        if n >= 3:
            scores_b = [j['score'] for j in items]
            deltas_b = [j['delta_preds'] for j in items]
            rho_b = spearman(scores_b, deltas_b)
            rho_str = f"{rho_b:+.3f}" if rho_b is not None else "n/a (no variance)"
        else:
            rho_str = "n/a (too few)"
        print(f"{bucket:<12} {n:<5} {n_single:<10} {n_multi:<10} {rho_str}")

    # Interaction with bucket: do single/multi outcomes differ within bucket?
    print()
    print("Per-bucket cascade-positive rate by binary purity:")
    print()
    print(f"{'Bucket':<12} {'Single':<25} {'Multi':<25}")
    print("-" * 65)
    for bucket in ['high', 'mid', 'low', 'document']:
        items = by_bucket.get(bucket, [])
        if not items:
            continue
        single_items = [j for j in items if j['binary'] == 'single']
        multi_items = [j for j in items if j['binary'] == 'multi']
        def rate_str(group):
            if not group:
                return "—"
            pos = sum(1 for j in group if j['delta_preds'] > 0)
            return f"{pos}/{len(group)} positive ({pos/len(group)*100:.0f}%)"
        print(f"{bucket:<12} {rate_str(single_items):<25} {rate_str(multi_items):<25}")

    # MEAN DELTA by binary group
    print()
    print("=" * 100)
    print("MEAN PREDICATE DELTA BY GROUP")
    print("=" * 100)
    print()
    for binary in ['single', 'multi']:
        items = [j for j in joined if j['binary'] == binary]
        if not items:
            continue
        n = len(items)
        mean_dp = sum(j['delta_preds'] for j in items) / n
        mean_de = sum(j['delta_edges'] for j in items) / n
        sum_pp = sum(j['prod_preds'] for j in items)
        sum_cp = sum(j['cascade_preds'] for j in items)
        pct_change = (sum_cp - sum_pp) / sum_pp * 100 if sum_pp else 0
        print(f"{binary}-domain (n={n}):")
        print(f"  Mean Δpredicates per source: {mean_dp:+.2f}")
        print(f"  Mean Δedges per source:      {mean_de:+.2f}")
        print(f"  Aggregate predicate change:   {sum_pp} → {sum_cp} ({pct_change:+.1f}%)")
        print()

    # Save joined data for the experiments log writeup
    out_path = "/home/aaron/aaronai/experiments/e16_joined_analysis.json"
    with open(out_path, "w") as f:
        json.dump(joined, f, indent=2)
    print(f"Joined data saved to {out_path}")


if __name__ == "__main__":
    main()