aaronAI/scripts/experiments/e1_compare_metrics.py

#!/usr/bin/env python3
"""E1 metrics comparison — A (Tier 1 aaron) vs B (cascade aaron_cascade_test) on the 10 sample sources."""
import json
import subprocess
from pathlib import Path

EXPERIMENTS = Path.home() / "aaronai" / "experiments"
SAMPLE_FILE = EXPERIMENTS / "cascade_reextract_sample.json"
COMPARISON_FILE = EXPERIMENTS / "cascade_reextract_comparison.json"

def query(group_id, cypher):
    result = subprocess.run(
        ["docker", "exec", "falkordb", "redis-cli", "GRAPH.QUERY", group_id, cypher],
        capture_output=True, text=True
    )
    return result.stdout

def parse_int_result(output):
    """Parse a single-integer result from redis-cli GRAPH.QUERY output."""
    lines = [l.strip() for l in output.split("\n") if l.strip()]
    for line in lines:
        if line.isdigit():
            return int(line)
    return 0

def parse_string_list(output):
    """Parse a list of strings from redis-cli output (skipping headers and timing)."""
    lines = [l.strip() for l in output.split("\n") if l.strip()]
    items = []
    started = False
    for line in lines:
        if line.startswith("Cached") or line.startswith("Query internal"):
            break
        if started:
            items.append(line)
        # The header is the column name; everything after is data
        # But we don't know column names a priori, so detect transition by length pattern
        if not started and len(line) < 60 and not any(c in line for c in "{}[]"):
            # Likely a header row, skip first one
            started = True
    return items

def metrics_for_source(group_id, source_name):
    """Get metrics for one source's episode in one group_id."""
    # Total entities connected to this episode
    q = f'MATCH (e:Episodic {{name: "{source_name}"}})-[]-(n:Entity) RETURN count(distinct n) AS entities'
    entities = parse_int_result(query(group_id, q))

    # Total edges from this episode (all relationship types)
    q = f'MATCH (e:Episodic {{name: "{source_name}"}})-[r]-() RETURN count(r) AS edges'
    edges = parse_int_result(query(group_id, q))

    # Distinct relationship types in edges from entities of this episode
    q = (f'MATCH (e:Episodic {{name: "{source_name}"}})-[]-(n:Entity)-[r]-() '
         f'RETURN count(distinct type(r)) AS types')
    rel_types = parse_int_result(query(group_id, q))

    return {"entities": entities, "edges": edges, "rel_types": rel_types}

def main():
    with open(SAMPLE_FILE) as f:
        sample = json.load(f)
    selected = sample["selected"]

    print(f"E1 metrics comparison — {len(selected)} sources, A=aaron vs B=aaron_cascade_test\n")
    print(f"{'Source':<60} {'A.ent':>6} {'B.ent':>6} {'A.edg':>6} {'B.edg':>6} {'A.typ':>6} {'B.typ':>6}")
    print("-" * 110)

    results = []
    for ep in selected:
        name = ep["name"]
        bucket = ep["bucket"]
        a = metrics_for_source("aaron", name)
        b = metrics_for_source("aaron_cascade_test", name)
        record = {
            "name": name, "bucket": bucket,
            "a_entities": a["entities"], "b_entities": b["entities"],
            "a_edges": a["edges"], "b_edges": b["edges"],
            "a_rel_types": a["rel_types"], "b_rel_types": b["rel_types"],
        }
        results.append(record)
        # Truncate name for display
        display_name = name if len(name) <= 58 else name[:55] + "..."
        print(f"{display_name:<60} {a['entities']:>6} {b['entities']:>6} {a['edges']:>6} {b['edges']:>6} {a['rel_types']:>6} {b['rel_types']:>6}")

    # Aggregates
    print("\n" + "=" * 110)
    n = len(results)
    a_ent_sum = sum(r["a_entities"] for r in results)
    b_ent_sum = sum(r["b_entities"] for r in results)
    a_edge_sum = sum(r["a_edges"] for r in results)
    b_edge_sum = sum(r["b_edges"] for r in results)
    a_types_sum = sum(r["a_rel_types"] for r in results)
    b_types_sum = sum(r["b_rel_types"] for r in results)
    print(f"\nAggregate (n={n}):")
    print(f"  Entities:  A mean={a_ent_sum/n:.1f}  B mean={b_ent_sum/n:.1f}  delta={(b_ent_sum-a_ent_sum)/a_ent_sum*100:+.1f}%")
    print(f"  Edges:     A mean={a_edge_sum/n:.1f}  B mean={b_edge_sum/n:.1f}  delta={(b_edge_sum-a_edge_sum)/a_edge_sum*100:+.1f}%")
    print(f"  Rel types: A mean={a_types_sum/n:.1f}  B mean={b_types_sum/n:.1f}  delta={(b_types_sum-a_types_sum)/a_types_sum*100:+.1f}%")

    # Global predicate diversity check (unique types in each group_id)
    print(f"\nGlobal predicate diversity:")
    a_global = parse_int_result(query("aaron", "MATCH ()-[r]-() RETURN count(distinct type(r)) AS t"))
    b_global = parse_int_result(query("aaron_cascade_test", "MATCH ()-[r]-() RETURN count(distinct type(r)) AS t"))
    print(f"  A (aaron):              {a_global} distinct relationship types across whole graph")
    print(f"  B (aaron_cascade_test): {b_global} distinct relationship types across whole graph")

    # Per-bucket
    print(f"\nPer-bucket aggregates:")
    for bucket in ["high", "mid", "low", "document"]:
        bucket_results = [r for r in results if r["bucket"] == bucket]
        if not bucket_results:
            continue
        bn = len(bucket_results)
        a_e = sum(r["a_entities"] for r in bucket_results) / bn
        b_e = sum(r["b_entities"] for r in bucket_results) / bn
        a_ed = sum(r["a_edges"] for r in bucket_results) / bn
        b_ed = sum(r["b_edges"] for r in bucket_results) / bn
        print(f"  [{bucket:>8}] n={bn}  A.ent={a_e:.1f} B.ent={b_e:.1f} ({(b_e-a_e)/a_e*100:+.0f}%)  "
              f"A.edg={a_ed:.1f} B.edg={b_ed:.1f} ({(b_ed-a_ed)/a_ed*100:+.0f}%)")

    with open(COMPARISON_FILE, "w") as f:
        json.dump({
            "results": results,
            "aggregate": {
                "a_entities_total": a_ent_sum, "b_entities_total": b_ent_sum,
                "a_edges_total": a_edge_sum, "b_edges_total": b_edge_sum,
                "global_predicate_diversity": {"a": a_global, "b": b_global},
            },
        }, f, indent=2)
    print(f"\nSaved to {COMPARISON_FILE}")


if __name__ == "__main__":
    main()