chore: archive deprecated chromadb and migration scripts
This commit is contained in:
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
Aaron AI — Migration: pgvector to Graphiti
|
||||
One-time migration. Test with limit first: python3 migrate_to_graphiti.py 100
|
||||
"""
|
||||
import os, sys, json, time, requests, psycopg2
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(Path.home() / "aaronai" / ".env")
|
||||
|
||||
GRAPHITI_URL = "http://localhost:8001"
|
||||
PG_DSN = os.getenv("PG_DSN")
|
||||
GROUP_ID = "aaron"
|
||||
BATCH_PAUSE = 0.5
|
||||
PROGRESS_FILE = Path.home() / "aaronai" / "migration_progress.json"
|
||||
|
||||
def load_progress():
|
||||
if PROGRESS_FILE.exists():
|
||||
return json.loads(PROGRESS_FILE.read_text())
|
||||
return {"completed_ids": [], "failed_ids": []}
|
||||
|
||||
def save_progress(progress):
|
||||
PROGRESS_FILE.write_text(json.dumps(progress, indent=2))
|
||||
|
||||
def migrate(limit=None):
|
||||
try:
|
||||
resp = requests.get(f"{GRAPHITI_URL}/health", timeout=5)
|
||||
print(f"Graphiti: {resp.json()}")
|
||||
except Exception as e:
|
||||
print(f"ERROR: sidecar not reachable — {e}"); sys.exit(1)
|
||||
|
||||
progress = load_progress()
|
||||
completed_ids = set(progress["completed_ids"])
|
||||
failed_ids = progress["failed_ids"]
|
||||
if completed_ids:
|
||||
print(f"Resuming — {len(completed_ids)} done, {len(failed_ids)} failed")
|
||||
|
||||
pg = psycopg2.connect(PG_DSN)
|
||||
cur = pg.cursor()
|
||||
query = "SELECT id, document, source, created_at FROM embeddings ORDER BY created_at ASC"
|
||||
if limit:
|
||||
query += f" LIMIT {limit}"
|
||||
cur.execute(query)
|
||||
rows = cur.fetchall()
|
||||
pg.close()
|
||||
|
||||
pending = [r for r in rows if r[0] not in completed_ids]
|
||||
print(f"Total: {len(rows)} | Pending: {len(pending)}{' [TEST]' if limit else ''}\n")
|
||||
|
||||
success = len(completed_ids)
|
||||
failed = len(failed_ids)
|
||||
start = time.time()
|
||||
|
||||
for i, (id, document, source, created_at) in enumerate(pending):
|
||||
try:
|
||||
src = (source or "unknown").replace("/", "-").replace(" ", "-")[:80]
|
||||
name = f"{src}-{id[:8]}"
|
||||
requests.post(f"{GRAPHITI_URL}/episodes", json={
|
||||
"name": name,
|
||||
"content": document,
|
||||
"source_description": source or "nextcloud-corpus",
|
||||
"timestamp": created_at or datetime.now().isoformat(),
|
||||
"group_id": GROUP_ID,
|
||||
}, timeout=120).raise_for_status()
|
||||
success += 1
|
||||
progress["completed_ids"].append(id)
|
||||
if success % 10 == 0:
|
||||
save_progress(progress)
|
||||
if (i + 1) % 50 == 0:
|
||||
elapsed = time.time() - start
|
||||
rate = (i + 1) / elapsed
|
||||
remaining = (len(pending) - i - 1) / rate if rate > 0 else 0
|
||||
print(f" [{i+1}/{len(pending)}] {success} ok, {failed} failed | ~{remaining/60:.0f} min left")
|
||||
time.sleep(BATCH_PAUSE)
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
progress["failed_ids"].append({"id": id, "error": str(e)})
|
||||
print(f" FAILED {id}: {e}")
|
||||
save_progress(progress)
|
||||
time.sleep(2)
|
||||
|
||||
save_progress(progress)
|
||||
elapsed = time.time() - start
|
||||
print(f"\nDone — {success} ok, {failed} failed, {elapsed/60:.1f} min")
|
||||
if limit and len(pending) > 0:
|
||||
est = (elapsed / len(pending)) * 12915 / 60
|
||||
print(f"Estimated full run: ~{est:.0f} min")
|
||||
|
||||
if __name__ == "__main__":
|
||||
migrate(int(sys.argv[1]) if len(sys.argv) > 1 else None)
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"last_dream_timestamp": 1777276868.016728,
|
||||
"last_dream_mode": "pipeline",
|
||||
"last_dream_file": "Journal/Dreams/2026-04-27-synthesis-1.md"
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
{
|
||||
"completed_ids": [
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_16",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_70",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_12",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_80",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_68",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_46",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_2",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_78",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_66",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_6",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_50",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_44",
|
||||
"claude_33e44682-8e4f-42e4-974f-cf469f841f32_64",
|
||||
"claude_742dbbac-7ea4-4363-bde3-a86e241110df_2",
|
||||
"claude_742dbbac-7ea4-4363-bde3-a86e241110df_8",
|
||||
"claude_742dbbac-7ea4-4363-bde3-a86e241110df_24",
|
||||
"claude_8fd584a3-d7f4-4af4-be63-f2071049e3e2_2",
|
||||
"claude_2d8b5404-1c3e-42c1-8e24-507560ee6d6f_2",
|
||||
"claude_1eb9040b-6e0f-434f-8dcc-50f68138db6d_4",
|
||||
"claude_1eb9040b-6e0f-434f-8dcc-50f68138db6d_8",
|
||||
"claude_1eb9040b-6e0f-434f-8dcc-50f68138db6d_9",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_126",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_174",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_8",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_20",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_22",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_36",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_184",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_24",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_182",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_178",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_180",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_38",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_124",
|
||||
"claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_40"
|
||||
],
|
||||
"failed_ids": [
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_72",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_52",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_60",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_81",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_18",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_32",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_58",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_48",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_62",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_20",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_24",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_54",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_22",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_30",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_56",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_76",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_74",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_28",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_14",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_38",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_8",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_40",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_4",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_42",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_34",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_26",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_10",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_33e44682-8e4f-42e4-974f-cf469f841f32_36",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_4",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_6",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_32",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_10",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_26",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_20",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_18",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_30",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_14",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_22",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_16",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_12",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_36",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_34",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_742dbbac-7ea4-4363-bde3-a86e241110df_28",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_8fd584a3-d7f4-4af4-be63-f2071049e3e2_5",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_8fd584a3-d7f4-4af4-be63-f2071049e3e2_4",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_2d8b5404-1c3e-42c1-8e24-507560ee6d6f_4",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_2d8b5404-1c3e-42c1-8e24-507560ee6d6f_6",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_2d8b5404-1c3e-42c1-8e24-507560ee6d6f_9",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_2d8b5404-1c3e-42c1-8e24-507560ee6d6f_8",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_1eb9040b-6e0f-434f-8dcc-50f68138db6d_2",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_1eb9040b-6e0f-434f-8dcc-50f68138db6d_6",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_2",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_4",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_6",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_28",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_26",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_30",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_16",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_18",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_10",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_34",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_12",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_14",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_32",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
},
|
||||
{
|
||||
"id": "claude_dacf89e3-1ee7-400d-8461-ef5920c82fe3_42",
|
||||
"error": "500 Server Error: Internal Server Error for url: http://localhost:8001/episodes"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -565,6 +565,7 @@ async def get_status(auth: str = Depends(require_auth)):
|
||||
|
||||
# Watcher status
|
||||
watcher_running = False
|
||||
watcher_ingestion = {"status": "idle", "message": "", "file_count": 0}
|
||||
last_indexed = "Unknown"
|
||||
try:
|
||||
import time as _time, json as _json
|
||||
@@ -573,6 +574,7 @@ async def get_status(auth: str = Depends(require_auth)):
|
||||
_s = _json.loads(_sp.read_text())
|
||||
_age = _time.time() - _s.get("timestamp", 0)
|
||||
watcher_running = _s.get("running", False) and _age < 30
|
||||
watcher_ingestion = _s.get("ingestion", watcher_ingestion)
|
||||
except:
|
||||
pass
|
||||
|
||||
@@ -613,6 +615,7 @@ async def get_status(auth: str = Depends(require_auth)):
|
||||
return JSONResponse({
|
||||
"aaron_ai": "running",
|
||||
"watcher": "running" if watcher_running else "stopped",
|
||||
"watcher_ingestion": watcher_ingestion,
|
||||
"chunk_count": chunk_count,
|
||||
"file_count": file_count,
|
||||
"last_indexed": last_indexed,
|
||||
|
||||
@@ -145,7 +145,6 @@ def ingest_folder(folder_path):
|
||||
total_chunks += ingest_file(f)
|
||||
|
||||
print(f"\nDone. Total chunks indexed: {total_chunks}")
|
||||
print(f"Database stored at: {db_path}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
target = sys.argv[1] if len(sys.argv) > 1 else str(Path.home() / "aaronai" / "docs")
|
||||
|
||||
+86
-10
@@ -2,6 +2,7 @@ import time
|
||||
import subprocess
|
||||
import logging
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from watchdog.observers import Observer
|
||||
from watchdog.events import FileSystemEventHandler
|
||||
@@ -25,16 +26,35 @@ logging.basicConfig(
|
||||
]
|
||||
)
|
||||
|
||||
ingestion_state = {
|
||||
"status": "idle",
|
||||
"message": "",
|
||||
"file_count": 0,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"last_error": "",
|
||||
}
|
||||
ingestion_lock = threading.Lock()
|
||||
ingestion_thread = None
|
||||
|
||||
|
||||
def set_ingestion_state(**kwargs):
|
||||
with ingestion_lock:
|
||||
ingestion_state.update(kwargs)
|
||||
|
||||
|
||||
def load_state():
|
||||
if Path(STATE_FILE).exists():
|
||||
with open(STATE_FILE) as f:
|
||||
return json.load(f)
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(state):
|
||||
with open(STATE_FILE, 'w') as f:
|
||||
json.dump(state, f)
|
||||
|
||||
|
||||
def get_changed_files():
|
||||
state = load_state()
|
||||
changed = []
|
||||
@@ -52,13 +72,25 @@ def get_changed_files():
|
||||
changed.append(path)
|
||||
return changed, state
|
||||
|
||||
|
||||
def run_ingestion():
|
||||
changed, state = get_changed_files()
|
||||
if not changed:
|
||||
logging.info("No new or changed files detected — skipping ingestion.")
|
||||
set_ingestion_state(status="idle", message="No changes detected", file_count=0)
|
||||
return
|
||||
|
||||
logging.info(f"Found {len(changed)} new or changed files — starting ingestion...")
|
||||
count = len(changed)
|
||||
logging.info(f"Found {count} new or changed files — starting ingestion...")
|
||||
set_ingestion_state(
|
||||
status="ingesting",
|
||||
message=f"Ingesting {count} file(s)...",
|
||||
file_count=count,
|
||||
started_at=time.time(),
|
||||
finished_at=None,
|
||||
last_error="",
|
||||
)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[PYTHON, INGEST_SCRIPT, NEXTCLOUD_PATH],
|
||||
@@ -67,19 +99,51 @@ def run_ingestion():
|
||||
timeout=1800
|
||||
)
|
||||
if result.returncode == 0:
|
||||
# Update state with new mtimes
|
||||
root = Path(NEXTCLOUD_PATH)
|
||||
for path in root.rglob("*"):
|
||||
if path.is_file() and path.suffix.lower() in SUPPORTED:
|
||||
state[str(path)] = str(path.stat().st_mtime)
|
||||
save_state(state)
|
||||
logging.info("Ingestion complete. State updated.")
|
||||
set_ingestion_state(
|
||||
status="idle",
|
||||
message=f"Last run: ingested {count} file(s) successfully",
|
||||
finished_at=time.time(),
|
||||
)
|
||||
else:
|
||||
logging.error(f"Ingestion error: {result.stderr}")
|
||||
set_ingestion_state(
|
||||
status="error",
|
||||
message="Ingestion failed — see log",
|
||||
last_error=result.stderr[-300:],
|
||||
finished_at=time.time(),
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
logging.error("Ingestion timed out.")
|
||||
set_ingestion_state(
|
||||
status="error",
|
||||
message="Ingestion timed out (>30 min)",
|
||||
last_error="TimeoutExpired",
|
||||
finished_at=time.time(),
|
||||
)
|
||||
except Exception as e:
|
||||
logging.error(f"Ingestion failed: {e}")
|
||||
set_ingestion_state(
|
||||
status="error",
|
||||
message=f"Ingestion exception: {e}",
|
||||
last_error=str(e),
|
||||
finished_at=time.time(),
|
||||
)
|
||||
|
||||
|
||||
def start_ingestion_thread():
|
||||
global ingestion_thread
|
||||
if ingestion_thread and ingestion_thread.is_alive():
|
||||
logging.info("Ingestion already running — skipping.")
|
||||
return
|
||||
ingestion_thread = threading.Thread(target=run_ingestion, daemon=True)
|
||||
ingestion_thread.start()
|
||||
|
||||
|
||||
class IngestHandler(FileSystemEventHandler):
|
||||
def __init__(self):
|
||||
@@ -98,9 +162,26 @@ class IngestHandler(FileSystemEventHandler):
|
||||
return
|
||||
if 'Journal/Media' in str(path):
|
||||
return
|
||||
if event.event_type not in ('modified', 'created', 'moved'):
|
||||
return
|
||||
logging.info(f"Event: {event.event_type} {event.src_path}")
|
||||
self.pending = True
|
||||
self.last_event = time.time()
|
||||
|
||||
|
||||
def write_status(handler):
|
||||
with ingestion_lock:
|
||||
status = {
|
||||
"running": True,
|
||||
"timestamp": time.time(),
|
||||
"pending": handler.pending,
|
||||
"last_event": handler.last_event,
|
||||
"ingestion": dict(ingestion_state),
|
||||
}
|
||||
with open(STATUS_FILE, 'w') as f:
|
||||
json.dump(status, f)
|
||||
|
||||
|
||||
def main():
|
||||
logging.info("Aaron AI Watcher starting...")
|
||||
logging.info(f"Watching: {NEXTCLOUD_PATH}")
|
||||
@@ -112,23 +193,18 @@ def main():
|
||||
|
||||
try:
|
||||
while True:
|
||||
import json as _json
|
||||
_json.dump({
|
||||
"running": True,
|
||||
"timestamp": time.time(),
|
||||
"pending": handler.pending,
|
||||
"last_event": handler.last_event
|
||||
}, open(STATUS_FILE, 'w'))
|
||||
write_status(handler)
|
||||
if handler.pending:
|
||||
elapsed = time.time() - handler.last_event
|
||||
if elapsed >= DEBOUNCE_SECONDS:
|
||||
handler.pending = False
|
||||
run_ingestion()
|
||||
start_ingestion_thread()
|
||||
time.sleep(5)
|
||||
except KeyboardInterrupt:
|
||||
observer.stop()
|
||||
observer.join()
|
||||
logging.info("Watcher stopped.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user