aaron
50b97e2998
Three refinements to retrieve_context, all keyed off observed failures from test_retrieval.py: - Library/personal split. classify_retrieval_intent now returns (type_filter, folder_exclude_prefixes). Biographical document intent excludes Library/* so philosophy/cognition books stop crowding out CVs and dossiers for queries like "write me a bio". - Near-duplicate collapse. Multi-folder copies of the same file (e.g., several Teaching Philosophy.pdf in different application folders) used to fill the top-N with the same content. Dedup by first-300-chars hash after rerank. - Folder in source citations. Surface metadata.folder alongside basename so the LLM can disambiguate among 21 CV.docx variants and the user can see which copy a citation refers to. Also: bump hnsw.ef_search to 500 when a WHERE filter is present. pgvector 0.6 doesn't iterate past its initial HNSW candidate list, so a restrictive filter that excludes the nearest neighbors otherwise returns empty.
61 lines
2.1 KiB
Python
61 lines
2.1 KiB
Python
"""End-to-end test of retrieve_context with intent routing + reranking.
|
|
|
|
Avoids loading the full FastAPI app; replicates the chat-handler retrieval
|
|
call shape and prints classifier output + final ranked sources for each query.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from dotenv import load_dotenv
|
|
load_dotenv(Path.home() / "aaronai" / ".env", override=True)
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
# Stub anthropic so api.py import doesn't fail without the SDK loaded.
|
|
# We only need retrieve_context + classify_retrieval_intent.
|
|
import types
|
|
sys.modules.setdefault("anthropic", types.ModuleType("anthropic"))
|
|
sys.modules["anthropic"].Anthropic = lambda **kw: None
|
|
|
|
# Same for whisper if present
|
|
if "faster_whisper" not in sys.modules:
|
|
sys.modules["faster_whisper"] = types.ModuleType("faster_whisper")
|
|
|
|
import importlib.util
|
|
spec = importlib.util.spec_from_file_location("api", Path(__file__).parent / "api.py")
|
|
api = importlib.util.module_from_spec(spec)
|
|
# Don't execute the whole module (it starts FastAPI). Instead, exec only definitions.
|
|
# Easier: just import the functions we need by exec'ing the file but catching errors.
|
|
try:
|
|
spec.loader.exec_module(api)
|
|
except Exception as e:
|
|
print(f"(continuing despite api.py side-effect error: {e})")
|
|
|
|
retrieve_context = api.retrieve_context
|
|
classify_retrieval_intent = api.classify_retrieval_intent
|
|
|
|
QUERIES = [
|
|
"write me a bio",
|
|
"my professional bio",
|
|
"draft a bio for the Utah application",
|
|
"Aaron Nelson CV consulting and design work",
|
|
"FWN3D consulting",
|
|
"syllabi I have taught",
|
|
"philosophy of teaching",
|
|
"what did I tell Claude about FWN3D",
|
|
"what did we discuss about the Utah job",
|
|
"Hudson Valley Additive Manufacturing Center",
|
|
]
|
|
|
|
for q in QUERIES:
|
|
type_filter, folder_excludes = classify_retrieval_intent(q)
|
|
pieces, sources = retrieve_context(
|
|
q, type_filter=type_filter, folder_exclude_prefixes=folder_excludes,
|
|
)
|
|
print(f"\n=== {q!r} ===")
|
|
print(f" type_filter: {type_filter} folder_excludes: {folder_excludes}")
|
|
for i, src in enumerate(sources, 1):
|
|
print(f" {i}. {src}")
|