M18: Evaluation & Testing

# eval_dataset.json — one entry per golden case
# WHAT: Each case captures input, expected tool sequence, and pass criteria
# WHY: Separating test data from test logic lets domain experts update
#      cases without touching Python
# GOTCHA: Keep expected_min_confidence realistic for your model tier —
#         Mistral-7B scores ~0.05 lower than Mistral-large on average

EVAL_SCHEMA = {
    "cases": [
        {
            "id": "ER-001",
            "description": "Obvious match — same company, different punctuation",
            "input": "Are 'Acme Logistics LLC' and 'ACME LOGISTICS, L.L.C.' the same entity?",
            "expected_tools_called": [
                "search_filings_by_name",
                "fuzzy_match_score",
                "get_business_registry_data",
                "merge_entity_profile"
            ],
            "expected_output": {
                "decision": "merge",
                "expected_min_confidence": 0.90,
                "must_contain_keywords": ["canonical", "match"]
            }
        },
        {
            "id": "ER-002",
            "description": "Ambiguous case — similar names, different businesses",
            "input": "Is 'Apex Capital Partners LP' the same as 'Apex Capital LLC'?",
            "expected_tools_called": [
                "search_filings_by_name",
                "fuzzy_match_score",
                "get_filing_details"
            ],
            "expected_output": {
                "decision": "no_merge",
                "expected_max_confidence": 0.70,
                "must_contain_keywords": ["different", "separate"]
            }
        },
        {
            "id": "ER-003",
            "description": "DBA (doing business as) alias detection",
            "input": "Is 'TechBridge Solutions' the same entity as 'TechBridge Solutions dba CloudBridge'?",
            "expected_tools_called": [
                "search_filings_by_name",
                "get_business_registry_data",
                "merge_entity_profile"
            ],
            "expected_output": {
                "decision": "merge",
                "expected_min_confidence": 0.85,
                "must_contain_keywords": ["alias", "dba"]
            }
        }
    ]
}

# WHAT: Loader function with Ollama availability guard
# WHY: Evals should fail gracefully — a missing Ollama server should
#      produce a clear error, not a cryptic connection timeout
# GOTCHA: Always check Ollama health before starting the eval run —
#         a silent Ollama failure corrupts your entire results file

import json
import sys
import httpx

def load_eval_cases(path: str = "eval_dataset.json") -> list[dict]:
    """Load golden test cases with validation."""
    try:
        with open(path) as f:
            data = json.load(f)
        cases = data.get("cases", [])
        if not cases:
            raise ValueError("Eval dataset is empty — add cases before running")
        print(f"Loaded {len(cases)} eval cases from {path}")
        return cases
    except FileNotFoundError:
        print(f"ERROR: {path} not found. Run `python build_eval_dataset.py` first.")
        sys.exit(1)

def check_ollama_health(base_url: str = "http://localhost:11434") -> bool:
    """Verify Ollama is running before spending time on evals."""
    try:
        resp = httpx.get(f"{base_url}/api/tags", timeout=3.0)
        models = [m["name"] for m in resp.json().get("models", [])]
        if "mistral" not in " ".join(models):
            print("WARNING: 'mistral' model not found. Run: ollama pull mistral")
            return False
        return True
    except (httpx.ConnectError, httpx.TimeoutException):
        print("ERROR: Ollama is not running. Start it with: ollama serve")
        return False

# tests/test_tools_unit.py
# WHAT: Parametrized unit tests for all five entity resolution tools
# WHY: Parametrize lets you add new cases without new test functions —
#      one failing case won't abort the rest
# GOTCHA: entity_tools.py returns None on not-found, not raises —
#         test for None explicitly or you'll miss silent failures

import pytest
from entity_tools import (
    search_filings_by_name,
    fuzzy_match_score,
    get_filing_details,
    get_business_registry_data,
    merge_entity_profile,
)

# --- fuzzy_match_score ---
@pytest.mark.parametrize("name_a, name_b, min_score, max_score", [
    # obvious match
    ("Acme Logistics LLC",     "ACME LOGISTICS, L.L.C.", 0.88, 1.0),
    # DBA alias
    ("TechBridge Solutions",   "TechBridge Solutions dba CloudBridge", 0.75, 1.0),
    # clearly different
    ("Apex Capital Partners",  "Riverdale Municipal Services",          0.0, 0.4),
    # empty string edge case
    ("",                        "Acme LLC",                             0.0, 0.1),
    # same exact string
    ("Unified Data Corp",       "Unified Data Corp",                    1.0, 1.0),
])
def test_fuzzy_match_score(name_a, name_b, min_score, max_score):
    result = fuzzy_match_score({"name_a": name_a, "name_b": name_b})
    assert result is not None, "fuzzy_match_score returned None"
    score = result["score"]
    assert min_score <= score <= max_score, (
        f"Expected score in [{min_score}, {max_score}] for "
        f"'{name_a}' vs '{name_b}', got {score:.3f}"
    )

# --- search_filings_by_name ---
@pytest.mark.parametrize("query, expected_min_results, expected_max_results", [
    ("Acme Logistics", 1, 20),     # should find something
    ("xyzzy_no_match_ever", 0, 0), # should return empty list
    ("",               0, 0),     # empty query — graceful empty return
])
def test_search_filings_by_name(query, expected_min_results, expected_max_results):
    result = search_filings_by_name({"query": query, "limit": 20})
    assert result is not None, "search_filings_by_name returned None"
    assert isinstance(result.get("filings", []), list)
    count = len(result.get("filings", []))
    assert expected_min_results <= count <= expected_max_results, (
        f"Query '{query}': expected {expected_min_results}–{expected_max_results} results, got {count}"
    )

# --- get_filing_details ---
def test_get_filing_details_valid():
    # First get a real filing ID from search
    search_result = search_filings_by_name({"query": "Acme", "limit": 1})
    filings = search_result.get("filings", [])
    if not filings:
        pytest.skip("No filings in test database — check entity_tools.py fixture data")
    filing_id = filings[0]["filing_id"]
    detail = get_filing_details({"filing_id": filing_id})
    assert detail is not None
    assert "entity_name" in detail
    assert "state" in detail

def test_get_filing_details_invalid_id():
    result = get_filing_details({"filing_id": "DOES_NOT_EXIST_99999"})
    # GOTCHA: function should return None or {"error": ...}, NOT raise
    assert result is None or "error" in result

# --- merge_entity_profile ---
def test_merge_entity_profile_produces_canonical_name():
    result = merge_entity_profile({
        "filing_ids": ["F001", "F002"],
        "canonical_name": "Acme Logistics LLC",
        "confidence": 0.94
    })
    assert result is not None
    assert result.get("canonical_name") == "Acme Logistics LLC"
    assert result.get("confidence") >= 0.90

def test_merge_entity_profile_rejects_low_confidence():
    """Merges below 0.7 should be flagged for human review."""
    result = merge_entity_profile({
        "filing_ids": ["F001", "F002"],
        "canonical_name": "Ambiguous Corp",
        "confidence": 0.55
    })
    # Either returns None, raises, or marks as needs_review
    if result is not None:
        assert result.get("needs_review") is True or result.get("confidence", 1) < 0.7

# tests/test_agent_integration.py
# WHAT: Run the full entity resolution agent and assert on tool traces
#       and output quality — no LLM output text matching, just structure
# WHY: Text matching is brittle. Assert on the *shape* of the output
#      (tools called, confidence range, presence of canonical_name)
#      so the test survives paraphrasing in Mistral's response
# GOTCHA: Set a per-test timeout — Ollama can hang on cold start.
#         Always skip gracefully when Ollama is unavailable so the
#         test suite doesn't block other CI jobs

import pytest
import time
from openai import OpenAI
from entity_agent import run_entity_resolution_agent  # your Capstone C3 agent
from eval_utils import check_ollama_health, load_eval_cases

# --- Fixture: skip all integration tests if Ollama is down ---
@pytest.fixture(scope="session", autouse=True)
def require_ollama():
    if not check_ollama_health():
        pytest.skip("Ollama not running — skipping integration tests")

# WHAT: One parametrized test covers every case in the golden set
# WHY: Adding cases = editing JSON, not editing Python
@pytest.fixture(scope="session")
def eval_cases():
    return load_eval_cases("eval_dataset.json")

@pytest.mark.parametrize("case", load_eval_cases("eval_dataset.json"),
                          ids=[c["id"] for c in load_eval_cases("eval_dataset.json")])
def test_agent_integration(case):
    """Run agent on one golden case and assert on output structure."""
    start = time.time()

    # WHAT: Run the full ReAct agent — this calls Ollama
    # GOTCHA: Wrap in try/except so one bad Ollama response doesn't
    #         poison your entire pytest run
    try:
        result = run_entity_resolution_agent(
            query=case["input"],
            max_iterations=8,      # prevent infinite loops in evals
            timeout_seconds=30,    # fail fast, don't hang CI
        )
    except Exception as e:
        pytest.fail(f"Agent raised unexpected exception: {e}")

    elapsed = time.time() - start

    # WHAT: Assert tools were actually called (not hallucinated)
    tools_called = result.get("tools_called", [])
    expected_tools = case["expected_tools_called"]
    for required_tool in expected_tools[:2]:  # check first two are present
        assert required_tool in tools_called, (
            f"Case {case['id']}: expected tool '{required_tool}' "
            f"but agent called: {tools_called}"
        )

    # WHAT: Assert output meets pass criteria
    expected_out = case["expected_output"]
    actual_confidence = result.get("confidence", 0)
    actual_decision = result.get("decision", "")

    if "expected_min_confidence" in expected_out:
        assert actual_confidence >= expected_out["expected_min_confidence"], (
            f"Case {case['id']}: confidence {actual_confidence:.3f} "
            f"< threshold {expected_out['expected_min_confidence']}"
        )
    if "expected_max_confidence" in expected_out:
        assert actual_confidence <= expected_out["expected_max_confidence"], (
            f"Case {case['id']}: confidence {actual_confidence:.3f} "
            f"> max {expected_out['expected_max_confidence']}"
        )
    if "decision" in expected_out:
        assert actual_decision == expected_out["decision"], (
            f"Case {case['id']}: expected decision '{expected_out['decision']}', "
            f"got '{actual_decision}'"
        )

    # WHAT: Performance guard — flag slow cases even if they pass
    if elapsed > 15:
        pytest.warns(UserWarning,
            match=f"Case {case['id']} took {elapsed:.1f}s — investigate")

# tests/test_deepeval_metrics.py
# WHAT: LLM quality metrics via DeepEval, using local Ollama as judge
# WHY: HallucinationMetric and AnswerRelevancyMetric catch quality
#      regressions that deterministic tests cannot see
# GOTCHA: DeepEval >= 1.4 required for local model support.
#         Install: pip install deepeval>=1.4 langchain-community
#         Set DEEPEVAL_TELEMETRY_OPT_OUT=YES to prevent phone-home

import os
import pytest
from deepeval import evaluate
from deepeval.metrics import HallucinationMetric, AnswerRelevancyMetric
from deepeval.models import DeepEvalBaseLLM
from deepeval.test_case import LLMTestCase
from openai import OpenAI

# Disable telemetry to Confident AI cloud
os.environ["DEEPEVAL_TELEMETRY_OPT_OUT"] = "YES"

# WHAT: Custom judge model wrapper pointing at local Ollama
# WHY: DeepEval requires a DeepEvalBaseLLM subclass to use a
#      non-OpenAI model as the evaluator
# GOTCHA: The judge's generate() method must return a plain string,
#         not an OpenAI response object

class OllamaJudge(DeepEvalBaseLLM):
    """Wraps local Ollama as a DeepEval judge model."""

    def __init__(self, model: str = "mistral", base_url: str = "http://localhost:11434/v1"):
        self.model = model
        self.client = OpenAI(base_url=base_url, api_key="ollama")

    def load_model(self):
        return self

    def generate(self, prompt: str) -> str:
        try:
            resp = self.client.chat.completions.create(
                model=self.model,
                messages=[{"role": "user", "content": prompt}],
                temperature=0,   # deterministic judge
                max_tokens=512,
            )
            return resp.choices[0].message.content or ""
        except Exception as e:
            return f"JUDGE_ERROR: {e}"

    async def a_generate(self, prompt: str) -> str:
        return self.generate(prompt)

    def get_model_name(self) -> str:
        return f"ollama/{self.model}"


# WHAT: Instantiate local judge — shared across all test cases
judge = OllamaJudge(model="mistral")

# WHAT: Hallucination metric — did the agent claim facts not in context?
# GOTCHA: context must be a LIST of strings; passing a single string
#         silently disables the metric and always returns 1.0

@pytest.fixture(scope="session")
def ollama_guard():
    """Skip all deepeval tests if Ollama is not running."""
    from eval_utils import check_ollama_health
    if not check_ollama_health():
        pytest.skip("Ollama not running")


@pytest.mark.parametrize("case_id, input_query, actual_output, context", [
    (
        "HALLUC-001",
        "Is Acme Logistics LLC the same as ACME LOGISTICS L.L.C.?",
        "Based on the UCC filing search, both names refer to the same entity. "
        "The fuzzy match score is 0.94 and the SOS registry confirms they share "
        "the same EIN. Confidence: 0.94. Decision: merge.",
        [
            "Filing F001: Acme Logistics LLC, Delaware, EIN 12-3456789",
            "Filing F002: ACME LOGISTICS L.L.C., New York, EIN 12-3456789",
            "Fuzzy match score: 0.94",
        ],
    ),
    (
        "HALLUC-002",
        "Is Apex Capital Partners LP the same as Apex Capital LLC?",
        "These appear to be different entities. Apex Capital Partners LP is a "
        "Delaware limited partnership. Apex Capital LLC is registered in Texas. "
        "No shared principals found. Confidence: 0.32. Decision: no_merge.",
        [
            "Filing F003: Apex Capital Partners LP, Delaware, LP registration",
            "Filing F004: Apex Capital LLC, Texas, LLC registration",
            "No shared officers or EIN found",
        ],
    ),
])
def test_hallucination_metric(ollama_guard, case_id, input_query, actual_output, context):
    """Agent output must not hallucinate facts beyond what's in context."""
    test_case = LLMTestCase(
        input=input_query,
        actual_output=actual_output,
        context=context,
    )
    metric = HallucinationMetric(
        model=judge,
        threshold=0.5,   # score > 0.5 means too much hallucination — fail
    )
    metric.measure(test_case)
    assert metric.score <= 0.5, (
        f"Case {case_id}: Hallucination score {metric.score:.3f} exceeds threshold 0.5. "
        f"Reason: {metric.reason}"
    )


@pytest.mark.parametrize("case_id, input_query, actual_output", [
    (
        "REL-001",
        "What is the confidence score for merging Acme Logistics LLC and ACME LOGISTICS L.L.C.?",
        "The confidence score is 0.94, based on a fuzzy match score of 0.94 "
        "and matching EIN numbers confirmed by the SOS registry.",
    ),
    (
        "REL-002",
        "Should Apex Capital Partners LP be merged with Apex Capital LLC?",
        "No. These are separate entities registered in different states with "
        "no shared principals or EIN. Confidence: 0.32.",
    ),
])
def test_answer_relevancy_metric(ollama_guard, case_id, input_query, actual_output):
    """Agent answer must be relevant to the question asked."""
    test_case = LLMTestCase(
        input=input_query,
        actual_output=actual_output,
    )
    metric = AnswerRelevancyMetric(
        model=judge,
        threshold=0.7,   # score >= 0.7 required to pass
    )
    metric.measure(test_case)
    assert metric.score >= 0.7, (
        f"Case {case_id}: Relevancy score {metric.score:.3f} below threshold 0.7. "
        f"Reason: {metric.reason}"
    )

# tests/test_ragas_rag.py
# WHAT: Ragas evaluation for the retrieval pipeline in entity resolution
# WHY: The agent's quality depends heavily on what it retrieved —
#      a high-confidence merge based on bad retrieval is a dangerous failure
# GOTCHA: ragas>=0.1.6 required for local LLM support.
#         sentence-transformers runs on CPU — first run downloads ~80MB model

import pytest
from datasets import Dataset

# WHAT: Wire Ragas to use Ollama (LLM) + sentence-transformers (embeddings)
# GOTCHA: Import order matters — set the model BEFORE importing metrics
#         otherwise Ragas silently falls back to OpenAI and raises an auth error

from langchain_community.llms import Ollama
from langchain_community.embeddings import HuggingFaceEmbeddings
from ragas.llms import LangchainLLMWrapper
from ragas.embeddings import LangchainEmbeddingsWrapper
from ragas.metrics import faithfulness, answer_relevancy, context_precision
from ragas import evaluate

# WHAT: Local LLM wrapper — uses Ollama Mistral as the judge
ollama_llm = LangchainLLMWrapper(
    Ollama(model="mistral", base_url="http://localhost:11434", temperature=0)
)

# WHAT: Local embedding model — runs on CPU, no network call
# WHY: Ragas uses embeddings to compute semantic similarity for
#      answer_relevancy. sentence-transformers/all-MiniLM-L6-v2 is
#      a 80MB model that runs fast on CPU.
local_embeddings = LangchainEmbeddingsWrapper(
    HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
)

# WHAT: Configure all metrics to use local models
# GOTCHA: Must set both llm and embeddings on each metric —
#         they don't inherit from a global config
for metric in [faithfulness, answer_relevancy, context_precision]:
    metric.llm = ollama_llm
    metric.embeddings = local_embeddings


# WHAT: Build a Ragas Dataset from agent run results
# Each row = one (question, retrieved_contexts, answer, ground_truth) tuple
def build_ragas_dataset(agent_results: list[dict]) -> Dataset:
    """
    Convert agent run output into Ragas evaluation format.
    agent_results[i] = {
        "question": str,
        "contexts": list[str],   # retrieved filing chunks
        "answer": str,            # agent final answer
        "ground_truth": str,      # from golden test set
    }
    """
    return Dataset.from_list(agent_results)


# WHAT: Ragas evaluation fixture — builds dataset from golden cases
@pytest.fixture(scope="module")
def ragas_results():
    """Run agent on RAG-specific cases and collect retrieved contexts."""
    from eval_utils import check_ollama_health
    if not check_ollama_health():
        pytest.skip("Ollama not running")

    # These are hand-crafted for the RAG eval — each includes the
    # contexts the agent actually retrieved during the run
    return [
        {
            "question": "Is Acme Logistics LLC the same entity as ACME LOGISTICS L.L.C.?",
            "contexts": [
                "Filing F001: Acme Logistics LLC, State: Delaware, EIN: 12-3456789, Status: Active",
                "Filing F002: ACME LOGISTICS, L.L.C., State: New York, EIN: 12-3456789, Status: Active",
                "Fuzzy match score (token_sort_ratio): 0.96",
            ],
            "answer": (
                "Both filings share EIN 12-3456789 and a fuzzy match score of 0.96. "
                "The SOS registry confirms these are the same entity under Delaware registration. "
                "Decision: MERGE. Canonical name: Acme Logistics LLC. Confidence: 0.94."
            ),
            "ground_truth": "Yes, these are the same entity. Confidence: 0.94.",
        },
        {
            "question": "Should Apex Capital Partners LP be merged with Apex Capital LLC?",
            "contexts": [
                "Filing F003: Apex Capital Partners LP, State: Delaware, Type: Limited Partnership",
                "Filing F004: Apex Capital LLC, State: Texas, Type: LLC",
                "No shared principals, no matching EIN",
            ],
            "answer": (
                "Apex Capital Partners LP and Apex Capital LLC are registered in different states "
                "with different entity types and no shared officers or EIN. "
                "Decision: NO MERGE. Confidence: 0.32."
            ),
            "ground_truth": "No — these are separate entities. Confidence: 0.32.",
        },
    ]


def test_ragas_faithfulness(ragas_results):
    """Agent answers must only use facts present in retrieved context."""
    dataset = build_ragas_dataset(ragas_results)
    results = evaluate(
        dataset=dataset,
        metrics=[faithfulness],
    )
    score = results["faithfulness"]
    assert score >= 0.70, (
        f"Faithfulness {score:.3f} below threshold 0.70 — "
        "agent may be hallucinating beyond retrieved context"
    )


def test_ragas_answer_relevancy(ragas_results):
    """Agent answers must address the actual question asked."""
    dataset = build_ragas_dataset(ragas_results)
    results = evaluate(
        dataset=dataset,
        metrics=[answer_relevancy],
    )
    score = results["answer_relevancy"]
    assert score >= 0.75, (
        f"Answer relevancy {score:.3f} below threshold 0.75 — "
        "agent may be answering a different question than asked"
    )


def test_ragas_context_precision(ragas_results):
    """Retrieved context must be relevant to the question — no noisy retrieval."""
    dataset = build_ragas_dataset(ragas_results)
    results = evaluate(
        dataset=dataset,
        metrics=[context_precision],
    )
    score = results["context_precision"]
    assert score >= 0.65, (
        f"Context precision {score:.3f} below threshold 0.65 — "
        "search_filings_by_name may be returning irrelevant results"
    )

# tests/llm_judge.py
# WHAT: LLM-as-judge for entity resolution agent quality
# WHY: Some quality dimensions (reasoning coherence, evidence use)
#      can only be assessed by another LLM, not by code
# GOTCHA: Use temperature=0 for the judge to get deterministic scores.
#         Still expect ~0.05 variance across identical runs with Mistral-7B —
#         run each case 3x and average if you need high precision

from openai import OpenAI
import json
import re

client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

# WHAT: Judge prompt template — explicit criteria, bias mitigations
# GOTCHA: The JSON parsing block must handle both ``` fences and raw JSON —
#         Mistral-7B sometimes wraps its output in markdown code blocks

JUDGE_PROMPT_TEMPLATE = """You are an impartial evaluator assessing the quality of an \
AI entity resolution agent's response. Score each dimension from 0.0 to 1.0.

QUESTION: {question}

AGENT RESPONSE: {answer}

RETRIEVED CONTEXT (what the agent had access to):
{context}

EVALUATION CRITERIA:
1. Reasoning quality (0–1): Does the agent reason step-by-step from evidence to conclusion? \
   Does it cite specific filings, scores, or registry data before deciding?
2. Faithfulness (0–1): Does the answer use ONLY facts present in the retrieved context? \
   Score 0 if it invents any fact not in context.
3. Evidence sufficiency (0–1): Did the agent use at least two independent pieces of evidence \
   before reaching its confidence score? One clue is not enough for a high-confidence merge.
4. Confidence calibration (0–1): Is the confidence score plausible given the evidence? \
   Score 1 if confidence matches evidence strength, 0 if they contradict.

IMPORTANT:
- Do NOT favor longer or shorter answers — length does not equal quality.
- Base ALL scores on the criteria above, not on whether you agree with the final decision.
- Return ONLY valid JSON — no markdown, no explanation outside the JSON.

Return:
{{
  "reasoning_quality": 0.0,
  "faithfulness": 0.0,
  "evidence_sufficiency": 0.0,
  "confidence_calibration": 0.0,
  "overall": 0.0,
  "explanation": "one sentence explaining the overall score"
}}"""


def run_judge(question: str, answer: str, context: list[str],
              model: str = "mistral") -> dict:
    """
    Score one (question, answer, context) triple with the local judge.
    Returns a score dict. On judge error, returns all zeros with an error key.
    """
    ctx_str = "\n".join(f"- {c}" for c in context)
    prompt = JUDGE_PROMPT_TEMPLATE.format(
        question=question, answer=answer, context=ctx_str
    )
    try:
        resp = client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": prompt}],
            temperature=0,
            max_tokens=512,
        )
        raw = resp.choices[0].message.content or ""

        # WHAT: Strip markdown fences if present
        # GOTCHA: Mistral sometimes wraps JSON in ```json ... ``` blocks
        raw = re.sub(r"^```(?:json)?\s*", "", raw.strip())
        raw = re.sub(r"\s*```$", "", raw)

        scores = json.loads(raw)
        return scores
    except json.JSONDecodeError as e:
        return {
            "reasoning_quality": 0, "faithfulness": 0,
            "evidence_sufficiency": 0, "confidence_calibration": 0,
            "overall": 0, "explanation": f"JSON parse error: {e}",
            "judge_error": True,
        }
    except Exception as e:
        return {
            "reasoning_quality": 0, "faithfulness": 0,
            "evidence_sufficiency": 0, "confidence_calibration": 0,
            "overall": 0, "explanation": str(e),
            "judge_error": True,
        }


def evaluate_with_judge(test_cases: list[dict],
                        threshold: float = 0.70) -> dict:
    """
    Run judge on all cases. Returns summary with pass/fail per case.
    test_cases[i] = {"question", "answer", "context", "id"}
    """
    results = []
    for tc in test_cases:
        scores = run_judge(tc["question"], tc["answer"], tc["context"])
        passed = (
            not scores.get("judge_error")
            and scores.get("overall", 0) >= threshold
        )
        results.append({
            "id": tc["id"],
            "scores": scores,
            "passed": passed,
            "overall": scores.get("overall", 0),
        })
        status = "PASS" if passed else "FAIL"
        print(f"  [{status}] {tc['id']}: overall={scores.get('overall', 0):.2f} — "
              f"{scores.get('explanation', '')}")

    passed_count = sum(1 for r in results if r["passed"])
    pass_rate = passed_count / len(results) if results else 0
    return {
        "results": results,
        "pass_rate": pass_rate,
        "passed": passed_count,
        "total": len(results),
    }

# .github/workflows/eval.yml
# WHAT: Three-layer eval pipeline running on every PR
# WHY: Catch regressions before they reach main — the cost is
#      ~5 minutes of CI time vs. days of production debugging
# GOTCHA: ollama serve must be backgrounded (&) and you must
#         wait for it to be ready before running tests

name: Agent Eval Suite

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  unit-tests:
    name: Unit Tests (tools)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        run: pip install -r requirements.txt pytest

      # WHAT: Unit tests have no Ollama dependency — run first, fast
      - name: Run unit tests
        run: pytest tests/test_tools_unit.py -v --tb=short
        timeout-minutes: 2


  integration-tests:
    name: Integration Tests (agent loop)
    runs-on: ubuntu-latest
    needs: unit-tests   # only run if units pass
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        run: pip install -r requirements.txt pytest httpx

      # WHAT: Install Ollama binary
      - name: Install Ollama
        run: curl -fsSL https://ollama.com/install.sh | sh

      # WHAT: Start Ollama server in background, pull mistral model
      # GOTCHA: Sleep 5 gives Ollama time to initialize before pull
      - name: Start Ollama and pull model
        run: |
          ollama serve &
          sleep 5
          ollama pull mistral
          echo "Ollama ready"
        timeout-minutes: 8

      - name: Run integration tests
        run: pytest tests/test_agent_integration.py -v --tb=short
        timeout-minutes: 10
        env:
          EVAL_PASS_THRESHOLD: "0.80"   # 80% of cases must pass


  llm-judge-tests:
    name: LLM-as-Judge (quality)
    runs-on: ubuntu-latest
    needs: integration-tests   # judge tests are expensive — run last
    if: github.event_name == 'push' || github.base_ref == 'main'
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        run: |
          pip install -r requirements.txt pytest deepeval>=1.4
          pip install langchain-community sentence-transformers ragas

      - name: Install Ollama
        run: curl -fsSL https://ollama.com/install.sh | sh

      - name: Start Ollama and pull model
        run: |
          ollama serve &
          sleep 5
          ollama pull mistral
        timeout-minutes: 8

      # WHAT: Run judge evals with regression threshold
      # GOTCHA: Set DEEPEVAL_TELEMETRY_OPT_OUT to prevent DeepEval
      #         from phoning home during CI
      - name: Run LLM judge evals
        run: |
          python -m pytest tests/test_deepeval_metrics.py \
            tests/test_ragas_rag.py -v --tb=short
        timeout-minutes: 15
        env:
          DEEPEVAL_TELEMETRY_OPT_OUT: "YES"
          JUDGE_PASS_THRESHOLD: "0.70"