M18: Evaluation & Testing

# eval_dataset.json — one entry per golden case
# WHAT: Each case captures input, expected tool sequence, and pass criteria
# WHY: Separating test data from test logic lets domain experts update
#      cases without touching Python
# GOTCHA: Keep expected_min_confidence realistic for your model tier —
#         Mistral-7B scores ~0.05 lower than Mistral-large on average

EVAL_SCHEMA = {
    "cases": [
        {
            "id": "ER-001",
            "description": "Obvious match — same company, different punctuation",
            "input": "Are 'Acme Logistics LLC' and 'ACME LOGISTICS, L.L.C.' the same entity?",
            "expected_tools_called": [
                "search_filings_by_name",
                "fuzzy_match_score",
                "get_business_registry_data",
                "merge_entity_profile"
            ],
            "expected_output": {
                "decision": "merge",
                "expected_min_confidence": 0.90,
                "must_contain_keywords": ["canonical", "match"]
            }
        },
        {
            "id": "ER-002",
            "description": "Ambiguous case — similar names, different businesses",
            "input": "Is 'Apex Capital Partners LP' the same as 'Apex Capital LLC'?",
            "expected_tools_called": [
                "search_filings_by_name",
                "fuzzy_match_score",
                "get_filing_details"
            ],
            "expected_output": {
                "decision": "no_merge",
                "expected_max_confidence": 0.70,
                "must_contain_keywords": ["different", "separate"]
            }
        },
        {
            "id": "ER-003",
            "description": "DBA (doing business as) alias detection",
            "input": "Is 'TechBridge Solutions' the same entity as 'TechBridge Solutions dba CloudBridge'?",
            "expected_tools_called": [
                "search_filings_by_name",
                "get_business_registry_data",
                "merge_entity_profile"
            ],
            "expected_output": {
                "decision": "merge",
                "expected_min_confidence": 0.85,
                "must_contain_keywords": ["alias", "dba"]
            }
        }
    ]
}

# WHAT: Loader function with Ollama availability guard
# WHY: Evals should fail gracefully — a missing Ollama server should
#      produce a clear error, not a cryptic connection timeout
# GOTCHA: Always check Ollama health before starting the eval run —
#         a silent Ollama failure corrupts your entire results file

import json
import sys
import httpx

def load_eval_cases(path: str = "eval_dataset.json") -> list[dict]:
    """Load golden test cases with validation."""
    try:
        with open(path) as f:
            data = json.load(f)
        cases = data.get("cases", [])
        if not cases:
            raise ValueError("Eval dataset is empty — add cases before running")
        print(f"Loaded {len(cases)} eval cases from {path}")
        return cases
    except FileNotFoundError:
        print(f"ERROR: {path} not found. Run `python build_eval_dataset.py` first.")
        sys.exit(1)

def check_ollama_health(base_url: str = "http://localhost:11434") -> bool:
    """Verify Ollama is running before spending time on evals."""
    try:
        resp = httpx.get(f"{base_url}/api/tags", timeout=3.0)
        models = [m["name"] for m in resp.json().get("models", [])]
        if "mistral" not in " ".join(models):
            print("WARNING: 'mistral' model not found. Run: ollama pull mistral")
            return False
        return True
    except (httpx.ConnectError, httpx.TimeoutException):
        print("ERROR: Ollama is not running. Start it with: ollama serve")
        return False

# tests/test_tools_unit.py
# WHAT: Parametrized unit tests for all five entity resolution tools
# WHY: Parametrize lets you add new cases without new test functions —
#      one failing case won't abort the rest
# GOTCHA: entity_tools.py returns None on not-found, not raises —
#         test for None explicitly or you'll miss silent failures

import pytest
from entity_tools import (
    search_filings_by_name,
    fuzzy_match_score,
    get_filing_details,
    get_business_registry_data,
    merge_entity_profile,
)

# --- fuzzy_match_score ---
@pytest.mark.parametrize("name_a, name_b, min_score, max_score", [
    # obvious match
    ("Acme Logistics LLC",     "ACME LOGISTICS, L.L.C.", 0.88, 1.0),
    # DBA alias
    ("TechBridge Solutions",   "TechBridge Solutions dba CloudBridge", 0.75, 1.0),
    # clearly different
    ("Apex Capital Partners",  "Riverdale Municipal Services",          0.0, 0.4),
    # empty string edge case
    ("",                        "Acme LLC",                             0.0, 0.1),
    # same exact string
    ("Unified Data Corp",       "Unified Data Corp",                    1.0, 1.0),
])
def test_fuzzy_match_score(name_a, name_b, min_score, max_score):
    result = fuzzy_match_score({"name_a": name_a, "name_b": name_b})
    assert result is not None, "fuzzy_match_score returned None"
    score = result["score"]
    assert min_score <= score <= max_score, (
        f"Expected score in [{min_score}, {max_score}] for "
        f"'{name_a}' vs '{name_b}', got {score:.3f}"
    )

# --- search_filings_by_name ---
@pytest.mark.parametrize("query, expected_min_results, expected_max_results", [
    ("Acme Logistics", 1, 20),     # should find something
    ("xyzzy_no_match_ever", 0, 0), # should return empty list
    ("",               0, 0),     # empty query — graceful empty return
])
def test_search_filings_by_name(query, expected_min_results, expected_max_results):
    result = search_filings_by_name({"query": query, "limit": 20})
    assert result is not None, "search_filings_by_name returned None"
    assert isinstance(result.get("filings", []), list)
    count = len(result.get("filings", []))
    assert expected_min_results <= count <= expected_max_results, (
        f"Query '{query}': expected {expected_min_results}–{expected_max_results} results, got {count}"
    )

# --- get_filing_details ---
def test_get_filing_details_valid():
    # First get a real filing ID from search
    search_result = search_filings_by_name({"query": "Acme", "limit": 1})
    filings = search_result.get("filings", [])
    if not filings:
        pytest.skip("No filings in test database — check entity_tools.py fixture data")
    filing_id = filings[0]["filing_id"]
    detail = get_filing_details({"filing_id": filing_id})
    assert detail is not None
    assert "entity_name" in detail
    assert "state" in detail

def test_get_filing_details_invalid_id():
    result = get_filing_details({"filing_id": "DOES_NOT_EXIST_99999"})
    # GOTCHA: function should return None or {"error": ...}, NOT raise
    assert result is None or "error" in result

# --- merge_entity_profile ---
def test_merge_entity_profile_produces_canonical_name():
    result = merge_entity_profile({
        "filing_ids": ["F001", "F002"],
        "canonical_name": "Acme Logistics LLC",
        "confidence": 0.94
    })
    assert result is not None
    assert result.get("canonical_name") == "Acme Logistics LLC"
    assert result.get("confidence") >= 0.90

def test_merge_entity_profile_rejects_low_confidence():
    """Merges below 0.7 should be flagged for human review."""
    result = merge_entity_profile({
        "filing_ids": ["F001", "F002"],
        "canonical_name": "Ambiguous Corp",
        "confidence": 0.55
    })
    # Either returns None, raises, or marks as needs_review
    if result is not None:
        assert result.get("needs_review") is True or result.get("confidence", 1) < 0.7

# tests/test_agent_integration.py
# WHAT: Run the full entity resolution agent and assert on tool traces
#       and output quality — no LLM output text matching, just structure
# WHY: Text matching is brittle. Assert on the *shape* of the output
#      (tools called, confidence range, presence of canonical_name)
#      so the test survives paraphrasing in Mistral's response
# GOTCHA: Set a per-test timeout — Ollama can hang on cold start.
#         Always skip gracefully when Ollama is unavailable so the
#         test suite doesn't block other CI jobs

import pytest
import time
from openai import OpenAI
from entity_agent import run_entity_resolution_agent  # your Capstone C3 agent
from eval_utils import check_ollama_health, load_eval_cases

# --- Fixture: skip all integration tests if Ollama is down ---
@pytest.fixture(scope="session", autouse=True)
def require_ollama():
    if not check_ollama_health():
        pytest.skip("Ollama not running — skipping integration tests")

# WHAT: One parametrized test covers every case in the golden set
# WHY: Adding cases = editing JSON, not editing Python
@pytest.fixture(scope="session")
def eval_cases():
    return load_eval_cases("eval_dataset.json")

@pytest.mark.parametrize("case", load_eval_cases("eval_dataset.json"),
                          ids=[c["id"] for c in load_eval_cases("eval_dataset.json")])
def test_agent_integration(case):
    """Run agent on one golden case and assert on output structure."""
    start = time.time()

    # WHAT: Run the full ReAct agent — this calls Ollama
    # GOTCHA: Wrap in try/except so one bad Ollama response doesn't
    #         poison your entire pytest run
    try:
        result = run_entity_resolution_agent(
            query=case["input"],
            max_iterations=8,      # prevent infinite loops in evals
            timeout_seconds=30,    # fail fast, don't hang CI
        )
    except Exception as e:
        pytest.fail(f"Agent raised unexpected exception: {e}")

    elapsed = time.time() - start

    # WHAT: Assert tools were actually called (not hallucinated)
    tools_called = result.get("tools_called", [])
    expected_tools = case["expected_tools_called"]
    for required_tool in expected_tools[:2]:  # check first two are present
        assert required_tool in tools_called, (
            f"Case {case['id']}: expected tool '{required_tool}' "
            f"but agent called: {tools_called}"
        )

    # WHAT: Assert output meets pass criteria
    expected_out = case["expected_output"]
    actual_confidence = result.get("confidence", 0)
    actual_decision = result.get("decision", "")

    if "expected_min_confidence" in expected_out:
        assert actual_confidence >= expected_out["expected_min_confidence"], (
            f"Case {case['id']}: confidence {actual_confidence:.3f} "
            f"< threshold {expected_out['expected_min_confidence']}"
        )
    if "expected_max_confidence" in expected_out:
        assert actual_confidence <= expected_out["expected_max_confidence"], (
            f"Case {case['id']}: confidence {actual_confidence:.3f} "
            f"> max {expected_out['expected_max_confidence']}"
        )
    if "decision" in expected_out:
        assert actual_decision == expected_out["decision"], (
            f"Case {case['id']}: expected decision '{expected_out['decision']}', "
            f"got '{actual_decision}'"
        )

    # WHAT: Performance guard — flag slow cases even if they pass
    if elapsed > 15:
        pytest.warns(UserWarning,
            match=f"Case {case['id']} took {elapsed:.1f}s — investigate")

# tests/test_deepeval_metrics.py
# WHAT: LLM quality metrics via DeepEval, using local Ollama as judge
# WHY: HallucinationMetric and AnswerRelevancyMetric catch quality
#      regressions that deterministic tests cannot see
# GOTCHA: DeepEval >= 1.4 required for local model support.
#         Install: pip install deepeval>=1.4 langchain-community
#         Set DEEPEVAL_TELEMETRY_OPT_OUT=YES to prevent phone-home

import os
import pytest
from deepeval import evaluate
from deepeval.metrics import HallucinationMetric, AnswerRelevancyMetric
from deepeval.models import DeepEvalBaseLLM
from deepeval.test_case import LLMTestCase
from openai import OpenAI

# Disable telemetry to Confident AI cloud
os.environ["DEEPEVAL_TELEMETRY_OPT_OUT"] = "YES"

# WHAT: Custom judge model wrapper pointing at local Ollama
# WHY: DeepEval requires a DeepEvalBaseLLM subclass to use a
#      non-OpenAI model as the evaluator
# GOTCHA: The judge's generate() method must return a plain string,
#         not an OpenAI response object

class OllamaJudge(DeepEvalBaseLLM):
    """Wraps local Ollama as a DeepEval judge model."""

    def __init__(self, model: str = "mistral", base_url: str = "http://localhost:11434/v1"):
        self.model = model
        self.client = OpenAI(base_url=base_url, api_key="ollama")

    def load_model(self):
        return self

    def generate(self, prompt: str) -> str:
        try:
            resp = self.client.chat.completions.create(
                model=self.model,
                messages=[{"role": "user", "content": prompt}],
                temperature=0,   # deterministic judge
                max_tokens=512,
            )
            return resp.choices[0].message.content or ""
        except Exception as e:
            return f"JUDGE_ERROR: {e}"

    async def a_generate(self, prompt: str) -> str:
        return self.generate(prompt)

    def get_model_name(self) -> str:
        return f"ollama/{self.model}"


# WHAT: Instantiate local judge — shared across all test cases
judge = OllamaJudge(model="mistral")

# WHAT: Hallucination metric — did the agent claim facts not in context?
# GOTCHA: context must be a LIST of strings; passing a single string
#         silently disables the metric and always returns 1.0

@pytest.fixture(scope="session")
def ollama_guard():
    """Skip all deepeval tests if Ollama is not running."""
    from eval_utils import check_ollama_health
    if not check_ollama_health():
        pytest.skip("Ollama not running")


@pytest.mark.parametrize("case_id, input_query, actual_output, context", [
    (
        "HALLUC-001",
        "Is Acme Logistics LLC the same as ACME LOGISTICS L.L.C.?",
        "Based on the UCC filing search, both names refer to the same entity. "
        "The fuzzy match score is 0.94 and the SOS registry confirms they share "
        "the same EIN. Confidence: 0.94. Decision: merge.",
        [
            "Filing F001: Acme Logistics LLC, Delaware, EIN 12-3456789",
            "Filing F002: ACME LOGISTICS L.L.C., New York, EIN 12-3456789",
            "Fuzzy match score: 0.94",
        ],
    ),
    (
        "HALLUC-002",
        "Is Apex Capital Partners LP the same as Apex Capital LLC?",
        "These appear to be different entities. Apex Capital Partners LP is a "
        "Delaware limited partnership. Apex Capital LLC is registered in Texas. "
        "No shared principals found. Confidence: 0.32. Decision: no_merge.",
        [
            "Filing F003: Apex Capital Partners LP, Delaware, LP registration",
            "Filing F004: Apex Capital LLC, Texas, LLC registration",
            "No shared officers or EIN found",
        ],
    ),
])
def test_hallucination_metric(ollama_guard, case_id, input_query, actual_output, context):
    """Agent output must not hallucinate facts beyond what's in context."""
    test_case = LLMTestCase(
        input=input_query,
        actual_output=actual_output,
        context=context,
    )
    metric = HallucinationMetric(
        model=judge,
        threshold=0.5,   # score > 0.5 means too much hallucination — fail
    )
    metric.measure(test_case)
    assert metric.score <= 0.5, (
        f"Case {case_id}: Hallucination score {metric.score:.3f} exceeds threshold 0.5. "
        f"Reason: {metric.reason}"
    )


@pytest.mark.parametrize("case_id, input_query, actual_output", [
    (
        "REL-001",
        "What is the confidence score for merging Acme Logistics LLC and ACME LOGISTICS L.L.C.?",
        "The confidence score is 0.94, based on a fuzzy match score of 0.94 "
        "and matching EIN numbers confirmed by the SOS registry.",
    ),
    (
        "REL-002",
        "Should Apex Capital Partners LP be merged with Apex Capital LLC?",
        "No. These are separate entities registered in different states with "
        "no shared principals or EIN. Confidence: 0.32.",
    ),
])
def test_answer_relevancy_metric(ollama_guard, case_id, input_query, actual_output):
    """Agent answer must be relevant to the question asked."""
    test_case = LLMTestCase(
        input=input_query,
        actual_output=actual_output,
    )
    metric = AnswerRelevancyMetric(
        model=judge,
        threshold=0.7,   # score >= 0.7 required to pass
    )
    metric.measure(test_case)
    assert metric.score >= 0.7, (
        f"Case {case_id}: Relevancy score {metric.score:.3f} below threshold 0.7. "
        f"Reason: {metric.reason}"
    )

# tests/test_ragas_rag.py
# WHAT: Ragas evaluation for the retrieval pipeline in entity resolution
# WHY: The agent's quality depends heavily on what it retrieved —
#      a high-confidence merge based on bad retrieval is a dangerous failure
# GOTCHA: ragas>=0.1.6 required for local LLM support.
#         sentence-transformers runs on CPU — first run downloads ~80MB model

import pytest
from datasets import Dataset

# WHAT: Wire Ragas to use Ollama (LLM) + sentence-transformers (embeddings)
# GOTCHA: Import order matters — set the model BEFORE importing metrics
#         otherwise Ragas silently falls back to OpenAI and raises an auth error

from langchain_community.llms import Ollama
from langchain_community.embeddings import HuggingFaceEmbeddings
from ragas.llms import LangchainLLMWrapper
from ragas.embeddings import LangchainEmbeddingsWrapper
from ragas.metrics import faithfulness, answer_relevancy, context_precision
from ragas import evaluate

# WHAT: Local LLM wrapper — uses Ollama Mistral as the judge
ollama_llm = LangchainLLMWrapper(
    Ollama(model="mistral", base_url="http://localhost:11434", temperature=0)
)

# WHAT: Local embedding model — runs on CPU, no network call
# WHY: Ragas uses embeddings to compute semantic similarity for
#      answer_relevancy. sentence-transformers/all-MiniLM-L6-v2 is
#      a 80MB model that runs fast on CPU.
local_embeddings = LangchainEmbeddingsWrapper(
    HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
)

# WHAT: Configure all metrics to use local models
# GOTCHA: Must set both llm and embeddings on each metric —
#         they don't inherit from a global config
for metric in [faithfulness, answer_relevancy, context_precision]:
    metric.llm = ollama_llm
    metric.embeddings = local_embeddings


# WHAT: Build a Ragas Dataset from agent run results
# Each row = one (question, retrieved_contexts, answer, ground_truth) tuple
def build_ragas_dataset(agent_results: list[dict]) -> Dataset:
    """
    Convert agent run output into Ragas evaluation format.
    agent_results[i] = {
        "question": str,
        "contexts": list[str],   # retrieved filing chunks
        "answer": str,            # agent final answer
        "ground_truth": str,      # from golden test set
    }
    """
    return Dataset.from_list(agent_results)


# WHAT: Ragas evaluation fixture — builds dataset from golden cases
@pytest.fixture(scope="module")
def ragas_results():
    """Run agent on RAG-specific cases and collect retrieved contexts."""
    from eval_utils import check_ollama_health
    if not check_ollama_health():
        pytest.skip("Ollama not running")

    # These are hand-crafted for the RAG eval — each includes the
    # contexts the agent actually retrieved during the run
    return [
        {
            "question": "Is Acme Logistics LLC the same entity as ACME LOGISTICS L.L.C.?",
            "contexts": [
                "Filing F001: Acme Logistics LLC, State: Delaware, EIN: 12-3456789, Status: Active",
                "Filing F002: ACME LOGISTICS, L.L.C., State: New York, EIN: 12-3456789, Status: Active",
                "Fuzzy match score (token_sort_ratio): 0.96",
            ],
            "answer": (
                "Both filings share EIN 12-3456789 and a fuzzy match score of 0.96. "
                "The SOS registry confirms these are the same entity under Delaware registration. "
                "Decision: MERGE. Canonical name: Acme Logistics LLC. Confidence: 0.94."
            ),
            "ground_truth": "Yes, these are the same entity. Confidence: 0.94.",
        },
        {
            "question": "Should Apex Capital Partners LP be merged with Apex Capital LLC?",
            "contexts": [
                "Filing F003: Apex Capital Partners LP, State: Delaware, Type: Limited Partnership",
                "Filing F004: Apex Capital LLC, State: Texas, Type: LLC",
                "No shared principals, no matching EIN",
            ],
            "answer": (
                "Apex Capital Partners LP and Apex Capital LLC are registered in different states "
                "with different entity types and no shared officers or EIN. "
                "Decision: NO MERGE. Confidence: 0.32."
            ),
            "ground_truth": "No — these are separate entities. Confidence: 0.32.",
        },
    ]


def test_ragas_faithfulness(ragas_results):
    """Agent answers must only use facts present in retrieved context."""
    dataset = build_ragas_dataset(ragas_results)
    results = evaluate(
        dataset=dataset,
        metrics=[faithfulness],
    )
    score = results["faithfulness"]
    assert score >= 0.70, (
        f"Faithfulness {score:.3f} below threshold 0.70 — "
        "agent may be hallucinating beyond retrieved context"
    )


def test_ragas_answer_relevancy(ragas_results):
    """Agent answers must address the actual question asked."""
    dataset = build_ragas_dataset(ragas_results)
    results = evaluate(
        dataset=dataset,
        metrics=[answer_relevancy],
    )
    score = results["answer_relevancy"]
    assert score >= 0.75, (
        f"Answer relevancy {score:.3f} below threshold 0.75 — "
        "agent may be answering a different question than asked"
    )


def test_ragas_context_precision(ragas_results):
    """Retrieved context must be relevant to the question — no noisy retrieval."""
    dataset = build_ragas_dataset(ragas_results)
    results = evaluate(
        dataset=dataset,
        metrics=[context_precision],
    )
    score = results["context_precision"]
    assert score >= 0.65, (
        f"Context precision {score:.3f} below threshold 0.65 — "
        "search_filings_by_name may be returning irrelevant results"
    )

# tests/llm_judge.py
# WHAT: LLM-as-judge for entity resolution agent quality
# WHY: Some quality dimensions (reasoning coherence, evidence use)
#      can only be assessed by another LLM, not by code
# GOTCHA: Use temperature=0 for the judge to get deterministic scores.
#         Still expect ~0.05 variance across identical runs with Mistral-7B —
#         run each case 3x and average if you need high precision

from openai import OpenAI
import json
import re

client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

# WHAT: Judge prompt template — explicit criteria, bias mitigations
# GOTCHA: The JSON parsing block must handle both ``` fences and raw JSON —
#         Mistral-7B sometimes wraps its output in markdown code blocks

JUDGE_PROMPT_TEMPLATE = """You are an impartial evaluator assessing the quality of an \
AI entity resolution agent's response. Score each dimension from 0.0 to 1.0.

QUESTION: {question}

AGENT RESPONSE: {answer}

RETRIEVED CONTEXT (what the agent had access to):
{context}

EVALUATION CRITERIA — four dimensions, each scored on the anchored
scale below. Pick the anchor whose description matches what you actually see.
If an answer sits between two anchors, choose the LOWER one.

1. reasoning_quality — does it reason FROM evidence TO the conclusion?
   1.00 = every claim is tied to a named filing, score, or registry field, in order
   0.75 = reasons from evidence, but one step is asserted without a source
   0.50 = conclusion and evidence both present, no chain connecting them
   0.25 = evidence appears only after the conclusion, as decoration
   0.00 = a conclusion with no reasoning at all

2. faithfulness — does the answer use ONLY facts present in the retrieved context?
   1.00 = every fact is verbatim or a direct paraphrase of the context
   0.75 = all facts grounded, but one is stated more strongly than context supports
   0.50 = one unsupported detail that does not change the decision
   0.25 = one unsupported fact that materially supports the decision
   0.00 = invents a filing, address, officer, EIN, or date not in the context

3. evidence_sufficiency — independent evidence behind a high-confidence call.
   Two fields from the SAME filing are one piece of evidence, not two.
   A fuzzy name score derived from the two names is not independent of them.
   1.00 = three or more independent sources, and it says why each is independent
   0.75 = exactly two independent sources
   0.50 = two sources that are not actually independent (same filing or submitter)
   0.25 = a single source, and the answer admits the evidence is thin
   0.00 = a single source presented as sufficient

4. confidence_calibration — is the stated confidence consistent with the evidence?
   1.00 = confidence matches evidence strength within about 0.05
   0.75 = about 0.10 too LOW for the evidence (under-confident)
   0.50 = about 0.10 too HIGH for the evidence (over-confident)
   0.25 = a strong claim on weak evidence, or a refusal despite conclusive evidence
   0.00 = confidence and evidence flatly contradict (0.99 on one weak clue)

IMPORTANT:
- Use ONLY the five anchor values: 0.00, 0.25, 0.50, 0.75, 1.00.
- Do NOT favor longer or shorter answers — length does not equal quality.
- Score against the anchors, not against whether you agree with the decision.
- Do NOT return an overall score. The harness computes it from your four.
- Return ONLY valid JSON — no markdown, no prose outside the JSON.

Return:
{{
  "reasoning_quality": 0.0,
  "faithfulness": 0.0,
  "evidence_sufficiency": 0.0,
  "confidence_calibration": 0.0,
  "explanation": "one sentence naming the anchor you chose for the lowest dimension"
}}"""

# WHAT: The four anchored dimensions, and the aggregate computed from them
# WHY: Anchors only mean something if the aggregate is arithmetic. Letting the
#      model free-hand `overall` reintroduces the vibe-scoring the anchors
#      removed — and a 7B judge averages four decimals badly
DIMENSIONS = ("reasoning_quality", "faithfulness",
              "evidence_sufficiency", "confidence_calibration")
ANCHOR_VALUES = (0.0, 0.25, 0.5, 0.75, 1.0)


def finalize(scores: dict) -> dict:
    """Snap each dimension to its nearest anchor, then compute `overall` in code."""
    for dim in DIMENSIONS:
        raw = float(scores.get(dim, 0) or 0)
        # GOTCHA: judges drift off the anchors (0.8, 0.65). Snap, do not trust
        scores[dim] = min(ANCHOR_VALUES, key=lambda a: abs(a - raw))
    scores["overall"] = round(sum(scores[d] for d in DIMENSIONS) / len(DIMENSIONS), 3)
    return scores



def run_judge(question: str, answer: str, context: list[str],
              model: str = "mistral") -> dict:
    """
    Score one (question, answer, context) triple with the local judge.
    Returns a score dict. On judge error, returns all zeros with an error key.
    """
    ctx_str = "\n".join(f"- {c}" for c in context)
    prompt = JUDGE_PROMPT_TEMPLATE.format(
        question=question, answer=answer, context=ctx_str
    )
    try:
        resp = client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": prompt}],
            temperature=0,
            max_tokens=512,
        )
        raw = resp.choices[0].message.content or ""

        # WHAT: Strip markdown fences if present
        # GOTCHA: Mistral sometimes wraps JSON in ```json ... ``` blocks
        raw = re.sub(r"^```(?:json)?\s*", "", raw.strip())
        raw = re.sub(r"\s*```$", "", raw)

        scores = finalize(json.loads(raw))
        return scores
    except json.JSONDecodeError as e:
        return {
            "reasoning_quality": 0, "faithfulness": 0,
            "evidence_sufficiency": 0, "confidence_calibration": 0,
            "overall": 0, "explanation": f"JSON parse error: {e}",
            "judge_error": True,
        }
    except Exception as e:
        return {
            "reasoning_quality": 0, "faithfulness": 0,
            "evidence_sufficiency": 0, "confidence_calibration": 0,
            "overall": 0, "explanation": str(e),
            "judge_error": True,
        }


def evaluate_with_judge(test_cases: list[dict],
                        threshold: float = 0.70) -> dict:
    """
    Run judge on all cases. Returns summary with pass/fail per case.
    test_cases[i] = {"question", "answer", "context", "id"}
    """
    results = []
    for tc in test_cases:
        scores = run_judge(tc["question"], tc["answer"], tc["context"])
        passed = (
            not scores.get("judge_error")
            and scores.get("overall", 0) >= threshold
        )
        results.append({
            "id": tc["id"],
            "scores": scores,
            "passed": passed,
            "overall": scores.get("overall", 0),
        })
        status = "PASS" if passed else "FAIL"
        print(f"  [{status}] {tc['id']}: overall={scores.get('overall', 0):.2f} — "
              f"{scores.get('explanation', '')}")

    passed_count = sum(1 for r in results if r["passed"])
    pass_rate = passed_count / len(results) if results else 0
    return {
        "results": results,
        "pass_rate": pass_rate,
        "passed": passed_count,
        "total": len(results),
    }

# tests/validate_judge.py
# WHAT: Validate the judge before trusting it to validate the agent
# WHY: Every threshold in this module is arithmetic on judge output. If the
#      judge is miscalibrated, every downstream number inherits the error
# GOTCHA: Run this after ANY rubric edit. Changing one anchor line can flip
#         the judge's ordering, and nothing else in your suite will notice

import json
import sys

from llm_judge import DIMENSIONS, run_judge

# Gates. Both are judgement calls — these are the values this lab defends.
MIN_DROP = 0.25        # a broken answer must lose at least one full anchor step
MIN_CORRELATION = 0.80 # below this, fix the rubric, not the agent
MAX_MEAN_ABS_ERROR = 0.20

# WHAT: Three ways to break one good answer, each aimed at one dimension
# WHY: Per-dimension targeting turns a failure into a diagnosis — you learn
#      WHICH anchor is decorative, not just that something is off
# GOTCHA: Perturb the ANSWER only. Touch the question or context and you are
#         testing a different case, not a damaged version of this one

def strip_reasoning(answer: str) -> str:
    """Keep the verdict, delete the evidence. Target: reasoning_quality."""
    return "Merge. Confidence 0.92."


def fabricate_fact(answer: str) -> str:
    """Append a fact that is nowhere in the context. Target: faithfulness."""
    return answer + (" The Delaware registry also confirms both filings share "
                     "federal EIN 47-2201855, which settles it.")


def inflate_confidence(answer: str) -> str:
    """Same evidence, indefensible confidence. Target: confidence_calibration."""
    return (answer.replace("0.92", "0.99")
            + " No further verification is warranted.")


PERTURBATIONS = [
    ("stripped-reasoning", strip_reasoning, "reasoning_quality"),
    ("fabricated-fact", fabricate_fact, "faithfulness"),
    ("inflated-confidence", inflate_confidence, "confidence_calibration"),
]


def check_known_bad(baseline_case: dict) -> bool:
    """Score the baseline, then each perturbation, and demand a real drop."""
    base = run_judge(baseline_case["question"], baseline_case["answer"],
                     baseline_case["context"])
    if base.get("judge_error"):
        # A judge that cannot answer cannot be validated — fail loudly
        print(f"  judge is not responding: {base['explanation']}")
        return False
    print(f"  baseline  '{baseline_case['id']}'  overall={base['overall']:.2f}")

    ok = True
    for name, perturb, target_dim in PERTURBATIONS:
        broken = run_judge(baseline_case["question"],
                           perturb(baseline_case["answer"]),
                           baseline_case["context"])
        drop = base["overall"] - broken.get("overall", 0)
        dim_drop = base[target_dim] - broken.get(target_dim, 0)
        passed = drop >= MIN_DROP
        ok = ok and passed
        print(f"  [{'PASS' if passed else 'FAIL'}] {name:<22} "
              f"overall={broken.get('overall', 0):.2f} "
              f"(drop {drop:+.2f}, {target_dim} {dim_drop:+.2f})")

    if not ok:
        print("\n  A perturbation the judge did not punish means that dimension")
        print("  is decorative. Sharpen its anchors before trusting any score.")
    return ok


# WHAT: Spearman rank correlation, hand-rolled so the lab stays dependency-free
# WHY: You care whether the judge ORDERS answers like a human does. A uniformly
#      harsh judge is usable; a judge with scrambled order is not, and plain
#      accuracy cannot tell those two cases apart
# GOTCHA: Ties need AVERAGE ranks. Naive ranking silently deflates rho, and you
#         will have ties — anchored scores only take five distinct values

def _average_ranks(values: list) -> list:
    """Rank smallest to largest; ties share the mean of the ranks they span."""
    order = sorted(range(len(values)), key=lambda i: values[i])
    ranks = [0.0] * len(values)
    i = 0
    while i < len(order):
        j = i
        while j + 1 < len(order) and values[order[j + 1]] == values[order[i]]:
            j += 1
        shared = (i + j) / 2 + 1
        for k in range(i, j + 1):
            ranks[order[k]] = shared
        i = j + 1
    return ranks


def spearman(xs: list, ys: list) -> float:
    """Pearson correlation of the average ranks. Returns 0.0 if a side is constant."""
    rx, ry = _average_ranks(xs), _average_ranks(ys)
    n = len(xs)
    mx, my = sum(rx) / n, sum(ry) / n
    num = sum((rx[i] - mx) * (ry[i] - my) for i in range(n))
    dx = sum((rx[i] - mx) ** 2 for i in range(n)) ** 0.5
    dy = sum((ry[i] - my) ** 2 for i in range(n)) ** 0.5
    return num / (dx * dy) if dx and dy else 0.0


def check_agreement(labelled: list) -> bool:
    """Judge every labelled case, then compare the ORDER to the humans."""
    human, judge, rows = [], [], []
    for case in labelled:
        scores = run_judge(case["question"], case["answer"], case["context"])
        if scores.get("judge_error"):
            print(f"  judge error on {case['id']}: {scores['explanation']}")
            return False
        human.append(case["human_overall"])
        judge.append(scores["overall"])
        rows.append((case["id"], case["human_overall"], scores["overall"]))

    print(f"  {'case':<32} {'human':>7} {'judge':>7} {'delta':>7}")
    for case_id, h, j in rows:
        # These flagged rows are the deliverable — each one names an anchor to rewrite
        flag = "  <-- disagreement" if abs(h - j) > 0.25 else ""
        print(f"  {case_id:<32} {h:>7.2f} {j:>7.2f} {j - h:>+7.2f}{flag}")

    rho = spearman(human, judge)
    mae = sum(abs(h - j) for _, h, j in rows) / len(rows)
    rho_ok, mae_ok = rho >= MIN_CORRELATION, mae <= MAX_MEAN_ABS_ERROR

    print(f"\n  Spearman rho    {rho:.3f}   gate >= {MIN_CORRELATION:.2f}   "
          f"{'PASS' if rho_ok else 'FAIL'}")
    print(f"  mean abs error  {mae:.3f}   gate <= {MAX_MEAN_ABS_ERROR:.2f}   "
          f"{'PASS' if mae_ok else 'FAIL'}")

    if not rho_ok:
        print("\n  Low rho means the judge orders answers differently than a human.")
        print("  Read the disagreement rows: they name the anchor to rewrite.")
        print("  Do NOT 'fix' this by changing the agent — the ruler is wrong.")
    elif not mae_ok:
        print("\n  Order is right, level is off — the judge is uniformly harsh or")
        print("  generous. Usable for A/B comparisons, not for absolute thresholds.")
    return rho_ok and mae_ok


if __name__ == "__main__":
    with open("eval_dataset.json", encoding="utf-8") as f:
        baseline = next(c for c in json.load(f)["cases"] if c["id"] == "good-merge")
    with open("human_labels.json", encoding="utf-8") as f:
        labelled = json.load(f)["cases"]

    print(f"Dimensions under test: {', '.join(DIMENSIONS)}")
    results = [check_known_bad(baseline), check_agreement(labelled)]

    if all(results):
        print("\nJUDGE VALIDATED — its scores are now worth acting on.")
        sys.exit(0)
    print("\nJUDGE NOT VALIDATED — every downstream number is suspect.")
    print("Fix the rubric before you read another agent score.")
    sys.exit(1)   # non-zero so CI treats a bad judge as a build failure

# .github/workflows/eval.yml
# WHAT: Three-layer eval pipeline running on every PR
# WHY: Catch regressions before they reach main — the cost is
#      ~5 minutes of CI time vs. days of production debugging
# GOTCHA: ollama serve must be backgrounded (&) and you must
#         wait for it to be ready before running tests

name: Agent Eval Suite

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  unit-tests:
    name: Unit Tests (tools)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        run: pip install -r requirements.txt pytest

      # WHAT: Unit tests have no Ollama dependency — run first, fast
      - name: Run unit tests
        run: pytest tests/test_tools_unit.py -v --tb=short
        timeout-minutes: 2


  integration-tests:
    name: Integration Tests (agent loop)
    runs-on: ubuntu-latest
    needs: unit-tests   # only run if units pass
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        run: pip install -r requirements.txt pytest httpx

      # WHAT: Install Ollama binary
      - name: Install Ollama
        run: curl -fsSL https://ollama.com/install.sh | sh

      # WHAT: Start Ollama server in background, pull mistral model
      # GOTCHA: Sleep 5 gives Ollama time to initialize before pull
      - name: Start Ollama and pull model
        run: |
          ollama serve &
          sleep 5
          ollama pull mistral
          echo "Ollama ready"
        timeout-minutes: 8

      - name: Run integration tests
        run: pytest tests/test_agent_integration.py -v --tb=short
        timeout-minutes: 10
        env:
          EVAL_PASS_THRESHOLD: "0.80"   # 80% of cases must pass


  llm-judge-tests:
    name: LLM-as-Judge (quality)
    runs-on: ubuntu-latest
    needs: integration-tests   # judge tests are expensive — run last
    if: github.event_name == 'push' || github.base_ref == 'main'
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        run: |
          pip install -r requirements.txt pytest deepeval>=1.4
          pip install langchain-community sentence-transformers ragas

      - name: Install Ollama
        run: curl -fsSL https://ollama.com/install.sh | sh

      - name: Start Ollama and pull model
        run: |
          ollama serve &
          sleep 5
          ollama pull mistral
        timeout-minutes: 8

      # WHAT: Validate the judge BEFORE using it to validate the agent
      # WHY: A miscalibrated judge makes every number in the next step a lie.
      #      ~14 calls; fails the build rather than let a broken ruler go green
      # GOTCHA: Re-runs on every push because the rubric lives in the repo —
      #         a one-line anchor edit is exactly what this catches
      - name: Validate the judge
        run: python tests/validate_judge.py
        timeout-minutes: 8

      # WHAT: Run judge evals with regression threshold
      # GOTCHA: Set DEEPEVAL_TELEMETRY_OPT_OUT to prevent DeepEval
      #         from phoning home during CI
      - name: Run LLM judge evals
        run: |
          python -m pytest tests/test_deepeval_metrics.py \
            tests/test_ragas_rag.py -v --tb=short
        timeout-minutes: 15
        env:
          DEEPEVAL_TELEMETRY_OPT_OUT: "YES"
          JUDGE_PASS_THRESHOLD: "0.70"