"""Step 24 — AI evaluation monitoring: datasets, regression runs, prompt versions."""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

# NOTE: The eval‑monitoring API has been disabled. The service functions are kept for
# internal dataset loading and case checking utilities used by tests.

_DATASETS_DIR = Path(__file__).resolve().parents[2] / "eval" / "datasets"

_DATASET_FILES = {
    "evaluate_answer": "evaluate_answer_cases.json",
}


def load_dataset(name: str) -> list[dict[str, Any]]:
    """Load a JSON dataset file for the given name.

    Returns an empty list if the dataset is unknown or the file does not exist.
    """
    filename = _DATASET_FILES.get(name)
    if not filename:
        return []
    path = _DATASETS_DIR / filename
    if not path.exists():
        return []
    data = json.loads(path.read_text(encoding="utf-8"))
    return data if isinstance(data, list) else []


def _check_case(case: dict[str, Any], result: Any) -> dict:
    """Validate a single evaluation case against the provided result.

    The function returns a plain dictionary rather than a Pydantic model to avoid
    importing the now‑removed ``EvalCaseResult`` schema.
    """
    expect = case.get("expect") or {}
    failures: list[str] = []
    score = getattr(result, "score", None)
    band = getattr(result, "score_band", None)
    band_val = band.value if hasattr(band, "value") else str(band) if band else None

    if score is not None:
        if "min_score" in expect and score < float(expect["min_score"]):
            failures.append(f"score {score} < min {expect['min_score']}")
        if "max_score" in expect and score > float(expect["max_score"]):
            failures.append(f"score {score} > max {expect['max_score']}")
    else:
        failures.append("missing score")

    if "score_band_in" in expect:
        allowed = set(expect["score_band_in"])
        if band_val not in allowed:
            failures.append(f"band {band_val} not in {sorted(allowed)}")
    if "score_band_not_in" in expect:
        banned = set(expect["score_band_not_in"])
        if band_val in banned:
            failures.append(f"band {band_val} in banned {sorted(banned)}")

    if expect.get("require_dimensions"):
        dims = getattr(result, "dimensions", None) or []
        keys = {getattr(d, "key", None) for d in dims}
        for req in expect["require_dimensions"]:
            if req not in keys:
                failures.append(f"missing dimension {req}")

    if expect.get("calibrated") is True and not getattr(result, "calibrated", False):
        if band_val not in {"needs_improvement"}:
            failures.append("expected calibrated=true")

    return {
        "case_id": str(case.get("id") or "unknown"),
        "passed": len(failures) == 0,
        "score": score,
        "score_band": band_val,
        "failures": failures,
    }

# The original ``EvalMonitoringService`` class has been commented out because the
# corresponding API endpoint is disabled.
# class EvalMonitoringService:
#     def __init__(self) -> None:
#         self._interviews = InterviewsService()
#
#     async def review(self, payload: Any) -> Any:
#         raise NotImplementedError("EvalMonitoringService is disabled.")
