RAG Pipeline Evaluation & Faithfulness Auditing

from dataclasses import dataclass
from typing import List, Dict, Optional

@dataclass
class RetrievedChunk:
    chunk_id: str
    text: str
    relevance_score: float
    is_ground_truth: bool

@dataclass
class RAGEvalMetrics:
    context_precision: float
    context_recall: float
    faithfulness_score: float
    abstention_correct: bool
    citation_accuracy: float

class RAGPipelineEvaluator:
    """Evaluates RAG retriever precision and generator faithfulness against gold ground truth."""
    def __init__(self, retrieved_chunks: List[RetrievedChunk], gold_facts: List[str], unanswerable_keys: List[str]):
        self.chunks = retrieved_chunks
        self.gold_facts = gold_facts
        self.unanswerable = unanswerable_keys

    def evaluate(self, generated_response: str, cited_chunk_ids: List[str]) -> RAGEvalMetrics:
        # 1. Retriever Precision@K: Ratio of retrieved chunks that are true gold sources
        true_positives = sum(1 for c in self.chunks if c.is_ground_truth)
        precision = true_positives / len(self.chunks) if self.chunks else 0.0
        recall = 1.0 if true_positives >= len(self.gold_facts) else (true_positives / len(self.gold_facts))

        # 2. Generator Abstention Check: Did model explicitly refuse unanswerable variables?
        refused_missing = all(
            key.lower() in generated_response.lower() and ("not provided" in generated_response.lower() or "absent" in generated_response.lower())
            for key in self.unanswerable
        )

        # 3. Citation Fidelity: Are citations pointing strictly to retrieved ground-truth chunks?
        valid_citations = sum(1 for cid in cited_chunk_ids if any(c.chunk_id == cid and c.is_ground_truth for c in self.chunks))
        citation_acc = valid_citations / len(cited_chunk_ids) if cited_chunk_ids else 0.0

        return RAGEvalMetrics(
            context_precision=precision,
            context_recall=recall,
            faithfulness_score=1.0 if (refused_missing and citation_acc >= 0.8) else 0.25,
            abstention_correct=refused_missing,
            citation_accuracy=citation_acc
        )