LLM-as-Judge with Claude: Complete Evaluation Pattern Guide

from anthropic import Anthropic
from typing import Dict, Any, Optional
import json

class ClaudeJudge:
    """Claude-powered evaluation judge"""
    
    def __init__(
        self,
        api_key: str,
        model: str = "claude-3-5-sonnet-20241022",
        temperature: float = 0.0
    ):
        self.client = Anthropic(api_key=api_key)
        self.model = model
        self.temperature = temperature
    
    def evaluate(
        self,
        task_description: str,
        reference_answer: str,
        model_output: str,
        evaluation_criteria: str,
        context: Optional[Dict[str, Any]] = None
    ) -> Dict[str, Any]:
        """
        Evaluate model output using Claude as judge
        
        Returns:
        {
            "score": float,  # 0.0-1.0
            "reasoning": str,
            "pass": bool
        }
        """
        prompt = self._build_judge_prompt(
            task_description=task_description,
            reference_answer=reference_answer,
            model_output=model_output,
            evaluation_criteria=evaluation_criteria,
            context=context
        )
        
        response = self.client.messages.create(
            model=self.model,
            max_tokens=1024,
            temperature=self.temperature,
            messages=[{"role": "user", "content": prompt}]
        )
        
        # Parse structured response
        result_text = response.content[0].text
        try:
            result = json.loads(result_text)
            return {
                "score": result["score"] / 10.0,  # Normalize to 0-1
                "reasoning": result["reasoning"],
                "pass": result["score"] >= 7  # 7/10 threshold
            }
        except (json.JSONDecodeError, KeyError) as e:
            # Fallback parsing if JSON extraction fails
            return {
                "score": 0.0,
                "reasoning": f"Judge parsing error: {e}",
                "pass": False
            }
    
    def _build_judge_prompt(
        self,
        task_description: str,
        reference_answer: str,
        model_output: str,
        evaluation_criteria: str,
        context: Optional[Dict[str, Any]]
    ) -> str:
        """Construct evaluation prompt"""
        
        context_section = ""
        if context:
            context_section = f"\n## Additional Context\n{json.dumps(context, indent=2)}\n"
        
        prompt = f"""You are an expert evaluator assessing AI system outputs.

## Task Description
{task_description}

## Reference Answer
{reference_answer}

## Model Output to Evaluate
{model_output}
{context_section}
## Evaluation Criteria
{evaluation_criteria}

Carefully evaluate the model output against the criteria. Be strict but fair.

Provide your evaluation as JSON:
{{
  "score": <0-10>,
  "reasoning": "<detailed explanation of your score>",
  "strengths": ["<what the output does well>"],
  "weaknesses": ["<what could be improved>"]
}}

Evaluation:"""
        
        return prompt

judge = ClaudeJudge(api_key=os.getenv("ANTHROPIC_API_KEY"))

result = judge.evaluate(
    task_description="Explain quantum entanglement to a 10-year-old",
    reference_answer="Quantum entanglement is when two particles are connected so that what happens to one instantly affects the other, no matter how far apart they are. It's like magic twins who always know what the other is doing.",
    model_output="Quantum entanglement occurs when particles become correlated such that the quantum state of one particle cannot be described independently of the others.",
    evaluation_criteria="""
    - Uses age-appropriate language (critical)
    - Explains the concept correctly (critical)
    - Includes a relatable analogy (important)
    - Engaging and memorable (nice-to-have)
    """
)

print(f"Score: {result['score']:.2f}")
print(f"Pass: {result['pass']}")
print(f"Reasoning: {result['reasoning']}")

JUDGE_PROMPT_TEMPLATE = """You are an expert evaluator for {domain} systems.

## Your Role
{role_description}

## Task
{task_description}

## Reference Materials
{reference_materials}

## Output to Evaluate
{model_output}

## Evaluation Criteria
{criteria_with_weights}

## Scoring Instructions
- Assign scores from 0-10 for each criterion
- 0-3: Major issues, unusable
- 4-6: Significant problems, needs revision
- 7-8: Good, minor issues
- 9-10: Excellent, meets or exceeds expectations

## Response Format
Return JSON only:
{{
  "criteria_scores": {{"criterion_name": score, ...}},
  "overall_score": weighted_average,
  "reasoning": "step-by-step explanation",
  "critical_issues": ["list any blockers"],
  "recommendations": ["concrete improvement suggestions"]
}}

Begin evaluation:"""

Helpfulness (weight: 0.3):
- Directly answers the question asked (required)
- Provides actionable steps or next actions (required)
- Includes relevant examples or context (preferred)
- Anticipates follow-up questions (bonus)

Score 9-10 if all required + preferred elements present
Score 7-8 if all required elements present
Score 4-6 if missing one required element
Score 0-3 if missing multiple required elements

CHAIN_OF_THOUGHT_PROMPT = """## Evaluation Process

Step 1: Summarize what the output is trying to achieve
Step 2: Check each evaluation criterion and note compliance
Step 3: Identify critical issues or strengths
Step 4: Assign scores based on your findings
Step 5: Explain your overall judgment

Follow this process explicitly in your reasoning section."""

def build_few_shot_judge_prompt(
    criteria: str,
    examples: List[Dict[str, Any]]
) -> str:
    """Build judge prompt with few-shot examples"""
    
    examples_section = "## Example Evaluations\n\n"
    
    for i, ex in enumerate(examples, 1):
        examples_section += f"""### Example {i}
Model Output: {ex['output']}
Score: {ex['score']}/10
Reasoning: {ex['reasoning']}

"""
    
    prompt = f"""You are evaluating AI outputs using these criteria:

{criteria}

{examples_section}

Now evaluate this output following the same standards:

Output: {{model_output}}

Evaluation:"""
    
    return prompt

EDGE_CASE_HANDLING = """## Edge Case Instructions

**If the output is empty or just whitespace:**
- Score 0, note "Empty response"

**If the output is in the wrong language:**
- Score 0-1, note "Language mismatch"

**If the output refuses to answer:**
- If refusal is correct (unsafe/impossible request): Score 8-10
- If refusal is unnecessary: Score 0-2

**If the output is off-topic:**
- Score 0-3 based on how far off-topic

**If the output is too short to evaluate:**
- Score based on what's present, note "Incomplete"

**If you're uncertain about correctness:**
- Default to lower score (6-7) and note uncertainty
- Never guess scores higher than 7 if unsure"""

# calibration/build_calibration_set.py
from typing import List, Dict, Any
import random

def build_calibration_set(
    full_dataset: List[Dict[str, Any]],
    num_samples: int = 200,
    stratify_by: str = "category"
) -> List[Dict[str, Any]]:
    """
    Build stratified calibration set
    
    Args:
        full_dataset: All test cases
        num_samples: Target calibration set size
        stratify_by: Field to stratify on (ensures balanced representation)
    
    Returns:
        Calibration set with human labels needed
    """
    # Group by stratification field
    groups = {}
    for case in full_dataset:
        key = case.get(stratify_by, "unknown")
        if key not in groups:
            groups[key] = []
        groups[key].append(case)
    
    # Sample proportionally from each group
    calibration_set = []
    samples_per_group = num_samples // len(groups)
    
    for group_name, cases in groups.items():
        sample_size = min(samples_per_group, len(cases))
        sampled = random.sample(cases, sample_size)
        calibration_set.extend(sampled)
    
    # If we didn't reach num_samples, sample more from largest groups
    while len(calibration_set) < num_samples:
        largest_group = max(groups.values(), key=len)
        remaining = [c for c in largest_group if c not in calibration_set]
        if not remaining:
            break
        calibration_set.append(random.choice(remaining))
    
    return calibration_set[:num_samples]

# calibration/annotation_tool.py
import json
from pathlib import Path

class AnnotationTool:
    """Simple CLI tool for human annotation"""
    
    def __init__(self, calibration_set: List[Dict], output_path: Path):
        self.cases = calibration_set
        self.output_path = output_path
        self.annotations = []
    
    def annotate(self):
        """Run annotation session"""
        print(f"Annotating {len(self.cases)} cases")
        print("For each case, provide:")
        print("- Score (0-10)")
        print("- Brief reasoning")
        print("- Type 'quit' to save and exit\n")
        
        for i, case in enumerate(self.cases, 1):
            print(f"\n{'='*60}")
            print(f"Case {i}/{len(self.cases)}")
            print(f"{'='*60}")
            print(f"\nTask: {case['task_description']}")
            print(f"\nModel Output:\n{case['model_output']}")
            print(f"\nReference: {case.get('reference_answer', 'N/A')}")
            
            score_input = input("\nYour score (0-10): ").strip()
            if score_input.lower() == 'quit':
                break
            
            try:
                score = float(score_input)
                if not 0 <= score <= 10:
                    print("Score must be 0-10. Skipping...")
                    continue
            except ValueError:
                print("Invalid score. Skipping...")
                continue
            
            reasoning = input("Brief reasoning: ").strip()
            
            self.annotations.append({
                'case_id': case['id'],
                'human_score': score,
                'human_reasoning': reasoning,
                'model_output': case['model_output']
            })
            
            # Auto-save every 10 cases
            if i % 10 == 0:
                self.save()
        
        self.save()
        print(f"\nAnnotation complete! {len(self.annotations)} cases labeled.")
    
    def save(self):
        """Save annotations to disk"""
        with open(self.output_path, 'w') as f:
            json.dump(self.annotations, f, indent=2)

# Usage
tool = AnnotationTool(
    calibration_set=calibration_cases,
    output_path=Path("calibration/human_labels.json")
)
tool.annotate()

# calibration/measure_agreement.py
import numpy as np
from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import cohen_kappa_score

def measure_judge_agreement(
    human_scores: List[float],
    judge_scores: List[float],
    threshold: float = 7.0  # Pass/fail cutoff
) -> Dict[str, float]:
    """
    Measure agreement between judge and human scores
    
    Returns:
    - pearson: Linear correlation
    - spearman: Rank correlation
    - mae: Mean absolute error
    - accuracy: % of pass/fail agreement
    - cohen_kappa: Agreement accounting for chance
    """
    # Correlation metrics
    pearson_r, _ = pearsonr(human_scores, judge_scores)
    spearman_r, _ = spearmanr(human_scores, judge_scores)
    
    # Error metrics
    mae = np.mean(np.abs(np.array(human_scores) - np.array(judge_scores)))
    
    # Pass/fail agreement
    human_pass = [s >= threshold for s in human_scores]
    judge_pass = [s >= threshold for s in judge_scores]
    accuracy = sum(h == j for h, j in zip(human_pass, judge_pass)) / len(human_scores)
    
    # Cohen's kappa (accounts for chance agreement)
    kappa = cohen_kappa_score(human_pass, judge_pass)
    
    return {
        'pearson_correlation': pearson_r,
        'spearman_correlation': spearman_r,
        'mean_absolute_error': mae,
        'pass_fail_accuracy': accuracy,
        'cohen_kappa': kappa,
        'num_samples': len(human_scores)
    }

# Example
agreement = measure_judge_agreement(
    human_scores=[8.0, 6.5, 9.0, 4.0, 7.5],
    judge_scores=[7.5, 6.0, 9.5, 3.5, 8.0]
)

print(f"Pearson r: {agreement['pearson_correlation']:.3f}")
print(f"Pass/fail accuracy: {agreement['pass_fail_accuracy']:.1%}")
print(f"Cohen's kappa: {agreement['cohen_kappa']:.3f}")

def find_optimal_threshold(
    human_scores: List[float],
    judge_scores: List[float],
    human_threshold: float = 7.0
) -> Dict[str, Any]:
    """
    Find judge threshold that maximizes agreement with human pass/fail
    """
    human_pass = [s >= human_threshold for s in human_scores]
    
    best_threshold = None
    best_accuracy = 0.0
    
    # Try thresholds from 5.0 to 9.0 in 0.1 increments
    for threshold in np.arange(5.0, 9.1, 0.1):
        judge_pass = [s >= threshold for s in judge_scores]
        accuracy = sum(h == j for h, j in zip(human_pass, judge_pass)) / len(human_scores)
        
        if accuracy > best_accuracy:
            best_accuracy = accuracy
            best_threshold = threshold
    
    return {
        'optimal_threshold': best_threshold,
        'accuracy_at_optimal': best_accuracy
    }

result = find_optimal_threshold(human_scores, judge_scores)
print(f"Use threshold {result['optimal_threshold']:.1f} for {result['accuracy_at_optimal']:.1%} accuracy")

def detect_length_bias(
    outputs: List[str],
    scores: List[float]
) -> Dict[str, Any]:
    """Check if longer outputs score systematically higher"""
    lengths = [len(output.split()) for output in outputs]
    
    correlation, p_value = pearsonr(lengths, scores)
    
    return {
        'length_score_correlation': correlation,
        'p_value': p_value,
        'significant_bias': p_value < 0.05 and correlation > 0.3
    }

def detect_position_bias(judgments: List[Dict]) -> float:
    """
    Run same comparison twice with swapped positions
    Returns: % of times judge flips preference
    """
    flips = 0
    for judgment in judgments:
        if judgment['first_run_winner'] != judgment['second_run_winner']:
            flips += 1
    
    flip_rate = flips / len(judgments)
    return flip_rate

# Position bias present if flip rate > 20%

def detect_self_preference_bias(
    claude_outputs: List[tuple[str, float]],  # (output, judge_score)
    gpt_outputs: List[tuple[str, float]]
) -> Dict[str, Any]:
    """Check if judge favors outputs from its own model family"""
    claude_scores = [score for _, score in claude_outputs]
    gpt_scores = [score for _, score in gpt_outputs]
    
    from scipy.stats import ttest_ind
    t_stat, p_value = ttest_ind(claude_scores, gpt_scores)
    
    return {
        'claude_mean': np.mean(claude_scores),
        'gpt_mean': np.mean(gpt_scores),
        'difference': np.mean(claude_scores) - np.mean(gpt_scores),
        'statistically_significant': p_value < 0.05
    }

class BiasAwareJudge(ClaudeJudge):
    """Judge with automatic bias detection and mitigation"""
    
    def evaluate_with_bias_check(
        self,
        *args,
        detect_bias: bool = True,
        **kwargs
    ) -> Dict[str, Any]:
        """Run evaluation with optional bias checks"""
        
        # Run base evaluation
        result = self.evaluate(*args, **kwargs)
        
        if not detect_bias:
            return result
        
        # Check for length bias
        output_length = len(kwargs['model_output'].split())
        if output_length > 500 and result['score'] > 8.5:
            # Long output with high score - potential length bias
            # Re-evaluate with length warning
            kwargs_copy = kwargs.copy()
            kwargs_copy['evaluation_criteria'] += "\n\nIMPORTANT: Score based on quality, not length. Do not reward verbosity."
            
            adjusted_result = self.evaluate(*args, **kwargs_copy)
            
            if abs(adjusted_result['score'] - result['score']) > 0.5:
                result['bias_warning'] = f"Length bias suspected (score changed from {result['score']:.2f} to {adjusted_result['score']:.2f} with anti-length prompt)"
                result['adjusted_score'] = adjusted_result['score']
        
        return result

@dataclass
class EvaluationCriterion:
    name: str
    description: str
    weight: float
    scale: str  # e.g., "0-10" or "binary"
    examples: Dict[int, str]  # score -> example description

@dataclass
class EvaluationRubric:
    criteria: List[EvaluationCriterion]
    aggregation_method: str = "weighted_average"  # or "minimum" or "custom"
    
    def validate(self):
        """Ensure rubric is well-formed"""
        total_weight = sum(c.weight for c in self.criteria)
        if not np.isclose(total_weight, 1.0):
            raise ValueError(f"Weights must sum to 1.0, got {total_weight}")

# Example: Customer support response rubric
support_rubric = EvaluationRubric(
    criteria=[
        EvaluationCriterion(
            name="accuracy",
            description="Information provided is factually correct per knowledge base",
            weight=0.35,
            scale="0-10",
            examples={
                10: "All facts verified against KB, no errors",
                7: "Mostly accurate with minor imprecision",
                3: "Contains significant factual errors",
                0: "Completely incorrect information"
            }
        ),
        EvaluationCriterion(
            name="completeness",
            description="Addresses all aspects of customer question",
            weight=0.25,
            scale="0-10",
            examples={
                10: "Answers question fully plus relevant context",
                7: "Answers main question but misses minor details",
                3: "Only partially addresses question",
                0: "Does not address the question"
            }
        ),
        EvaluationCriterion(
            name="tone",
            description="Professional, empathetic, brand-appropriate",
            weight=0.20,
            scale="0-10",
            examples={
                10: "Perfect tone for situation and brand",
                7: "Acceptable but could be warmer",
                3: "Tone issues (too formal/casual/cold)",
                0: "Inappropriate or offensive tone"
            }
        ),
        EvaluationCriterion(
            name="actionability",
            description="Provides clear next steps for customer",
            weight=0.20,
            scale="0-10",
            examples={
                10: "Clear, specific next steps provided",
                7: "Some guidance but could be more specific",
                3: "Vague or confusing guidance",
                0: "No actionable information"
            }
        )
    ]
)

class RubricJudge(ClaudeJudge):
    """Judge that evaluates against multi-criteria rubric"""
    
    def __init__(self, *args, rubric: EvaluationRubric, **kwargs):
        super().__init__(*args, **kwargs)
        self.rubric = rubric
        self.rubric.validate()
    
    def evaluate_with_rubric(
        self,
        model_output: str,
        reference: str,
        context: Optional[Dict] = None
    ) -> Dict[str, Any]:
        """Evaluate output against rubric"""
        
        # Build rubric prompt
        rubric_text = self._format_rubric_for_prompt()
        
        prompt = f"""Evaluate this output against the rubric below.

## Output to Evaluate
{model_output}

## Reference/Expected
{reference}

## Evaluation Rubric
{rubric_text}

For each criterion, assign a score and explain your reasoning.

Return JSON:
{{
  "criteria_scores": {{
    "criterion_name": {{
      "score": 0-10,
      "reasoning": "why this score"
    }},
    ...
  }},
  "overall_score": weighted_average,
  "summary": "brief overall assessment"
}}

Evaluation:"""
        
        response = self.client.messages.create(
            model=self.model,
            max_tokens=2048,
            temperature=0.0,
            messages=[{"role": "user", "content": prompt}]
        )
        
        result = json.loads(response.content[0].text)
        
        # Compute weighted score
        weighted_score = 0.0
        for criterion in self.rubric.criteria:
            score = result['criteria_scores'][criterion.name]['score']
            weighted_score += (score / 10.0) * criterion.weight
        
        return {
            'criteria_scores': result['criteria_scores'],
            'overall_score': weighted_score,
            'pass': weighted_score >= 0.7,
            'summary': result['summary']
        }
    
    def _format_rubric_for_prompt(self) -> str:
        """Convert rubric to prompt-friendly text"""
        lines = []
        for criterion in self.rubric.criteria:
            lines.append(f"\n### {criterion.name.title()} (weight: {criterion.weight})")
            lines.append(criterion.description)
            lines.append("\nScoring guide:")
            for score, example in sorted(criterion.examples.items(), reverse=True):
                lines.append(f"  {score}: {example}")
        
        return "\n".join(lines)

def analyze_rubric_results(
    results: List[Dict[str, Any]],
    rubric: EvaluationRubric
) -> Dict[str, Any]:
    """Analyze where system needs improvement"""
    
    criterion_averages = {}
    for criterion in rubric.criteria:
        scores = [
            r['criteria_scores'][criterion.name]['score']
            for r in results
        ]
        criterion_averages[criterion.name] = {
            'mean': np.mean(scores),
            'std': np.std(scores),
            'min': np.min(scores),
            'failing_cases': sum(1 for s in scores if s < 7)
        }
    
    # Find weakest criterion
    weakest = min(
        criterion_averages.items(),
        key=lambda x: x[1]['mean']
    )
    
    # Find most variable criterion
    most_variable = max(
        criterion_averages.items(),
        key=lambda x: x[1]['std']
    )
    
    return {
        'criterion_averages': criterion_averages,
        'weakest_criterion': weakest[0],
        'weakest_score': weakest[1]['mean'],
        'most_variable_criterion': most_variable[0],
        'most_variable_std': most_variable[1]['std']
    }

# Usage
analysis = analyze_rubric_results(eval_results, support_rubric)
print(f"Focus improvement on: {analysis['weakest_criterion']}")
print(f"Average score: {analysis['weakest_score']:.2f}/10")

class HybridJudge:
    """Fast rule checks + LLM judgment for complex cases"""
    
    def __init__(self, llm_judge: ClaudeJudge):
        self.llm_judge = llm_judge
    
    def evaluate(
        self,
        model_output: str,
        expected: str,
        context: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Evaluate with rule pre-filters"""
        
        # Fast rule checks
        rule_result = self._check_rules(model_output, expected, context)
        
        if rule_result['auto_fail']:
            return {
                'score': 0.0,
                'pass': False,
                'reasoning': rule_result['reason'],
                'method': 'rule_based'
            }
        
        if rule_result['auto_pass']:
            return {
                'score': 1.0,
                'pass': True,
                'reasoning': rule_result['reason'],
                'method': 'rule_based'
            }
        
        # Complex case - use LLM judge
        llm_result = self.llm_judge.evaluate(
            task_description=context.get('task', ''),
            reference_answer=expected,
            model_output=model_output,
            evaluation_criteria=context.get('criteria', '')
        )
        llm_result['method'] = 'llm_judge'
        return llm_result
    
    def _check_rules(
        self,
        output: str,
        expected: str,
        context: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Fast rule-based checks"""
        
        # Auto-fail conditions
        if not output or output.isspace():
            return {'auto_fail': True, 'reason': 'Empty output'}
        
        if len(output) < 10:
            return {'auto_fail': True, 'reason': 'Output too short'}
        
        # Check for required phrases (if specified)
        required_phrases = context.get('required_phrases', [])
        for phrase in required_phrases:
            if phrase.lower() not in output.lower():
                return {'auto_fail': True, 'reason': f'Missing required phrase: {phrase}'}
        
        # Check for forbidden phrases
        forbidden_phrases = context.get('forbidden_phrases', [])
        for phrase in forbidden_phrases:
            if phrase.lower() in output.lower():
                return {'auto_fail': True, 'reason': f'Contains forbidden phrase: {phrase}'}
        
        # Auto-pass conditions (exact match)
        if output.strip() == expected.strip():
            return {'auto_pass': True, 'reason': 'Exact match'}
        
        # Need LLM judgment
        return {'auto_fail': False, 'auto_pass': False}

class EnsembleJudge:
    """Multiple judges with voting/averaging"""
    
    def __init__(
        self,
        judges: List[ClaudeJudge],
        aggregation: str = "mean"  # or "median" or "vote"
    ):
        self.judges = judges
        self.aggregation = aggregation
    
    def evaluate(self, *args, **kwargs) -> Dict[str, Any]:
        """Evaluate with all judges and aggregate"""
        
        judge_results = []
        for judge in self.judges:
            result = judge.evaluate(*args, **kwargs)
            judge_results.append(result)
        
        # Aggregate scores
        scores = [r['score'] for r in judge_results]
        
        if self.aggregation == "mean":
            final_score = np.mean(scores)
        elif self.aggregation == "median":
            final_score = np.median(scores)
        elif self.aggregation == "vote":
            # Majority vote on pass/fail
            passes = sum(1 for r in judge_results if r['pass'])
            final_score = passes / len(judge_results)
        else:
            raise ValueError(f"Unknown aggregation: {self.aggregation}")
        
        # Measure agreement
        score_std = np.std(scores)
        
        return {
            'score': final_score,
            'pass': final_score >= 0.7,
            'individual_scores': scores,
            'score_std': score_std,
            'high_disagreement': score_std > 0.2,  # Flag for manual review
            'reasoning': judge_results[0]['reasoning']  # Use first judge's reasoning
        }

# Usage with different temperature judges
ensemble = EnsembleJudge(
    judges=[
        ClaudeJudge(api_key=key, temperature=0.0),
        ClaudeJudge(api_key=key, temperature=0.0),  # Same temp, different random seed
        ClaudeJudge(api_key=key, temperature=0.3)
    ],
    aggregation="median"
)

class HITLJudge:
    """LLM judge with human escalation for uncertain cases"""
    
    def __init__(
        self,
        llm_judge: ClaudeJudge,
        confidence_threshold: float = 0.8
    ):
        self.llm_judge = llm_judge
        self.confidence_threshold = confidence_threshold
        self.human_queue = []
    
    def evaluate(self, *args, **kwargs) -> Dict[str, Any]:
        """Evaluate with human fallback"""
        
        result = self.llm_judge.evaluate(*args, **kwargs)
        
        # Check confidence (based on reasoning length, hedging language, etc.)
        confidence = self._estimate_confidence(result['reasoning'])
        result['confidence'] = confidence
        
        if confidence < self.confidence_threshold:
            result['needs_human_review'] = True
            self.human_queue.append({
                'args': args,
                'kwargs': kwargs,
                'llm_result': result
            })
        else:
            result['needs_human_review'] = False
        
        return result
    
    def _estimate_confidence(self, reasoning: str) -> float:
        """Estimate judge confidence from reasoning text"""
        # Look for hedging language
        hedging_phrases = [
            'unclear', 'uncertain', 'hard to say', 'difficult to judge',
            'could be', 'might be', 'possibly', 'perhaps'
        ]
        
        hedging_count = sum(
            1 for phrase in hedging_phrases
            if phrase in reasoning.lower()
        )
        
        # More hedging = lower confidence
        confidence = max(0.0, 1.0 - (hedging_count * 0.15))
        return confidence
    
    def get_human_queue(self) -> List[Dict]:
        """Get cases requiring human review"""
        return self.human_queue

def calculate_judge_cost(
    num_evaluations: int,
    avg_input_tokens: int = 1500,  # Typical with context
    avg_output_tokens: int = 300,  # Typical reasoning length
    model: str = "claude-3-5-sonnet-20241022"
) -> Dict[str, float]:
    """Estimate judging costs"""
    
    # Pricing as of 2026-09
    pricing = {
        'claude-3-5-sonnet-20241022': {
            'input': 3.00 / 1_000_000,  # per token
            'output': 15.00 / 1_000_000
        },
        'claude-3-haiku-20240307': {
            'input': 0.25 / 1_000_000,
            'output': 1.25 / 1_000_000
        },
        'gpt-4o': {
            'input': 5.00 / 1_000_000,
            'output': 15.00 / 1_000_000
        }
    }
    
    rates = pricing[model]
    
    input_cost = num_evaluations * avg_input_tokens * rates['input']
    output_cost = num_evaluations * avg_output_tokens * rates['output']
    total_cost = input_cost + output_cost
    
    return {
        'input_cost': input_cost,
        'output_cost': output_cost,
        'total_cost': total_cost,
        'cost_per_eval': total_cost / num_evaluations
    }

# Example: 10,000 evaluations
cost = calculate_judge_cost(num_evaluations=10_000)
print(f"Total cost: ${cost['total_cost']:.2f}")
print(f"Cost per evaluation: ${cost['cost_per_eval']:.4f}")

class TieredJudge:
    """Route to appropriate judge based on complexity"""
    
    def __init__(
        self,
        cheap_judge: ClaudeJudge,  # Haiku
        premium_judge: ClaudeJudge  # Sonnet
    ):
        self.cheap = cheap_judge
        self.premium = premium_judge
    
    def evaluate(self, *args, complexity: str = "auto", **kwargs) -> Dict[str, Any]:
        """Route based on complexity"""
        
        if complexity == "auto":
            complexity = self._estimate_complexity(kwargs.get('model_output', ''))
        
        if complexity == "simple":
            result = self.cheap.evaluate(*args, **kwargs)
            result['judge_model'] = 'haiku'
        else:
            result = self.premium.evaluate(*args, **kwargs)
            result['judge_model'] = 'sonnet'
        
        return result
    
    def _estimate_complexity(self, output: str) -> str:
        """Estimate task complexity"""
        # Simple heuristics
        if len(output) < 100:
            return "simple"
        if any(word in output.lower() for word in ['however', 'although', 'nuance']):
            return "complex"
        return "simple"

import hashlib
from functools import lru_cache

class CachedJudge(ClaudeJudge):
    """Judge with result caching"""
    
    def __init__(self, *args, cache_size: int = 10000, **kwargs):
        super().__init__(*args, **kwargs)
        self.cache = {}
        self.cache_size = cache_size
    
    def evaluate(self, *args, **kwargs) -> Dict[str, Any]:
        """Evaluate with caching"""
        
        # Generate cache key
        cache_key = self._make_cache_key(args, kwargs)
        
        if cache_key in self.cache:
            return self.cache[cache_key].copy()
        
        # Cache miss - run evaluation
        result = super().evaluate(*args, **kwargs)
        
        # Store in cache
        if len(self.cache) >= self.cache_size:
            # Evict oldest entry (simplified LRU)
            self.cache.pop(next(iter(self.cache)))
        
        self.cache[cache_key] = result.copy()
        return result
    
    def _make_cache_key(self, args, kwargs) -> str:
        """Generate cache key from inputs"""
        # Combine all inputs into deterministic string
        key_parts = [str(arg) for arg in args]
        key_parts.extend([f"{k}={v}" for k, v in sorted(kwargs.items())])
        key_string = "|".join(key_parts)
        
        # Hash for compact key
        return hashlib.sha256(key_string.encode()).hexdigest()

class BatchJudge(ClaudeJudge):
    """Process evaluations in batches"""
    
    def evaluate_batch(
        self,
        cases: List[Dict[str, Any]],
        max_batch_size: int = 20
    ) -> List[Dict[str, Any]]:
        """
        Evaluate multiple cases in single API call
        
        Reduces per-call overhead and can enable future batch pricing
        """
        results = []
        
        for i in range(0, len(cases), max_batch_size):
            batch = cases[i:i+max_batch_size]
            
            # Build batch prompt
            batch_prompt = self._build_batch_prompt(batch)
            
            # Single API call for entire batch
            response = self.client.messages.create(
                model=self.model,
                max_tokens=4096,
                temperature=0.0,
                messages=[{"role": "user", "content": batch_prompt}]
            )
            
            # Parse batch results
            batch_results = self._parse_batch_response(response.content[0].text)
            results.extend(batch_results)
        
        return results
    
    def _build_batch_prompt(self, cases: List[Dict]) -> str:
        """Build prompt for batch evaluation"""
        prompt = "Evaluate the following outputs. Return JSON array of evaluations.\n\n"
        
        for i, case in enumerate(cases):
            prompt += f"""
## Case {i+1}
Task: {case['task']}
Output: {case['output']}
Expected: {case['expected']}

"""
        
        prompt += """
Return JSON array:
[
  {"case_id": 1, "score": X, "reasoning": "..."},
  {"case_id": 2, "score": Y, "reasoning": "..."},
  ...
]
"""
        return prompt
    
    def _parse_batch_response(self, response: str) -> List[Dict]:
        """Parse batch evaluation response"""
        try:
            return json.loads(response)
        except json.JSONDecodeError:
            # Fallback parsing
            return []