evaluate_mmlu_real.py

#!/usr/bin/env python3
"""
REAL MMLU Evaluation with Full Dataset
====================================
Evaluates on the actual MMLU benchmark using real models and full dataset.
NO MORE HARDCODED SAMPLES OR FAKE 100% ACCURACY!

Author: Principal Neuro-AI Engineer
Date: January 8, 2026
"""

import sys
import time
import json
import argparse
from pathlib import Path
from typing import Dict, List, Any
import logging

# Import real evaluation components
from real_dataset_loader import RealDatasetLoader, install_datasets_if_needed
from real_model_inference import RealSpecialistSystem, install_required_packages

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def run_real_mmlu_evaluation(max_samples: int = None, use_real_models: bool = True, 
                            subjects_filter: List[str] = None):
    """Run REAL MMLU evaluation on full dataset with real models"""
    
    print("[TARGET] REAL MMLU Evaluation - Full Dataset")
    print("=" * 60)
    print(f"[CHART] Max samples: {max_samples or 'ALL'}")
    print(f"? Real models: {'YES' if use_real_models else 'NO (simulation)'}")
    print(f"[DATA] Subject filter: {subjects_filter or 'ALL SUBJECTS'}")
    
    # Check and install dependencies
    if use_real_models:
        try:
            from transformers import AutoTokenizer
        except ImportError:
            print("? Installing model dependencies...")
            if not install_required_packages():
                print("[FAIL] Failed to install model dependencies")
                return None
    
    if not install_datasets_if_needed():
        print("[FAIL] Failed to install dataset dependencies")
        return None
    
    # Load real datasets
    print("\n[CYCLE] Loading REAL MMLU dataset...")
    loader = RealDatasetLoader()
    
    try:
        mmlu_samples = loader.load_mmlu_full(max_samples=max_samples)
        if not mmlu_samples:
            print("[FAIL] No MMLU samples loaded")
            return None
            
        print(f"[OK] Loaded {len(mmlu_samples)} REAL MMLU samples")
        
        # Filter by subjects if specified
        if subjects_filter:
            original_count = len(mmlu_samples)
            mmlu_samples = [s for s in mmlu_samples if s['subject'] in subjects_filter]
            print(f"[SEARCH] Filtered to {len(mmlu_samples)} samples from {original_count}")
            
    except Exception as e:
        logger.error(f"Failed to load MMLU dataset: {e}")
        print(f"[FAIL] Failed to load dataset: {e}")
        return None
    
    # Initialize real specialist system
    print(f"\n? Initializing {'REAL' if use_real_models else 'SIMULATION'} specialist system...")
    specialist_system = RealSpecialistSystem(use_real_models=use_real_models)
    
    # Run evaluation
    print(f"\n[ROCKET] Starting evaluation on {len(mmlu_samples)} questions...")
    
    results = []
    subject_performance = {}
    specialist_usage = {}
    start_time = time.time()
    
    for i, sample in enumerate(mmlu_samples):
        print(f"\r? Processing {i+1}/{len(mmlu_samples)} ({sample['subject']})...", end='', flush=True)
        
        try:
            result = specialist_system.evaluate_question(
                question=sample['question'],
                choices=sample['choices'], 
                correct_answer=sample['answer'],
                subject=sample['subject']
            )
            
            results.append(result)
            
            # Track performance by subject
            subject = sample['subject']
            if subject not in subject_performance:
                subject_performance[subject] = {'correct': 0, 'total': 0, 'confidence_sum': 0}
            
            subject_performance[subject]['total'] += 1
            subject_performance[subject]['confidence_sum'] += result['confidence']
            
            if result['is_correct']:
                subject_performance[subject]['correct'] += 1
            
            # Track specialist usage
            specialist = result['selected_specialist']
            if specialist not in specialist_usage:
                specialist_usage[specialist] = 0
            specialist_usage[specialist] += 1
            
        except Exception as e:
            logger.error(f"Error processing sample {i}: {e}")
            continue
    
    print()  # New line after progress
    
    # Calculate overall metrics
    total_time = time.time() - start_time
    total_correct = sum(1 for r in results if r['is_correct'])
    total_questions = len(results)
    
    if total_questions == 0:
        print("[FAIL] No questions processed successfully")
        return None
        
    overall_accuracy = total_correct / total_questions
    avg_confidence = sum(r['confidence'] for r in results) / len(results)
    avg_processing_time = sum(r['processing_time_ms'] for r in results) / len(results)
    
    # Display results
    print(f"\n? REAL MMLU Results Summary")
    print("=" * 60)
    print(f"   Overall Accuracy: {overall_accuracy:.1%} ({total_correct}/{total_questions})")
    print(f"   Average Confidence: {avg_confidence:.3f}")
    print(f"   Average Processing Time: {avg_processing_time:.1f}ms")
    print(f"   Total Runtime: {total_time:.1f}s")
    print(f"   Model Mode: {'REAL MODELS' if use_real_models else 'SIMULATION'}")
    
    print(f"\n[STATS] Subject Performance:")
    for subject, perf in sorted(subject_performance.items()):
        accuracy = perf['correct'] / perf['total']
        avg_conf = perf['confidence_sum'] / perf['total']
        print(f"   {subject:20s}: {accuracy:5.1%} ({perf['correct']:3d}/{perf['total']:3d}) conf={avg_conf:.2f}")
    
    print(f"\n[TARGET] Specialist Selection Analysis:")
    for specialist, count in sorted(specialist_usage.items()):
        percentage = (count / len(results)) * 100
        print(f"   {specialist:20s}: {count:3d} times ({percentage:4.1f}%)")
    
    # Save results
    timestamp = int(time.time())
    results_data = {
        "metadata": {
            "evaluation_type": "real_mmlu",
            "use_real_models": use_real_models,
            "total_samples": total_questions,
            "subjects_filter": subjects_filter,
            "timestamp": timestamp,
            "date": time.strftime("%Y-%m-%d %H:%M:%S")
        },
        "summary": {
            "overall_accuracy": overall_accuracy,
            "total_correct": total_correct,
            "total_questions": total_questions,
            "avg_confidence": avg_confidence,
            "avg_processing_time_ms": avg_processing_time,
            "total_runtime_seconds": total_time
        },
        "subject_performance": {
            subject: {
                "accuracy": perf['correct'] / perf['total'],
                "correct": perf['correct'],
                "total": perf['total'],
                "avg_confidence": perf['confidence_sum'] / perf['total']
            }
            for subject, perf in subject_performance.items()
        },
        "specialist_usage": specialist_usage,
        "detailed_results": results[:100]  # Save first 100 detailed results
    }
    
    results_file = f"real_mmlu_results_{timestamp}.json"
    with open(results_file, 'w') as f:
        json.dump(results_data, f, indent=2)
    
    print(f"\n? Results saved to: {results_file}")
    
    # Performance assessment (realistic expectations)
    if overall_accuracy >= 0.70:
        assessment = "🎉 EXCELLENT - Strong performance on real benchmark!"
    elif overall_accuracy >= 0.60:
        assessment = "[OK] VERY GOOD - Solid real-world performance!" 
    elif overall_accuracy >= 0.50:
        assessment = "👍 GOOD - Reasonable performance!"
    elif overall_accuracy >= 0.40:
        assessment = "[WARN] FAIR - Room for improvement"
    else:
        assessment = "[FAIL] POOR - Significant improvements needed"
    
    print(f"\n   Assessment: {assessment}")
    
    return results_data

def main():
    """Main function with command line arguments"""
    parser = argparse.ArgumentParser(description='Real MMLU Evaluation')
    parser.add_argument('--max-samples', type=int, default=None, 
                       help='Maximum number of samples to evaluate (default: all)')
    parser.add_argument('--simulation', action='store_true',
                       help='Use simulation instead of real models')
    parser.add_argument('--subjects', nargs='+', default=None,
                       help='Filter specific subjects (e.g. mathematics physics)')
    parser.add_argument('--quick-test', action='store_true',
                       help='Quick test with 50 samples')
    
    args = parser.parse_args()
    
    max_samples = args.max_samples
    if args.quick_test:
        max_samples = 50
    
    use_real_models = not args.simulation
    
    # Run evaluation
    results = run_real_mmlu_evaluation(
        max_samples=max_samples,
        use_real_models=use_real_models,
        subjects_filter=args.subjects
    )
    
    if results is None:
        print("[FAIL] Evaluation failed")
        return 1
    
    return 0

if __name__ == "__main__":
    exit(main())