evaluate_hellaswag.py

#!/usr/bin/env python3
"""
HellaSwag Challenge Evaluation with High-Performance BIOMIND
==========================================================

Direct evaluation of BIOMIND system on HellaSwag (commonsense reasoning).
"""

import sys
import time
import json
import random
import numpy as np
from pathlib import Path
from typing import Dict, List, Any, Optional, Tuple

# Add project root to Python path  
sys.path.insert(0, str(Path(__file__).parent / ".cleanup_backup"))

# Import working BIOMIND system
from integrated_biomind_evaluation import IntegratedBIOMINDEvaluator

def load_hellaswag_samples(num_samples: int = 25) -> List[Dict[str, Any]]:
    """Load HellaSwag commonsense reasoning questions."""
    
    # Sample HellaSwag questions (commonsense reasoning)
    hellaswag_samples = [
        {
            "activity_label": "cooking",
            "context": "A woman is seen standing in a kitchen mixing ingredients in a bowl while speaking to the camera.",
            "endings": [
                "She continues mixing while adding more ingredients and occasionally tasting the mixture.",  # Correct
                "She throws the bowl across the room and starts dancing on the counter.",
                "She suddenly transforms into a butterfly and flies away.",
                "She begins juggling flaming torches while blindfolded."
            ],
            "answer": "A",
            "category": "daily_activities"
        },
        {
            "activity_label": "driving",
            "context": "A man gets into his car, adjusts his mirrors, and puts on his seatbelt.",
            "endings": [
                "He honks the horn repeatedly for no reason.",
                "He starts the engine and begins driving down the road.",  # Correct
                "He exits the car and starts doing jumping jacks.",
                "He transforms the car into a robot and flies to Mars."
            ],
            "answer": "B",
            "category": "transportation"
        },
        {
            "activity_label": "exercising",
            "context": "A person is at the gym doing bench press exercises with proper form.",
            "endings": [
                "They suddenly start reciting poetry in ancient Latin.",
                "They complete their set and move to the next exercise machine.",  # Correct
                "They throw the weights through the ceiling and fly away.",
                "They begin tap dancing on the bench while still holding weights."
            ],
            "answer": "B",
            "category": "fitness"
        },
        {
            "activity_label": "studying",
            "context": "A student sits at a desk with textbooks open, taking notes while reading.",
            "endings": [
                "She turns the pages into paper airplanes and launches them.",
                "She continues reading and highlighting important passages.",  # Correct
                "She eats the entire textbook page by page.",
                "She begins communicating telepathically with the book."
            ],
            "answer": "B",
            "category": "education"
        },
        {
            "activity_label": "gardening",
            "context": "An elderly man is kneeling in his garden, carefully planting flower seeds in prepared soil.",
            "endings": [
                "He waters the newly planted seeds and covers them with a light layer of soil.",  # Correct
                "He begins singing opera to the earthworms.",
                "He digs a hole to China using only a spoon.",
                "He plants himself in the ground hoping to become a tree."
            ],
            "answer": "A",
            "category": "outdoor_activities"
        },
        {
            "activity_label": "cleaning",
            "context": "A person is vacuuming their living room carpet, moving furniture to reach all areas.",
            "endings": [
                "They finish vacuuming and put the furniture back in place.",  # Correct
                "They use the vacuum to suck up their pet cat.",
                "They ride the vacuum cleaner like a surfboard.",
                "They vacuum their own feet until they disappear."
            ],
            "answer": "A",
            "category": "household_chores"
        },
        {
            "activity_label": "shopping",
            "context": "A woman pushes a shopping cart through the grocery store, checking items off her list.",
            "endings": [
                "She pays for her groceries at the checkout counter.",  # Correct
                "She builds a fort out of cereal boxes in aisle 5.",
                "She trades her cart for a wild shopping cart in the parking lot.",
                "She begins juggling all the produce while riding the cart."
            ],
            "answer": "A",
            "category": "retail"
        },
        {
            "activity_label": "reading",
            "context": "A child sits on a comfortable chair with a picture book, turning pages slowly.",
            "endings": [
                "He points at pictures and sounds out words he recognizes.",  # Correct
                "He uses the book as a hat and pretends to be a wizard.",
                "He teaches the book characters how to escape from the pages.",
                "He eats the book thinking it's made of chocolate."
            ],
            "answer": "A",
            "category": "entertainment"
        },
        {
            "activity_label": "cooking",
            "context": "A chef is preparing pasta, boiling water in a large pot on the stove.",
            "endings": [
                "He adds salt to the water and waits for it to reach a rolling boil before adding pasta.",  # Correct
                "He decides to take a bath in the boiling water.",
                "He uses the steam to communicate with aliens.",
                "He throws the pasta at the wall to see if it sticks permanently."
            ],
            "answer": "A",
            "category": "culinary"
        },
        {
            "activity_label": "pet_care",
            "context": "A dog owner fills a bowl with fresh water and sets it down for their thirsty dog.",
            "endings": [
                "The dog runs over and begins drinking the water eagerly.",  # Correct
                "The dog performs a backflip and lands in the water bowl.",
                "The dog begins teaching the water bowl how to fetch.",
                "The dog transforms the water into solid gold."
            ],
            "answer": "A",
            "category": "animal_care"
        },
        {
            "activity_label": "technology",
            "context": "A person sits at their computer, typing an email to send to their colleague.",
            "endings": [
                "They proofread the email and click the send button.",  # Correct
                "They print the email and mail it via carrier pigeon.",
                "They teach the computer to type emails by itself.",
                "They upload their consciousness into the email."
            ],
            "answer": "A",
            "category": "communication"
        },
        {
            "activity_label": "sports",
            "context": "A basketball player dribbles the ball down the court during a game.",
            "endings": [
                "She passes the ball to a teammate who is in a better position to score.",  # Correct
                "She throws the ball into orbit around Earth.",
                "She teaches the ball to play defense against the opposing team.",
                "She transforms into the ball and bounces herself to victory."
            ],
            "answer": "A",
            "category": "athletics"
        },
        {
            "activity_label": "music",
            "context": "A musician sits with their guitar, tuning each string carefully before practice.",
            "endings": [
                "They begin playing scales to warm up their fingers.",  # Correct
                "They use the guitar strings to lasso passing clouds.",
                "They teach the guitar to play itself while they nap.",
                "They string themselves up like a guitar and play beautiful music."
            ],
            "answer": "A",
            "category": "performance"
        },
        {
            "activity_label": "travel",
            "context": "A family arrives at the airport with their luggage, checking in for their flight.",
            "endings": [
                "They proceed through security and head to their departure gate.",  # Correct
                "They decide to build a new airport terminal out of their suitcases.",
                "They teach their luggage to fly independently to the destination.",
                "They transform into airplanes and fly themselves to their vacation."
            ],
            "answer": "A",
            "category": "transportation"
        },
        {
            "activity_label": "work",
            "context": "An office worker sits at their desk, reviewing documents and making notes in margins.",
            "endings": [
                "They organize the documents into folders for filing.",  # Correct
                "They origami the documents into a flock of paper birds.",
                "They teach the documents to review themselves.",
                "They become one with the paperwork and achieve enlightenment."
            ],
            "answer": "A",
            "category": "professional"
        },
        {
            "activity_label": "social",
            "context": "Friends gather around a table to play a board game on a weekend evening.",
            "endings": [
                "They take turns rolling dice and moving their game pieces according to the rules.",  # Correct
                "They eat all the game pieces thinking they're exotic snacks.",
                "They teach the game pieces to play by themselves.",
                "They transform the board into a portal to another dimension."
            ],
            "answer": "A",
            "category": "recreation"
        },
        {
            "activity_label": "health",
            "context": "A person visits their doctor for a routine check-up appointment.",
            "endings": [
                "The doctor takes their vital signs and asks about their current health status.",  # Correct
                "The doctor prescribes them to become a superhero.",
                "They teach the stethoscope to diagnose patients independently.",
                "They merge with the medical equipment to become a cyborg."
            ],
            "answer": "A",
            "category": "medical"
        },
        {
            "activity_label": "maintenance",
            "context": "A homeowner notices a dripping faucet and gets tools to fix the leak.",
            "endings": [
                "They turn off the water supply and replace the worn washer in the faucet.",  # Correct
                "They teach the water drops to fall upward instead of down.",
                "They transform the faucet into a fountain of youth.",
                "They communicate with the leak in ancient plumber language."
            ],
            "answer": "A",
            "category": "home_repair"
        },
        {
            "activity_label": "weather",
            "context": "Dark clouds gather overhead and people notice the first drops of rain beginning to fall.",
            "endings": [
                "People open umbrellas and seek shelter to stay dry.",  # Correct
                "People begin dancing to summon more rain clouds.",
                "People teach the raindrops to fall in perfect geometric patterns.",
                "People transform into clouds and join the storm system."
            ],
            "answer": "A",
            "category": "natural_phenomena"
        },
        {
            "activity_label": "sleep",
            "context": "A tired person gets into bed, fluffs their pillow, and pulls up the covers.",
            "endings": [
                "They close their eyes and gradually drift off to sleep.",  # Correct
                "They begin teaching their pillow advanced mathematics.",
                "They transform their bed into a spacecraft and fly to dreamland.",
                "They communicate telepathically with sheep to help them count."
            ],
            "answer": "A",
            "category": "rest"
        }
    ]
    
    # Extend with more samples if needed by cycling through
    if num_samples > len(hellaswag_samples):
        extended_samples = hellaswag_samples * ((num_samples // len(hellaswag_samples)) + 1)
        return extended_samples[:num_samples]
    
    return hellaswag_samples[:num_samples]


def evaluate_hellaswag_performance(evaluator: IntegratedBIOMINDEvaluator, samples: List[Dict], max_samples: int = None) -> Dict[str, Any]:
    """Evaluate BIOMIND on HellaSwag questions."""
    
    if max_samples:
        samples = samples[:max_samples]
    
    print(f"\n[BRAIN] Starting HellaSwag evaluation on {len(samples)} questions...")
    print("=" * 60)
    
    correct = 0
    total = len(samples)
    results = []
    category_performance = {}
    processing_times = []
    
    for i, sample in enumerate(samples, 1):
        context = sample['context']
        endings = sample['endings']
        correct_answer = sample['answer']
        category = sample['category']
        activity = sample['activity_label']
        
        # Format task for BIOMIND  
        choices = {chr(65 + j): ending for j, ending in enumerate(endings)}
        choices_text = "\n".join([f"{k}: {v}" for k, v in choices.items()])
        task = f"Context: {context}\n\nWhat happens next?\n{choices_text}"
        
        print(f"\n[SEARCH] Question {i}/{total} ({category})")
        print(f"Activity: {activity}")
        print(f"Context: {context}")
        
        start_time = time.time()
        
        # Process with BIOMIND using correct method
        choices_list = list(choices.values())
        result = evaluator.evaluate_with_reflection(task, choices_list, correct_answer, category)
        
        # Extract predicted answer from result
        predicted_answer = result.get('predicted_answer', '')
        processing_time = result.get('processing_time_ms', 0) / 1000  # Convert to seconds
        processing_times.append(processing_time)
        
        # Check if correct
        is_correct = result.get('is_correct', False)
        if is_correct:
            correct += 1
            
        # Track category performance
        if category not in category_performance:
            category_performance[category] = {'correct': 0, 'total': 0}
        category_performance[category]['total'] += 1
        if is_correct:
            category_performance[category]['correct'] += 1
        
        print(f"   Predicted: {predicted_answer} | Correct: {correct_answer} | {'[OK]' if is_correct else '[FAIL]'}")
        print(f"   Time: {processing_time*1000:.1f}ms | Confidence: {result.get('confidence', 0):.3f}")
        
        results.append({
            'context': context,
            'choices': choices,
            'correct_answer': correct_answer,
            'predicted_answer': predicted_answer,
            'is_correct': is_correct,
            'confidence': result.get('confidence', 0),
            'processing_time_ms': processing_time * 1000,
            'category': category,
            'activity_label': activity,
            'specialist': result.get('selected_specialist', 'unknown'),
            'routing_method': result.get('routing_method', 'unknown')
        })
        
        # Progress indicator
        current_accuracy = (correct / i) * 100
        print(f"   Progress: {i}/{total} ({i/total*100:.1f}%) | Running accuracy: {current_accuracy:.1f}%")
    
    # Calculate final metrics
    accuracy = (correct / total) * 100
    avg_processing_time = np.mean(processing_times) * 1000  # Convert to ms
    
    print(f"\n" + "=" * 60)
    print(f"[TARGET] HELLASWAG EVALUATION COMPLETE")
    print(f"=" * 60)
    print(f"[CHART] Overall Accuracy: {correct}/{total} ({accuracy:.1f}%)")
    print(f"?  Average Processing Time: {avg_processing_time:.1f}ms")
    print(f"[BRAIN] Total Questions: {total}")
    
    # Category breakdown
    print(f"\n[DATA] Performance by Category:")
    for category, stats in category_performance.items():
        cat_accuracy = (stats['correct'] / stats['total']) * 100
        print(f"   {category.replace('_', ' ').title()}: {stats['correct']}/{stats['total']} ({cat_accuracy:.1f}%)")
    
    return {
        'total_questions': total,
        'correct_answers': correct,
        'accuracy_percent': accuracy,
        'average_processing_time_ms': avg_processing_time,
        'category_performance': category_performance,
        'detailed_results': results
    }


def main():
    """Run HellaSwag evaluation."""
    
    print("[ROCKET] Initializing BIOMIND for HellaSwag Challenge Evaluation")
    print("=" * 60)
    
    # Initialize BIOMIND evaluator with specialists
    specialists = ['qwen_math_expert', 'qwen_general_reasoner', 'tiny_llama_planner', 'tiny_llama_critic']
    evaluator = IntegratedBIOMINDEvaluator(specialists)
    print("[OK] BIOMIND system initialized")
    
    # Load HellaSwag samples
    num_questions = 20  # Start with smaller set for testing
    samples = load_hellaswag_samples(num_questions)
    print(f"[OK] Loaded {len(samples)} HellaSwag questions")
    
    # Run evaluation
    results = evaluate_hellaswag_performance(evaluator, samples, max_samples=num_questions)
    
    # Save results
    timestamp = time.strftime("%Y%m%d_%H%M%S")
    results_file = f"hellaswag_evaluation_results_{timestamp}.json"
    
    with open(results_file, 'w') as f:
        json.dump(results, f, indent=2, default=str)
    
    print(f"\n? Results saved to: {results_file}")
    print(f"[TARGET] Final HellaSwag Accuracy: {results['accuracy_percent']:.1f}%")
    

if __name__ == "__main__":
    main()