Merge branch 'main' into feature/runnable

Author: hokindeng

data/questions/tests_task/tests_0001/final_frame.png

@@ -6,6 +6,8 @@
 python run.py configs/demo.yaml
 ```
 
+you could create your own config file in configs/your_config.yaml.
+
 ## 📚 Core Concepts
 
 ### Task Pairs: The Evaluation Unit

data/questions/tests_task/tests_0001/first_frame.png

@@ -10,9 +10,14 @@
     "sudoku_task": "Check if the numbers placed in the final frame match the expected solution.",
     "counting_objects_task": "Check if the count shown in the final frame matches the ground_truth_count. Award 1 point if counts match, 0 otherwise.",
     "letter_counting_task": "Check if the count shown in the final frame matches the ground_truth_count for the target letter. Award 1 point if counts match, 0 otherwise.",
-    "subway_pathfinding_task": "Check if the agent icon in the final frame is at the correct destination_station. Award 1 point if destination matches, 0 otherwise."
+    "subway_pathfinding_task": "Check if the agent icon in the final frame is at the correct destination_station. Award 1 point if destination matches, 0 otherwise.",
     "object_subtraction_task": "Verify that the specified object(s) have been correctly removed from the scene, while other objects remain unchanged and the scene remains complete.",
+    "majority_color_task": "Verify that all objects in the final frame are recolored to match the most common color in the first frame, while positions and shapes remain unchanged.",
+    "grid_shift_task": "Verify that all blocks are shifted 1-2 steps in the instructed direction, without changing their color or relative positions.",
     "object_permanence_task": "Verify that the object(s) remain unchanged in position, color, and shape, and the occluder is moved out of the frame.",
     "light_sequence_task": "Verify that the correct lights are on and all other lights are off in the final frame.",
-    "tower_of_hanoi_task": "Check if exactly one disk moved between frames. Verify the move is legal (top disk moved to empty peg or larger disk). Compare final disk positions to expected."
+    "tower_of_hanoi_task": "Check if exactly one disk moved between frames. Verify the move is legal (top disk moved to empty peg or larger disk). Compare final disk positions to expected.",
+    "dice_2d_task": "Verify that the final frame shows the correct opposite face of the dice. On a standard dice, opposite faces sum to 7 (1↔6, 2↔5, 3↔4). Award 1 point if the shown face matches the expected opposite face, 0 otherwise."
+    "symmetry_completion_task": "Verify that the right half of the grid in the final frame is correctly mirrored from the left half, creating a symmetric pattern. Check that all missing cells have been filled correctly to complete the vertical symmetry.",
+    "nonogram_task": "Verify that all cells in the final frame are correctly filled according to the row and column hints. Check that the filled cells match the expected pattern and that all hints are satisfied. Each row and column must have the correct sequence of filled blocks as indicated by the hints."
 }

data/questions/tests_task/tests_0001/prompt.txt

@@ -19,6 +19,19 @@
 
 logger = logging.getLogger(__name__)
 
+TASK_GUIDANCE = {
+    "chess_task": "Check if the final board position matches the expected position after the correct move.",
+    "maze_task": "Verify that the final frame shows a complete path from start to end that matches the expected solution.",
+    "rotation_task": "Check if the final rotation angle and position match the expected result.",
+    "raven_task": "Verify that the pattern completion in the final frame matches the expected pattern.",
+    "sudoku_task": "Check if the numbers placed in the final frame match the expected solution.",
+    "object_subtraction_task": "Verify that the specified object(s) have been correctly removed from the scene, while other objects remain unchanged and the scene remains complete.",
+    "object_permanence_task": "Verify that the object(s) remain unchanged in position, color, and shape, and the occluder is moved out of the frame.",
+    "light_sequence_task": "Verify that the correct lights are on and all other lights are off in the final frame.",
+    "sequence_completion_task": "Verify that the sequence is correctly completed with the next element that follows the pattern. The final frame should show the complete sequence with the correct answer element."
+}
+
+
 class GPT4OEvaluator:
     """Automatic evaluation using GPT-4O vision model."""
 

data/questions/tests_task/tests_0002/final_frame.png

@@ -18,6 +18,18 @@
 
 from .eval_prompt import TASK_PROMPTS
 
+TASK_GUIDANCE = {
+    "object_permanence_task": "Verify that the object(s) remain unchanged in position, color, and shape, and the occluder is moved out of the frame.",
+    "chess_task": "Check if The black king is in checkmate.",
+    "maze_task": "Verify that the final frame at end of the maze is the red flag.",
+    "rotation_task": "Check if the final rotation angle and position match the expected result.",
+    "raven_task": "Verify that the pattern completion in the final frame matches the expected pattern.",
+    "sudoku_task": "Check if the numbers placed in the final frame match the expected solution.",
+    "clock_task": "Check if the time is correct in the final frame.",
+    "object_subtraction_task": "Verify that the specified object(s) have been correctly removed from the scene, while other objects remain unchanged and the scene remains complete.",
+    "light_sequence_task": "Verify that the correct lights are on and all other lights are off in the final frame.",
+    "sequence_completion_task": "Verify that the sequence is correctly completed with the next element that follows the pattern. The final frame should show the complete sequence with the correct answer element."
+}
 logger = logging.getLogger(__name__)