{ "generated": "2026-11-10T06:10+00:00", "kind": "mvp", "answers_scored_floor": 70000, "categories": [ { "id": "reading", "name": "Reading", "short": "Read", "what": "Telling the time, reading meters, charts, labels, codes and signals.", "tests": [ "clock_scene", "meter_scene", "eye_chart", "label_scene", "chart_screen", "test_strips", "coin_sum", "dashcam_light" ] }, { "id": "counting", "name": "Counting", "short": "Count", "what": "How many things there are, in a picture or a video.", "tests": [ "count_scene", "pallet_count", "conveyor_count" ] }, { "id": "space", "name": "Space, 3D and motion", "short": "Space", "what": "Depth, direction, shape, balance and following things as they move.", "tests": [ "depth_scene", "facing_dir", "part_mirror", "stack_scene", "match_render", "parking_bay", "shell_game" ] }, { "id": "changes", "name": "Spotting changes and errors", "short": "Changes", "what": "What changed, what is wrong, what does not match.", "tests": [ "change_scene", "shelf_plan", "render_bug", "invoice_check" ] }, { "id": "diagrams", "name": "Diagrams", "short": "Diagrams", "what": "Following cables and pipes and reading technical drawings.", "tests": [ "cable_scene", "pipe_follow", "missing_dim" ] }, { "id": "fooled", "name": "Not being fooled", "short": "Fooled?", "what": "Absent objects, illusions, fakes, and saying \"I can't tell\" when it is true.", "tests": [ "absent_object", "cant_tell_scene", "illusion", "ai_or_real" ] } ], "tests": { "clock_scene": { "name": "Reading a clock", "what": "Tell the time on an analogue clock in a rendered scene, to the minute.", "category": "reading", "chance": 0.0, "guess": "free answer" }, "meter_scene": { "name": "Reading a meter", "what": "Read the value a meter or dial gauge shows.", "category": "reading", "chance": 0.0, "guess": "free answer" }, "eye_chart": { "name": "Eye chart", "what": "Read a short code printed at smaller and smaller sizes.", "category": "reading", "chance": 0.0, "guess": "free answer" }, "label_scene": { "name": "Reading a label", "what": "Read a code or serial number printed on a label in a scene.", "category": "reading", "chance": 0.0, "guess": "free answer" }, "chart_screen": { "name": "Reading a chart", "what": "Read a value off a chart shown on a screen.", "category": "reading", "chance": 0.0, "guess": "free answer" }, "test_strips": { "name": "Test strips", "what": "Count how many rapid-test cassettes read positive: a valid control line plus a test line, however faint.", "category": "reading", "chance": 0.0, "guess": "free answer" }, "coin_sum": { "name": "Adding up coins", "what": "Add up the value of the coins in the picture.", "category": "reading", "chance": 0.0, "guess": "free answer" }, "dashcam_light": { "name": "Traffic light from a car", "what": "Say what a traffic light shows, seen from a car.", "category": "reading", "chance": 0.3333, "guess": "3 choices" }, "count_scene": { "name": "Counting objects", "what": "Count objects in a cluttered scene.", "category": "counting", "chance": 0.0, "guess": "free answer" }, "pallet_count": { "name": "Counting stacked goods", "what": "Count items stacked on pallets, warehouse style.", "category": "counting", "chance": 0.0, "guess": "free answer" }, "conveyor_count": { "name": "Counting on a conveyor", "what": "Count items passing by in a short video.", "category": "counting", "chance": 0.0, "guess": "free answer" }, "depth_scene": { "name": "Which is nearer", "what": "Judge which object is closer to the camera.", "category": "space", "chance": 0.5, "guess": "2 choices" }, "facing_dir": { "name": "Which way it faces", "what": "Say which way an object or figure is facing.", "category": "space", "chance": 0.15, "guess": "4-8 choices, varies by item" }, "part_mirror": { "name": "Same or mirror image", "what": "Say whether two views of a part are the same or mirror images.", "category": "space", "chance": 0.5, "guess": "2 choices" }, "stack_scene": { "name": "Will it topple", "what": "Say whether a stack of blocks stays up, and which way it falls if not.", "category": "space", "chance": 0.3333, "guess": "3 choices" }, "match_render": { "name": "Match the render", "what": "Pick which of several 3D renders matches a reference picture in camera, layout and materials.", "category": "space", "chance": 0.25, "guess": "4 choices" }, "parking_bay": { "name": "Which bay can I park in?", "what": "Pick the bay you may use, following the parking rules in the question (reserved, wheelchair, EV, signs), or say none.", "category": "space", "chance": 0.1083, "guess": "7-14 choices, varies by item" }, "shell_game": { "name": "Follow the hidden object", "what": "Follow a hidden object as containers move around, across video frames.", "category": "space", "chance": 0.2733, "guess": "3-6 choices, varies by item" }, "change_scene": { "name": "Spot the change", "what": "Say how many things changed between two pictures of the same scene.", "category": "changes", "chance": 0.0, "guess": "free answer" }, "shelf_plan": { "name": "Shelf against plan", "what": "Compare a shelf photo with its planogram and count the slots holding the wrong product or left empty.", "category": "changes", "chance": 0.0, "guess": "free answer" }, "render_bug": { "name": "Rendering glitch", "what": "Spot a rendering error in a 3D scene, as a game artist would.", "category": "changes", "chance": 0.1667, "guess": "6 choices" }, "invoice_check": { "name": "Check the invoice", "what": "Find the line, subtotal, tax or total that doesn't add up on an invoice, or say none.", "category": "changes", "chance": 0.0666, "guess": "8-43 choices, varies by item" }, "cable_scene": { "name": "Follow the cable", "what": "Follow one cable through a tangle to find where it ends.", "category": "diagrams", "chance": 0.0, "guess": "free answer" }, "pipe_follow": { "name": "Follow the pipe", "what": "Follow one pipe through a diagram to read where it ends.", "category": "diagrams", "chance": 0.0, "guess": "free answer" }, "missing_dim": { "name": "Missing dimension", "what": "Work out a missing measurement on a technical drawing.", "category": "diagrams", "chance": 0.0, "guess": "free answer" }, "absent_object": { "name": "Absent objects", "what": "Questions about an object that may not be in the picture.", "category": "fooled", "chance": 0.1933, "guess": "5-6 choices, varies by item" }, "cant_tell_scene": { "name": "Saying \"I can't tell\"", "what": "Answer when the picture shows it, and say so when the picture hides the answer.", "category": "fooled", "chance": 0.2538, "guess": "3-6 choices, varies by item" }, "illusion": { "name": "Optical illusions", "what": "Compare two things in a picture built to fool the eye: which is bigger, or equal.", "category": "fooled", "chance": 0.3333, "guess": "3 choices" }, "ai_or_real": { "name": "Real or AI-made", "what": "Say whether a picture is a real photograph or AI-generated.", "category": "fooled", "chance": 0.5, "guess": "2 choices" } }, "ranked": [ { "model": "GPT-6 Astra", "maker": "OpenAI", "id": "gpt-6-astra", "access": "API", "type": "general", "group": "US labs", "effort": "medium", "overall": 0.8366, "overall_ci": [ 0.8169, 0.8573 ], "categories": { "reading": { "score": 0.8805, "ci": [ 0.8515, 0.9109 ], "partial": false }, "counting": { "score": 0.775, "ci": [ 0.725, 0.8169 ], "partial": false }, "space": { "score": 0.8367, "ci": [ 0.7846, 0.8861 ], "partial": false }, "changes": { "score": 0.8375, "ci": [ 0.7863, 0.8862 ], "partial": false }, "diagrams": { "score": 0.975, "ci": [ 0.95, 0.9917 ], "partial": false }, "fooled": { "score": 0.7146, "ci": [ 0.6495, 0.796 ], "partial": false } }, "tests": { "change_scene": { "score": 0.925, "ci": [ 0.85, 1.0 ], "acc": 0.925, "acc_ci": [ 0.85, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.875, 0.875, 0.875 ] }, "conveyor_count": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "match_render": { "score": 0.8667, "ci": [ 0.7667, 0.9667 ], "acc": 0.9, "acc_ci": [ 0.825, 0.975 ], "chance": 0.25, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.625 ] }, "shell_game": { "score": 0.9656, "ci": [ 0.8968, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "absent_object": { "score": 0.938, "ci": [ 0.845, 1.0 ], "acc": 0.95, "acc_ci": [ 0.875, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.75 ] }, "ai_or_real": { "score": 0.15, "ci": [ 0.0, 0.45 ], "acc": 0.575, "acc_ci": [ 0.425, 0.725 ], "chance": 0.5, "levels": [ 0.75, 0.75, 0.625, 0.5, 0.25 ] }, "cable_scene": { "score": 0.925, "ci": [ 0.85, 0.975 ], "acc": 0.925, "acc_ci": [ 0.85, 0.975 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.625 ] }, "cant_tell_scene": { "score": 0.958, "ci": [ 0.8741, 1.0 ], "acc": 0.9688, "acc_ci": [ 0.9062, 1.0 ], "chance": 0.2552, "levels": [ 1.0, 1.0, 0.875, 1.0 ] }, "chart_screen": { "score": 0.9062, "ci": [ 0.8125, 1.0 ], "acc": 0.9062, "acc_ci": [ 0.8125, 1.0 ], "chance": 0.0, "levels": [ 0.75, 1.0, 1.0, 0.875 ] }, "clock_scene": { "score": 0.975, "ci": [ 0.925, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.875, 1.0 ] }, "coin_sum": { "score": 0.8, "ci": [ 0.7, 0.9 ], "acc": 0.8, "acc_ci": [ 0.7, 0.9 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.625, 0.5, 0.875 ] }, "count_scene": { "score": 0.525, "ci": [ 0.45, 0.6 ], "acc": 0.525, "acc_ci": [ 0.45, 0.6 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.5, 0.0, 0.125 ] }, "dashcam_light": { "score": 0.8875, "ci": [ 0.775, 1.0 ], "acc": 0.925, "acc_ci": [ 0.85, 1.0 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.75, 0.875 ] }, "depth_scene": { "score": 0.6, "ci": [ 0.4, 0.8 ], "acc": 0.8, "acc_ci": [ 0.7, 0.9 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.5, 0.5 ] }, "eye_chart": { "score": 0.975, "ci": [ 0.925, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "facing_dir": { "score": 0.8529, "ci": [ 0.7647, 0.9412 ], "acc": 0.875, "acc_ci": [ 0.8, 0.95 ], "chance": 0.15, "levels": [ 1.0, 1.0, 0.875, 1.0, 0.5 ] }, "illusion": { "score": 0.8125, "ci": [ 0.7, 0.925 ], "acc": 0.875, "acc_ci": [ 0.8, 0.95 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.5 ] }, "invoice_check": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0663, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "label_scene": { "score": 0.775, "ci": [ 0.675, 0.875 ], "acc": 0.775, "acc_ci": [ 0.675, 0.875 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.375, 0.5 ] }, "meter_scene": { "score": 0.975, "ci": [ 0.925, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "missing_dim": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "pallet_count": { "score": 0.8, "ci": [ 0.675, 0.9 ], "acc": 0.8, "acc_ci": [ 0.675, 0.9 ], "chance": 0.0, "levels": [ 0.875, 1.0, 0.875, 0.625, 0.625 ] }, "parking_bay": { "score": 0.972, "ci": [ 0.9159, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "part_mirror": { "score": 0.75, "ci": [ 0.55, 0.95 ], "acc": 0.875, "acc_ci": [ 0.775, 0.975 ], "chance": 0.5, "levels": [ 0.875, 0.875, 0.875, 0.875, 0.875 ] }, "pipe_follow": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "render_bug": { "score": 0.55, "ci": [ 0.37, 0.73 ], "acc": 0.625, "acc_ci": [ 0.475, 0.775 ], "chance": 0.1667, "levels": [ 0.875, 0.75, 0.5, 0.625, 0.375 ] }, "shelf_plan": { "score": 0.875, "ci": [ 0.825, 0.95 ], "acc": 0.875, "acc_ci": [ 0.825, 0.95 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.375 ] }, "stack_scene": { "score": 0.85, "ci": [ 0.7, 0.9625 ], "acc": 0.9, "acc_ci": [ 0.8, 0.975 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.875, 0.875, 0.75 ] }, "test_strips": { "score": 0.75, "ci": [ 0.65, 0.85 ], "acc": 0.75, "acc_ci": [ 0.65, 0.85 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.375, 0.375 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 38.32, "cost_label": "$38.32", "runs_locally": false, "seconds_per_answer": 5.5, "open_weights": false }, { "model": "GPT-6.1 Sol", "maker": "OpenAI", "id": "gpt-6.1-sol", "access": "API", "type": "general", "group": "US labs", "effort": "medium", "overall": 0.8127, "overall_ci": [ 0.7999, 0.8255 ], "categories": { "reading": { "score": 0.8243, "ci": [ 0.8049, 0.8447 ], "partial": false }, "counting": { "score": 0.7627, "ci": [ 0.7333, 0.792 ], "partial": false }, "space": { "score": 0.8022, "ci": [ 0.7684, 0.8334 ], "partial": false }, "changes": { "score": 0.8104, "ci": [ 0.7801, 0.8408 ], "partial": false }, "diagrams": { "score": 0.9573, "ci": [ 0.9387, 0.9733 ], "partial": false }, "fooled": { "score": 0.7192, "ci": [ 0.6691, 0.7647 ], "partial": false } }, "tests": { "change_scene": { "score": 0.8717, "ci": [ 0.8157, 0.9277 ], "acc": 0.8717, "acc_ci": [ 0.8157, 0.9277 ], "chance": 0.0, "levels": [ 1.0, 0.9583, 0.84, 0.84, 0.72 ] }, "conveyor_count": { "score": 0.984, "ci": [ 0.96, 1.0 ], "acc": 0.984, "acc_ci": [ 0.96, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.92 ] }, "match_render": { "score": 0.8382, "ci": [ 0.7627, 0.9036 ], "acc": 0.8787, "acc_ci": [ 0.822, 0.9277 ], "chance": 0.25, "levels": [ 1.0, 0.92, 0.92, 0.8333, 0.72 ] }, "shell_game": { "score": 0.9229, "ci": [ 0.8679, 0.967 ], "acc": 0.944, "acc_ci": [ 0.904, 0.976 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.76 ] }, "absent_object": { "score": 0.894, "ci": [ 0.8177, 0.9619 ], "acc": 0.9145, "acc_ci": [ 0.8529, 0.9692 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 0.96, 0.92, 0.6923 ] }, "ai_or_real": { "score": 0.1867, "ci": [ 0.0267, 0.3333 ], "acc": 0.5933, "acc_ci": [ 0.5133, 0.6667 ], "chance": 0.5, "levels": [ 0.7, 0.6667, 0.6, 0.5, 0.5 ] }, "cable_scene": { "score": 0.912, "ci": [ 0.872, 0.952 ], "acc": 0.912, "acc_ci": [ 0.872, 0.952 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.96, 1.0, 0.6 ] }, "cant_tell_scene": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.2538, "levels": [ 1.0, 1.0, 1.0, 1.0 ] }, "chart_screen": { "score": 0.8, "ci": [ 0.73, 0.87 ], "acc": 0.8, "acc_ci": [ 0.73, 0.87 ], "chance": 0.0, "levels": [ 0.8, 0.96, 0.88, 0.56 ] }, "clock_scene": { "score": 0.984, "ci": [ 0.96, 1.0 ], "acc": 0.984, "acc_ci": [ 0.96, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.92 ] }, "coin_sum": { "score": 0.7427, "ci": [ 0.6703, 0.8073 ], "acc": 0.7427, "acc_ci": [ 0.6703, 0.8073 ], "chance": 0.0, "levels": [ 1.0, 0.8333, 0.8, 0.28, 0.8 ] }, "count_scene": { "score": 0.496, "ci": [ 0.44, 0.56 ], "acc": 0.496, "acc_ci": [ 0.44, 0.56 ], "chance": 0.0, "levels": [ 0.92, 0.88, 0.48, 0.12, 0.08 ] }, "dashcam_light": { "score": 0.796, "ci": [ 0.712, 0.88 ], "acc": 0.864, "acc_ci": [ 0.808, 0.92 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.72, 0.6 ] }, "depth_scene": { "score": 0.8, "ci": [ 0.7, 0.8833 ], "acc": 0.9, "acc_ci": [ 0.85, 0.9417 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.5833, 0.9167 ] }, "eye_chart": { "score": 0.88, "ci": [ 0.84, 0.92 ], "acc": 0.88, "acc_ci": [ 0.84, 0.92 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.4 ] }, "facing_dir": { "score": 0.7271, "ci": [ 0.6518, 0.8024 ], "acc": 0.768, "acc_ci": [ 0.704, 0.832 ], "chance": 0.15, "levels": [ 1.0, 0.8, 0.76, 0.84, 0.44 ] }, "illusion": { "score": 0.796, "ci": [ 0.712, 0.868 ], "acc": 0.864, "acc_ci": [ 0.808, 0.912 ], "chance": 0.3333, "levels": [ 1.0, 0.96, 1.0, 0.8, 0.56 ] }, "invoice_check": { "score": 0.9914, "ci": [ 0.9743, 1.0 ], "acc": 0.992, "acc_ci": [ 0.976, 1.0 ], "chance": 0.0666, "levels": [ 1.0, 1.0, 1.0, 0.96, 1.0 ] }, "label_scene": { "score": 0.784, "ci": [ 0.728, 0.84 ], "acc": 0.784, "acc_ci": [ 0.728, 0.84 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.48, 0.44 ] }, "meter_scene": { "score": 0.912, "ci": [ 0.864, 0.952 ], "acc": 0.912, "acc_ci": [ 0.864, 0.952 ], "chance": 0.0, "levels": [ 1.0, 0.92, 1.0, 1.0, 0.64 ] }, "missing_dim": { "score": 0.976, "ci": [ 0.952, 1.0 ], "acc": 0.976, "acc_ci": [ 0.952, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.88 ] }, "pallet_count": { "score": 0.808, "ci": [ 0.736, 0.872 ], "acc": 0.808, "acc_ci": [ 0.736, 0.872 ], "chance": 0.0, "levels": [ 0.92, 0.92, 0.84, 0.76, 0.6 ] }, "parking_bay": { "score": 0.9731, "ci": [ 0.9462, 1.0 ], "acc": 0.976, "acc_ci": [ 0.952, 1.0 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.88 ] }, "part_mirror": { "score": 0.6107, "ci": [ 0.4793, 0.7553 ], "acc": 0.8053, "acc_ci": [ 0.7397, 0.8777 ], "chance": 0.5, "levels": [ 0.92, 0.88, 0.8, 0.6667, 0.76 ] }, "pipe_follow": { "score": 0.984, "ci": [ 0.96, 1.0 ], "acc": 0.984, "acc_ci": [ 0.96, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.92 ] }, "render_bug": { "score": 0.5304, "ci": [ 0.4256, 0.6364 ], "acc": 0.6087, "acc_ci": [ 0.5213, 0.697 ], "chance": 0.1667, "levels": [ 0.8, 0.7083, 0.5455, 0.6087, 0.381 ] }, "shelf_plan": { "score": 0.848, "ci": [ 0.808, 0.888 ], "acc": 0.848, "acc_ci": [ 0.808, 0.888 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.28 ] }, "stack_scene": { "score": 0.7437, "ci": [ 0.6448, 0.8335 ], "acc": 0.8291, "acc_ci": [ 0.7632, 0.889 ], "chance": 0.3333, "levels": [ 1.0, 0.96, 0.8333, 0.7273, 0.625 ] }, "test_strips": { "score": 0.696, "ci": [ 0.648, 0.752 ], "acc": 0.696, "acc_ci": [ 0.648, 0.752 ], "chance": 0.0, "levels": [ 1.0, 0.96, 1.0, 0.32, 0.2 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 7.94, "cost_label": "$7.94", "runs_locally": false, "seconds_per_answer": 6.1, "open_weights": false }, { "model": "Claude Fable 5.2", "maker": "Anthropic", "id": "claude-fable-5-2", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.8074, "overall_ci": [ 0.7857, 0.8281 ], "categories": { "reading": { "score": 0.8141, "ci": [ 0.7805, 0.8453 ], "partial": false }, "counting": { "score": 0.6417, "ci": [ 0.5833, 0.6917 ], "partial": false }, "space": { "score": 0.8581, "ci": [ 0.8099, 0.901 ], "partial": false }, "changes": { "score": 0.8596, "ci": [ 0.8083, 0.9075 ], "partial": false }, "diagrams": { "score": 0.95, "ci": [ 0.9167, 0.9833 ], "partial": false }, "fooled": { "score": 0.7209, "ci": [ 0.6428, 0.7955 ], "partial": false } }, "tests": { "change_scene": { "score": 0.825, "ci": [ 0.725, 0.925 ], "acc": 0.825, "acc_ci": [ 0.725, 0.925 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.875, 0.625, 0.625 ] }, "conveyor_count": { "score": 0.975, "ci": [ 0.925, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.875, 1.0 ] }, "match_render": { "score": 0.9333, "ci": [ 0.8667, 1.0 ], "acc": 0.95, "acc_ci": [ 0.9, 1.0 ], "chance": 0.25, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.75 ] }, "shell_game": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "absent_object": { "score": 0.938, "ci": [ 0.845, 1.0 ], "acc": 0.95, "acc_ci": [ 0.875, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.75 ] }, "ai_or_real": { "score": 0.4, "ci": [ 0.15, 0.65 ], "acc": 0.7, "acc_ci": [ 0.575, 0.825 ], "chance": 0.5, "levels": [ 0.875, 0.875, 0.875, 0.625, 0.25 ] }, "cable_scene": { "score": 0.9, "ci": [ 0.825, 0.975 ], "acc": 0.9, "acc_ci": [ 0.825, 0.975 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.625 ] }, "cant_tell_scene": { "score": 0.958, "ci": [ 0.8741, 1.0 ], "acc": 0.9688, "acc_ci": [ 0.9062, 1.0 ], "chance": 0.2552, "levels": [ 1.0, 1.0, 0.875, 1.0 ] }, "chart_screen": { "score": 0.75, "ci": [ 0.625, 0.875 ], "acc": 0.75, "acc_ci": [ 0.625, 0.875 ], "chance": 0.0, "levels": [ 0.875, 0.875, 0.5, 0.75 ] }, "clock_scene": { "score": 0.95, "ci": [ 0.9, 1.0 ], "acc": 0.95, "acc_ci": [ 0.9, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.75, 1.0 ] }, "coin_sum": { "score": 0.7, "ci": [ 0.625, 0.775 ], "acc": 0.7, "acc_ci": [ 0.625, 0.775 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.875, 0.0, 0.625 ] }, "count_scene": { "score": 0.525, "ci": [ 0.45, 0.575 ], "acc": 0.525, "acc_ci": [ 0.45, 0.575 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.625, 0.0, 0.0 ] }, "dashcam_light": { "score": 0.9625, "ci": [ 0.8875, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "depth_scene": { "score": 0.8, "ci": [ 0.6, 0.95 ], "acc": 0.9, "acc_ci": [ 0.8, 0.975 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.75, 0.75 ] }, "eye_chart": { "score": 0.525, "ci": [ 0.425, 0.65 ], "acc": 0.525, "acc_ci": [ 0.425, 0.65 ], "chance": 0.0, "levels": [ 1.0, 0.875, 0.25, 0.25, 0.25 ] }, "facing_dir": { "score": 0.8235, "ci": [ 0.7353, 0.9118 ], "acc": 0.85, "acc_ci": [ 0.775, 0.925 ], "chance": 0.15, "levels": [ 1.0, 1.0, 0.875, 1.0, 0.375 ] }, "illusion": { "score": 0.5875, "ci": [ 0.4375, 0.7375 ], "acc": 0.725, "acc_ci": [ 0.625, 0.825 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.875, 0.5, 0.25 ] }, "invoice_check": { "score": 0.9732, "ci": [ 0.9197, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.0659, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "meter_scene": { "score": 0.95, "ci": [ 0.875, 1.0 ], "acc": 0.95, "acc_ci": [ 0.875, 1.0 ], "chance": 0.0, "levels": [ 1.0, 0.875, 1.0, 1.0, 0.875 ] }, "label_scene": { "score": 0.9, "ci": [ 0.825, 0.975 ], "acc": 0.9, "acc_ci": [ 0.825, 0.975 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.625, 0.875 ] }, "missing_dim": { "score": 0.95, "ci": [ 0.9, 1.0 ], "acc": 0.95, "acc_ci": [ 0.9, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.75 ] }, "pallet_count": { "score": 0.425, "ci": [ 0.3, 0.575 ], "acc": 0.425, "acc_ci": [ 0.3, 0.575 ], "chance": 0.0, "levels": [ 0.75, 0.25, 0.25, 0.375, 0.5 ] }, "parking_bay": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "part_mirror": { "score": 0.75, "ci": [ 0.55, 0.95 ], "acc": 0.875, "acc_ci": [ 0.775, 0.975 ], "chance": 0.5, "levels": [ 0.875, 0.75, 1.0, 0.75, 1.0 ] }, "pipe_follow": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "render_bug": { "score": 0.64, "ci": [ 0.46, 0.79 ], "acc": 0.7, "acc_ci": [ 0.55, 0.825 ], "chance": 0.1667, "levels": [ 0.875, 0.875, 0.5, 0.625, 0.625 ] }, "shelf_plan": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "stack_scene": { "score": 0.7, "ci": [ 0.5875, 0.8125 ], "acc": 0.8, "acc_ci": [ 0.725, 0.875 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.75, 0.25 ] }, "test_strips": { "score": 0.775, "ci": [ 0.675, 0.875 ], "acc": 0.775, "acc_ci": [ 0.675, 0.875 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.5, 0.375 ] } }, "unsupported_tests": [], "tested": "2026-11-10", "cost_per_1000": 53.1, "cost_label": "$53.10", "runs_locally": false, "seconds_per_answer": null, "open_weights": null }, { "model": "Gemini 4 Argon", "maker": "Google", "id": "gemini-4-argon", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.7724, "overall_ci": [ 0.7593, 0.785 ], "categories": { "reading": { "score": 0.796, "ci": [ 0.7762, 0.8145 ], "partial": false }, "counting": { "score": 0.5813, "ci": [ 0.5406, 0.6207 ], "partial": false }, "space": { "score": 0.7211, "ci": [ 0.685, 0.7552 ], "partial": false }, "changes": { "score": 0.8882, "ci": [ 0.8602, 0.9166 ], "partial": false }, "diagrams": { "score": 0.904, "ci": [ 0.88, 0.928 ], "partial": false }, "fooled": { "score": 0.7438, "ci": [ 0.7015, 0.7833 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.96, "ci": [ 0.92, 0.992 ], "acc": 0.96, "acc_ci": [ 0.92, 0.992 ], "chance": 0.0, "levels": [ 1.0, 0.92, 1.0, 0.88, 1.0 ] }, "change_scene": { "score": 0.944, "ci": [ 0.904, 0.984 ], "acc": 0.944, "acc_ci": [ 0.904, 0.984 ], "chance": 0.0, "levels": [ 0.96, 1.0, 0.88, 0.96, 0.92 ] }, "match_render": { "score": 0.816, "ci": [ 0.7404, 0.892 ], "acc": 0.862, "acc_ci": [ 0.8053, 0.919 ], "chance": 0.25, "levels": [ 1.0, 1.0, 0.92, 0.75, 0.64 ] }, "conveyor_count": { "score": 0.832, "ci": [ 0.764, 0.898 ], "acc": 0.832, "acc_ci": [ 0.764, 0.898 ], "chance": 0.0, "levels": [ 1.0, 0.9, 0.7, 0.8, 0.76 ] }, "shell_game": { "score": 0.5239, "ci": [ 0.4303, 0.6147 ], "acc": 0.654, "acc_ci": [ 0.586, 0.72 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.55, 0.2, 0.52 ] }, "absent_object": { "score": 0.9619, "ci": [ 0.9046, 1.0 ], "acc": 0.9692, "acc_ci": [ 0.9231, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.8462 ] }, "ai_or_real": { "score": 0.6133, "ci": [ 0.4933, 0.72 ], "acc": 0.8067, "acc_ci": [ 0.7467, 0.86 ], "chance": 0.5, "levels": [ 1.0, 0.8667, 0.9, 0.6333, 0.6333 ] }, "cable_scene": { "score": 0.848, "ci": [ 0.792, 0.896 ], "acc": 0.848, "acc_ci": [ 0.792, 0.896 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.96, 0.84, 0.44 ] }, "cant_tell_scene": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.2538, "levels": [ 1.0, 1.0, 1.0, 1.0 ] }, "chart_screen": { "score": 0.58, "ci": [ 0.5, 0.66 ], "acc": 0.58, "acc_ci": [ 0.5, 0.66 ], "chance": 0.0, "levels": [ 0.8, 0.8, 0.4, 0.32 ] }, "coin_sum": { "score": 0.656, "ci": [ 0.616, 0.704 ], "acc": 0.656, "acc_ci": [ 0.616, 0.704 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.96, 0.2, 0.12 ] }, "count_scene": { "score": 0.368, "ci": [ 0.312, 0.432 ], "acc": 0.368, "acc_ci": [ 0.312, 0.432 ], "chance": 0.0, "levels": [ 1.0, 0.48, 0.28, 0.08, 0.0 ] }, "dashcam_light": { "score": 0.844, "ci": [ 0.772, 0.916 ], "acc": 0.896, "acc_ci": [ 0.848, 0.944 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.96, 0.8, 0.72 ] }, "depth_scene": { "score": 0.9, "ci": [ 0.8167, 0.9667 ], "acc": 0.95, "acc_ci": [ 0.9083, 0.9833 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.9583, 0.7917 ] }, "eye_chart": { "score": 0.816, "ci": [ 0.8, 0.84 ], "acc": 0.816, "acc_ci": [ 0.8, 0.84 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.08 ] }, "facing_dir": { "score": 0.9718, "ci": [ 0.9435, 1.0 ], "acc": 0.976, "acc_ci": [ 0.952, 1.0 ], "chance": 0.15, "levels": [ 1.0, 1.0, 0.92, 1.0, 0.96 ] }, "illusion": { "score": 0.4, "ci": [ 0.304, 0.508 ], "acc": 0.6, "acc_ci": [ 0.536, 0.672 ], "chance": 0.3333, "levels": [ 1.0, 0.96, 0.48, 0.24, 0.32 ] }, "invoice_check": { "score": 0.94, "ci": [ 0.8971, 0.9743 ], "acc": 0.944, "acc_ci": [ 0.904, 0.976 ], "chance": 0.0666, "levels": [ 1.0, 1.0, 1.0, 0.92, 0.8 ] }, "label_scene": { "score": 0.888, "ci": [ 0.84, 0.936 ], "acc": 0.888, "acc_ci": [ 0.84, 0.936 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.68, 0.76 ] }, "meter_scene": { "score": 0.904, "ci": [ 0.864, 0.944 ], "acc": 0.904, "acc_ci": [ 0.864, 0.944 ], "chance": 0.0, "levels": [ 1.0, 0.96, 1.0, 1.0, 0.56 ] }, "missing_dim": { "score": 0.912, "ci": [ 0.872, 0.952 ], "acc": 0.912, "acc_ci": [ 0.872, 0.952 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.56 ] }, "pallet_count": { "score": 0.544, "ci": [ 0.464, 0.616 ], "acc": 0.544, "acc_ci": [ 0.464, 0.616 ], "chance": 0.0, "levels": [ 0.76, 0.76, 0.76, 0.24, 0.2 ] }, "parking_bay": { "score": 0.9821, "ci": [ 0.9551, 1.0 ], "acc": 0.984, "acc_ci": [ 0.96, 1.0 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.92 ] }, "part_mirror": { "score": 0.2413, "ci": [ 0.0793, 0.4033 ], "acc": 0.6207, "acc_ci": [ 0.5397, 0.7017 ], "chance": 0.5, "levels": [ 0.92, 0.64, 0.36, 0.5833, 0.6 ] }, "pipe_follow": { "score": 0.952, "ci": [ 0.92, 0.984 ], "acc": 0.952, "acc_ci": [ 0.92, 0.984 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.8 ] }, "render_bug": { "score": 0.7327, "ci": [ 0.6382, 0.819 ], "acc": 0.7772, "acc_ci": [ 0.6985, 0.8491 ], "chance": 0.1667, "levels": [ 0.92, 0.7917, 0.6818, 0.8261, 0.6667 ] }, "shelf_plan": { "score": 0.936, "ci": [ 0.896, 0.968 ], "acc": 0.936, "acc_ci": [ 0.896, 0.968 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.72 ] }, "stack_scene": { "score": 0.6125, "ci": [ 0.5102, 0.7159 ], "acc": 0.7417, "acc_ci": [ 0.6735, 0.8106 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.75, 0.5, 0.4583 ] }, "test_strips": { "score": 0.72, "ci": [ 0.672, 0.768 ], "acc": 0.72, "acc_ci": [ 0.672, 0.768 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.44, 0.16 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 6.52, "cost_label": "$6.52", "runs_locally": false, "seconds_per_answer": 7.5, "open_weights": false }, { "model": "Claude Opus 5.5", "maker": "Anthropic", "id": "claude-opus-5-5", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.7651, "overall_ci": [ 0.7478, 0.7829 ], "categories": { "reading": { "score": 0.7491, "ci": [ 0.7244, 0.7739 ], "partial": false }, "counting": { "score": 0.6257, "ci": [ 0.5713, 0.6807 ], "partial": false }, "space": { "score": 0.7111, "ci": [ 0.6606, 0.7651 ], "partial": false }, "changes": { "score": 0.8407, "ci": [ 0.7953, 0.884 ], "partial": false }, "diagrams": { "score": 0.9027, "ci": [ 0.867, 0.933 ], "partial": false }, "fooled": { "score": 0.7615, "ci": [ 0.7105, 0.8107 ], "partial": false } }, "tests": { "shell_game": { "score": 0.8789, "ci": [ 0.8128, 0.9339 ], "acc": 0.912, "acc_ci": [ 0.864, 0.952 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 1.0, 0.68, 0.88 ] }, "cant_tell_scene": { "score": 0.9732, "ci": [ 0.933, 1.0 ], "acc": 0.98, "acc_ci": [ 0.95, 1.0 ], "chance": 0.2538, "levels": [ 1.0, 0.96, 0.96, 1.0 ] }, "change_scene": { "score": 0.743, "ci": [ 0.6703, 0.8153 ], "acc": 0.743, "acc_ci": [ 0.6703, 0.8153 ], "chance": 0.0, "levels": [ 0.96, 0.875, 0.6, 0.64, 0.64 ] }, "conveyor_count": { "score": 0.92, "ci": [ 0.87, 0.966 ], "acc": 0.92, "acc_ci": [ 0.87, 0.966 ], "chance": 0.0, "levels": [ 1.0, 0.85, 0.95, 0.92, 0.88 ] }, "match_render": { "score": 0.8169, "ci": [ 0.7409, 0.8827 ], "acc": 0.8627, "acc_ci": [ 0.8057, 0.912 ], "chance": 0.25, "levels": [ 1.0, 0.96, 0.92, 0.8333, 0.6 ] }, "test_strips": { "score": 0.7, "ci": [ 0.625, 0.8 ], "acc": 0.7, "acc_ci": [ 0.625, 0.8 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.25, 0.25 ] }, "ai_or_real": { "score": 0.44, "ci": [ 0.32, 0.5733 ], "acc": 0.72, "acc_ci": [ 0.66, 0.7867 ], "chance": 0.5, "levels": [ 0.9333, 0.8667, 0.8333, 0.4667, 0.5 ] }, "cable_scene": { "score": 0.808, "ci": [ 0.752, 0.856 ], "acc": 0.808, "acc_ci": [ 0.752, 0.856 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.96, 0.68, 0.4 ] }, "chart_screen": { "score": 0.76, "ci": [ 0.68, 0.84 ], "acc": 0.76, "acc_ci": [ 0.68, 0.84 ], "chance": 0.0, "levels": [ 0.96, 0.88, 0.6, 0.6 ] }, "clock_scene": { "score": 0.912, "ci": [ 0.864, 0.96 ], "acc": 0.912, "acc_ci": [ 0.864, 0.96 ], "chance": 0.0, "levels": [ 0.96, 0.8, 0.96, 0.92, 0.92 ] }, "coin_sum": { "score": 0.6717, "ci": [ 0.6237, 0.72 ], "acc": 0.6717, "acc_ci": [ 0.6237, 0.72 ], "chance": 0.0, "levels": [ 0.96, 0.9583, 0.96, 0.04, 0.44 ] }, "count_scene": { "score": 0.432, "ci": [ 0.3758, 0.488 ], "acc": 0.432, "acc_ci": [ 0.3758, 0.488 ], "chance": 0.0, "levels": [ 0.92, 0.76, 0.4, 0.08, 0.0 ] }, "dashcam_light": { "score": 0.784, "ci": [ 0.7, 0.856 ], "acc": 0.856, "acc_ci": [ 0.8, 0.904 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.68, 0.6 ] }, "depth_scene": { "score": 0.7, "ci": [ 0.6, 0.8 ], "acc": 0.85, "acc_ci": [ 0.8, 0.9 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.4583, 0.7917 ] }, "eye_chart": { "score": 0.44, "ci": [ 0.392, 0.496 ], "acc": 0.44, "acc_ci": [ 0.392, 0.496 ], "chance": 0.0, "levels": [ 1.0, 0.84, 0.12, 0.16, 0.08 ] }, "facing_dir": { "score": 0.7941, "ci": [ 0.6471, 0.9118 ], "acc": 0.825, "acc_ci": [ 0.7, 0.925 ], "chance": 0.15, "levels": [ 1.0, 1.0, 0.75, 0.75, 0.625 ] }, "illusion": { "score": 0.7, "ci": [ 0.55, 0.85 ], "acc": 0.8, "acc_ci": [ 0.7, 0.9 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.875, 0.75, 0.375 ] }, "invoice_check": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0663, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "label_scene": { "score": 0.875, "ci": [ 0.775, 0.95 ], "acc": 0.875, "acc_ci": [ 0.775, 0.95 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.625, 0.75 ] }, "meter_scene": { "score": 0.85, "ci": [ 0.775, 0.925 ], "acc": 0.85, "acc_ci": [ 0.775, 0.925 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.375 ] }, "missing_dim": { "score": 0.925, "ci": [ 0.85, 1.0 ], "acc": 0.925, "acc_ci": [ 0.85, 1.0 ], "chance": 0.0, "levels": [ 0.875, 1.0, 1.0, 1.0, 0.75 ] }, "pallet_count": { "score": 0.525, "ci": [ 0.375, 0.675 ], "acc": 0.525, "acc_ci": [ 0.375, 0.675 ], "chance": 0.0, "levels": [ 0.75, 0.5, 0.5, 0.25, 0.625 ] }, "parking_bay": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "part_mirror": { "score": 0.2, "ci": [ 0.0, 0.45 ], "acc": 0.6, "acc_ci": [ 0.475, 0.725 ], "chance": 0.5, "levels": [ 0.75, 0.875, 0.5, 0.25, 0.625 ] }, "pipe_follow": { "score": 0.975, "ci": [ 0.925, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "render_bug": { "score": 0.67, "ci": [ 0.52, 0.82 ], "acc": 0.725, "acc_ci": [ 0.6, 0.85 ], "chance": 0.1667, "levels": [ 0.875, 0.75, 0.5, 1.0, 0.5 ] }, "shelf_plan": { "score": 0.95, "ci": [ 0.875, 1.0 ], "acc": 0.95, "acc_ci": [ 0.875, 1.0 ], "chance": 0.0, "levels": [ 1.0, 0.875, 1.0, 1.0, 0.875 ] }, "stack_scene": { "score": 0.5875, "ci": [ 0.4, 0.775 ], "acc": 0.725, "acc_ci": [ 0.6, 0.85 ], "chance": 0.3333, "levels": [ 1.0, 0.875, 0.5, 0.625, 0.625 ] }, "absent_object": { "score": 0.9329, "ci": [ 0.8657, 0.9809 ], "acc": 0.9458, "acc_ci": [ 0.8917, 0.9846 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.7692 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 26.16, "cost_label": "$26.16", "runs_locally": false, "seconds_per_answer": 3.9, "open_weights": false }, { "model": "Claude Sonnet 5.5", "maker": "Anthropic", "id": "claude-sonnet-5-5", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.7099, "overall_ci": [ 0.6884, 0.7314 ], "categories": { "reading": { "score": 0.6937, "ci": [ 0.6651, 0.7206 ], "partial": false }, "counting": { "score": 0.5293, "ci": [ 0.473, 0.588 ], "partial": false }, "space": { "score": 0.6787, "ci": [ 0.6242, 0.7309 ], "partial": false }, "changes": { "score": 0.7384, "ci": [ 0.6748, 0.8018 ], "partial": false }, "diagrams": { "score": 0.842, "ci": [ 0.798, 0.889 ], "partial": false }, "fooled": { "score": 0.7774, "ci": [ 0.7281, 0.8242 ], "partial": false } }, "tests": { "shell_game": { "score": 0.7, "ci": [ 0.6229, 0.7743 ], "acc": 0.782, "acc_ci": [ 0.726, 0.836 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.95, 0.28, 0.68 ] }, "cant_tell_scene": { "score": 0.9427, "ci": [ 0.8837, 0.9866 ], "acc": 0.9573, "acc_ci": [ 0.9132, 0.99 ], "chance": 0.2538, "levels": [ 1.0, 0.96, 0.96, 0.9091 ] }, "change_scene": { "score": 0.896, "ci": [ 0.84, 0.944 ], "acc": 0.896, "acc_ci": [ 0.84, 0.944 ], "chance": 0.0, "levels": [ 0.96, 1.0, 0.88, 0.84, 0.8 ] }, "conveyor_count": { "score": 0.532, "ci": [ 0.452, 0.612 ], "acc": 0.532, "acc_ci": [ 0.452, 0.612 ], "chance": 0.0, "levels": [ 0.9, 0.7, 0.5, 0.28, 0.28 ] }, "match_render": { "score": 0.7404, "ci": [ 0.6538, 0.8258 ], "acc": 0.8053, "acc_ci": [ 0.7403, 0.8693 ], "chance": 0.25, "levels": [ 1.0, 0.96, 0.84, 0.6667, 0.56 ] }, "test_strips": { "score": 0.725, "ci": [ 0.65, 0.825 ], "acc": 0.725, "acc_ci": [ 0.65, 0.825 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.375, 0.25 ] }, "ai_or_real": { "score": 0.44, "ci": [ 0.3067, 0.5733 ], "acc": 0.72, "acc_ci": [ 0.6533, 0.7867 ], "chance": 0.5, "levels": [ 0.8667, 0.9, 0.8333, 0.5, 0.5 ] }, "cable_scene": { "score": 0.776, "ci": [ 0.728, 0.824 ], "acc": 0.776, "acc_ci": [ 0.728, 0.824 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.64, 0.24 ] }, "chart_screen": { "score": 0.63, "ci": [ 0.55, 0.71 ], "acc": 0.63, "acc_ci": [ 0.55, 0.71 ], "chance": 0.0, "levels": [ 1.0, 0.72, 0.48, 0.32 ] }, "clock_scene": { "score": 0.872, "ci": [ 0.808, 0.928 ], "acc": 0.872, "acc_ci": [ 0.808, 0.928 ], "chance": 0.0, "levels": [ 0.96, 0.76, 0.96, 0.88, 0.8 ] }, "coin_sum": { "score": 0.6073, "ci": [ 0.551, 0.6637 ], "acc": 0.6073, "acc_ci": [ 0.551, 0.6637 ], "chance": 0.0, "levels": [ 0.96, 0.9167, 0.84, 0.04, 0.28 ] }, "count_scene": { "score": 0.456, "ci": [ 0.392, 0.52 ], "acc": 0.456, "acc_ci": [ 0.392, 0.52 ], "chance": 0.0, "levels": [ 0.84, 0.84, 0.44, 0.12, 0.04 ] }, "dashcam_light": { "score": 0.64, "ci": [ 0.544, 0.724 ], "acc": 0.76, "acc_ci": [ 0.696, 0.816 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.92, 0.48, 0.4 ] }, "depth_scene": { "score": 0.6533, "ci": [ 0.5367, 0.7693 ], "acc": 0.8267, "acc_ci": [ 0.7683, 0.8847 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.8, 0.5417, 0.7917 ] }, "eye_chart": { "score": 0.325, "ci": [ 0.25, 0.375 ], "acc": 0.325, "acc_ci": [ 0.25, 0.375 ], "chance": 0.0, "levels": [ 1.0, 0.625, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.7353, "ci": [ 0.6176, 0.8529 ], "acc": 0.775, "acc_ci": [ 0.675, 0.875 ], "chance": 0.15, "levels": [ 1.0, 1.0, 0.75, 0.75, 0.375 ] }, "illusion": { "score": 0.775, "ci": [ 0.6625, 0.8875 ], "acc": 0.85, "acc_ci": [ 0.775, 0.925 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.375 ] }, "invoice_check": { "score": 0.8127, "ci": [ 0.6789, 0.9197 ], "acc": 0.825, "acc_ci": [ 0.7, 0.925 ], "chance": 0.0658, "levels": [ 0.75, 1.0, 0.75, 0.75, 0.875 ] }, "label_scene": { "score": 0.85, "ci": [ 0.75, 0.95 ], "acc": 0.85, "acc_ci": [ 0.75, 0.95 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.5, 0.75 ] }, "meter_scene": { "score": 0.9, "ci": [ 0.825, 0.975 ], "acc": 0.9, "acc_ci": [ 0.825, 0.975 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.625 ] }, "missing_dim": { "score": 0.9, "ci": [ 0.825, 0.975 ], "acc": 0.9, "acc_ci": [ 0.825, 0.975 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.5 ] }, "pallet_count": { "score": 0.6, "ci": [ 0.45, 0.75 ], "acc": 0.6, "acc_ci": [ 0.45, 0.75 ], "chance": 0.0, "levels": [ 0.75, 0.75, 0.625, 0.375, 0.5 ] }, "parking_bay": { "score": 0.972, "ci": [ 0.9159, 1.0 ], "acc": 0.975, "acc_ci": [ 0.925, 1.0 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.875 ] }, "part_mirror": { "score": 0.55, "ci": [ 0.3, 0.8 ], "acc": 0.775, "acc_ci": [ 0.65, 0.9 ], "chance": 0.5, "levels": [ 0.875, 0.875, 0.625, 0.625, 0.875 ] }, "pipe_follow": { "score": 0.85, "ci": [ 0.75, 0.95 ], "acc": 0.85, "acc_ci": [ 0.75, 0.95 ], "chance": 0.0, "levels": [ 0.875, 0.75, 0.875, 1.0, 0.75 ] }, "render_bug": { "score": 0.52, "ci": [ 0.34, 0.7 ], "acc": 0.6, "acc_ci": [ 0.45, 0.75 ], "chance": 0.1667, "levels": [ 0.75, 0.875, 0.5, 0.5, 0.375 ] }, "shelf_plan": { "score": 0.725, "ci": [ 0.6, 0.85 ], "acc": 0.725, "acc_ci": [ 0.6, 0.85 ], "chance": 0.0, "levels": [ 0.875, 1.0, 0.75, 0.5, 0.5 ] }, "stack_scene": { "score": 0.4, "ci": [ 0.25, 0.55 ], "acc": 0.6, "acc_ci": [ 0.5, 0.7 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.25, 0.25, 0.5 ] }, "absent_object": { "score": 0.9519, "ci": [ 0.8947, 1.0 ], "acc": 0.9612, "acc_ci": [ 0.9151, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.8462 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 11.6, "cost_label": "$11.60", "runs_locally": false, "seconds_per_answer": 3.7, "open_weights": false }, { "model": "Kimi K3.1 Pro", "maker": "Moonshot", "id": "kimi-k3.1-pro", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.7062, "overall_ci": [ 0.6659, 0.7505 ], "categories": { "reading": { "score": 0.7208, "ci": [ 0.6417, 0.8021 ], "partial": false }, "counting": { "score": 0.5333, "ci": [ 0.4222, 0.6444 ], "partial": false }, "space": { "score": 0.6238, "ci": [ 0.5305, 0.7121 ], "partial": false }, "changes": { "score": 0.7733, "ci": [ 0.6766, 0.8667 ], "partial": false }, "diagrams": { "score": 0.9111, "ci": [ 0.8222, 0.9778 ], "partial": false }, "fooled": { "score": 0.675, "ci": [ 0.5667, 0.8167 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.8, "chance": 0.0 }, "change_scene": { "score": 0.87, "chance": 0.0 }, "conveyor_count": { "score": 0.93, "chance": 0.0 }, "match_render": { "score": 0.56, "chance": 0.25 }, "shell_game": { "score": 0.72, "chance": 0.2733 }, "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.87, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.2569 }, "chart_screen": { "score": 0.67, "chance": 0.0 }, "coin_sum": { "score": 0.6, "chance": 0.0 }, "count_scene": { "score": 0.47, "chance": 0.0 }, "dashcam_light": { "score": 0.9, "chance": 0.3333 }, "test_strips": { "score": 0.47, "chance": 0.0 }, "depth_scene": { "score": 1.0, "chance": 0.5 }, "stack_scene": { "score": 0.4, "chance": 0.3333 }, "eye_chart": { "score": 0.6, "chance": 0.0 }, "facing_dir": { "score": 0.69, "chance": 0.15 }, "shelf_plan": { "score": 0.87, "chance": 0.0 }, "render_bug": { "score": 0.36, "chance": 0.1667 }, "illusion": { "score": 0.5, "chance": 0.3333 }, "invoice_check": { "score": 1.0, "chance": 0.0664 }, "pipe_follow": { "score": 0.87, "chance": 0.0 }, "label_scene": { "score": 0.87, "chance": 0.0 }, "meter_scene": { "score": 0.87, "chance": 0.0 }, "missing_dim": { "score": 1.0, "chance": 0.0 }, "pallet_count": { "score": 0.2, "chance": 0.0 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "parking_bay": { "score": 1.0, "chance": 0.1083 } }, "unsupported_tests": [], "tested": "2026-11-08", "cost_per_1000": 71.79, "cost_label": "$71.79", "runs_locally": false, "seconds_per_answer": 23.2, "open_weights": true }, { "model": "Qwen3.8 Max", "maker": "Alibaba", "id": "qwen3.8-max-0902", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.7056, "overall_ci": [ 0.6613, 0.7516 ], "categories": { "reading": { "score": 0.6729, "ci": [ 0.5854, 0.7542 ], "partial": false }, "counting": { "score": 0.5555, "ci": [ 0.4, 0.6889 ], "partial": false }, "space": { "score": 0.748, "ci": [ 0.6477, 0.8516 ], "partial": false }, "changes": { "score": 0.7209, "ci": [ 0.6121, 0.8221 ], "partial": false }, "diagrams": { "score": 0.8444, "ci": [ 0.7556, 0.9333 ], "partial": false }, "fooled": { "score": 0.6917, "ci": [ 0.55, 0.825 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.47, "chance": 0.5 }, "cable_scene": { "score": 0.6, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.2569 }, "change_scene": { "score": 0.93, "chance": 0.0 }, "chart_screen": { "score": 0.42, "chance": 0.0 }, "clock_scene": { "score": 0.87, "chance": 0.0 }, "coin_sum": { "score": 0.6, "chance": 0.0 }, "conveyor_count": { "score": 0.6, "chance": 0.0 }, "count_scene": { "score": 0.53, "chance": 0.0 }, "dashcam_light": { "score": 0.7, "chance": 0.3333 }, "depth_scene": { "score": 0.87, "chance": 0.5 }, "eye_chart": { "score": 0.4, "chance": 0.0 }, "facing_dir": { "score": 0.69, "chance": 0.15 }, "illusion": { "score": 0.3, "chance": 0.3333 }, "invoice_check": { "score": 0.86, "chance": 0.0664 }, "label_scene": { "score": 0.87, "chance": 0.0 }, "match_render": { "score": 1.0, "chance": 0.25 }, "meter_scene": { "score": 0.73, "chance": 0.0 }, "missing_dim": { "score": 1.0, "chance": 0.0 }, "pallet_count": { "score": 0.53, "chance": 0.0 }, "parking_bay": { "score": 0.93, "chance": 0.1083 }, "part_mirror": { "score": 0.33, "chance": 0.5 }, "pipe_follow": { "score": 0.93, "chance": 0.0 }, "render_bug": { "score": 0.36, "chance": 0.1667 }, "shelf_plan": { "score": 0.73, "chance": 0.0 }, "shell_game": { "score": 0.72, "chance": 0.2733 }, "stack_scene": { "score": 0.7, "chance": 0.3333 }, "test_strips": { "score": 0.8, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-08", "cost_per_1000": 27.4, "cost_label": "$27.40", "runs_locally": false, "seconds_per_answer": 107.2, "open_weights": false }, { "model": "Claude Fable 5.1", "maker": "Anthropic", "id": "claude-fable-5-1", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.6956, "overall_ci": [ 0.6754, 0.7174 ], "categories": { "reading": { "score": 0.6258, "ci": [ 0.5878, 0.6685 ], "partial": false }, "counting": { "score": 0.6323, "ci": [ 0.5793, 0.6867 ], "partial": false }, "space": { "score": 0.6887, "ci": [ 0.6317, 0.745 ], "partial": false }, "changes": { "score": 0.7543, "ci": [ 0.7036, 0.8063 ], "partial": false }, "diagrams": { "score": 0.7607, "ci": [ 0.711, 0.8133 ], "partial": false }, "fooled": { "score": 0.7117, "ci": [ 0.6544, 0.7653 ], "partial": false } }, "tests": { "shell_game": { "score": 0.9229, "ci": [ 0.8679, 0.967 ], "acc": 0.944, "acc_ci": [ 0.904, 0.976 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 1.0, 0.8, 0.92 ] }, "cant_tell_scene": { "score": 0.9866, "ci": [ 0.9598, 1.0 ], "acc": 0.99, "acc_ci": [ 0.97, 1.0 ], "chance": 0.2538, "levels": [ 1.0, 1.0, 0.96, 1.0 ] }, "change_scene": { "score": 0.5973, "ci": [ 0.517, 0.6788 ], "acc": 0.5973, "acc_ci": [ 0.517, 0.6788 ], "chance": 0.0, "levels": [ 0.88, 0.6667, 0.4, 0.64, 0.4 ] }, "conveyor_count": { "score": 0.872, "ci": [ 0.802, 0.93 ], "acc": 0.872, "acc_ci": [ 0.802, 0.93 ], "chance": 0.0, "levels": [ 0.85, 0.85, 0.9, 0.84, 0.92 ] }, "match_render": { "score": 0.6098, "ci": [ 0.5329, 0.6867 ], "acc": 0.7073, "acc_ci": [ 0.6497, 0.765 ], "chance": 0.25, "levels": [ 0.96, 0.96, 0.92, 0.4167, 0.28 ] }, "ai_or_real": { "score": 0.48, "ci": [ 0.3467, 0.6 ], "acc": 0.74, "acc_ci": [ 0.6733, 0.8 ], "chance": 0.5, "levels": [ 0.8667, 0.9, 0.8667, 0.5333, 0.5333 ] }, "cable_scene": { "score": 0.632, "ci": [ 0.576, 0.688 ], "acc": 0.632, "acc_ci": [ 0.576, 0.688 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.76, 0.28, 0.12 ] }, "chart_screen": { "score": 0.5567, "ci": [ 0.4331, 0.686 ], "acc": 0.5567, "acc_ci": [ 0.4331, 0.686 ], "chance": 0.0, "levels": [ 0.92, 0.6818, 0.375, 0.25 ] }, "clock_scene": { "score": 0.65, "ci": [ 0.5, 0.7756 ], "acc": 0.65, "acc_ci": [ 0.5, 0.7756 ], "chance": 0.0, "levels": [ 0.625, 0.75, 0.875, 0.5, 0.5 ] }, "coin_sum": { "score": 0.525, "ci": [ 0.45, 0.6 ], "acc": 0.525, "acc_ci": [ 0.45, 0.6 ], "chance": 0.0, "levels": [ 1.0, 0.875, 0.75, 0.0, 0.0 ] }, "count_scene": { "score": 0.425, "ci": [ 0.325, 0.525 ], "acc": 0.425, "acc_ci": [ 0.325, 0.525 ], "chance": 0.0, "levels": [ 0.875, 0.75, 0.5, 0.0, 0.0 ] }, "dashcam_light": { "score": 0.625, "ci": [ 0.475, 0.775 ], "acc": 0.75, "acc_ci": [ 0.65, 0.85 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.875, 0.5, 0.375 ] }, "depth_scene": { "score": 0.75, "ci": [ 0.55, 0.9 ], "acc": 0.875, "acc_ci": [ 0.775, 0.95 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.75, 0.625 ] }, "eye_chart": { "score": 0.25, "ci": [ 0.15, 0.35 ], "acc": 0.25, "acc_ci": [ 0.15, 0.35 ], "chance": 0.0, "levels": [ 0.625, 0.5, 0.125, 0.0, 0.0 ] }, "facing_dir": { "score": 0.5882, "ci": [ 0.4412, 0.7353 ], "acc": 0.65, "acc_ci": [ 0.525, 0.775 ], "chance": 0.15, "levels": [ 1.0, 0.875, 0.5, 0.5, 0.375 ] }, "illusion": { "score": 0.4375, "ci": [ 0.25, 0.625 ], "acc": 0.625, "acc_ci": [ 0.5, 0.75 ], "chance": 0.3333, "levels": [ 1.0, 0.875, 0.375, 0.625, 0.25 ] }, "invoice_check": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.0658, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "label_scene": { "score": 0.85, "ci": [ 0.775, 0.925 ], "acc": 0.85, "acc_ci": [ 0.775, 0.925 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.375, 0.875 ] }, "meter_scene": { "score": 0.825, "ci": [ 0.725, 0.925 ], "acc": 0.825, "acc_ci": [ 0.725, 0.925 ], "chance": 0.0, "levels": [ 1.0, 0.875, 1.0, 0.875, 0.375 ] }, "missing_dim": { "score": 0.9, "ci": [ 0.825, 0.975 ], "acc": 0.9, "acc_ci": [ 0.825, 0.975 ], "chance": 0.0, "levels": [ 1.0, 0.875, 1.0, 1.0, 0.625 ] }, "pallet_count": { "score": 0.6, "ci": [ 0.5, 0.7 ], "acc": 0.6, "acc_ci": [ 0.5, 0.7 ], "chance": 0.0, "levels": [ 0.875, 1.0, 0.625, 0.375, 0.125 ] }, "parking_bay": { "score": 1.0, "ci": [ 1.0, 1.0 ], "acc": 1.0, "acc_ci": [ 1.0, 1.0 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 1.0, 1.0 ] }, "part_mirror": { "score": 0.4, "ci": [ 0.15, 0.65 ], "acc": 0.7, "acc_ci": [ 0.575, 0.825 ], "chance": 0.5, "levels": [ 0.875, 0.875, 0.625, 0.625, 0.5 ] }, "pipe_follow": { "score": 0.75, "ci": [ 0.625, 0.875 ], "acc": 0.75, "acc_ci": [ 0.625, 0.875 ], "chance": 0.0, "levels": [ 0.875, 0.875, 1.0, 0.5, 0.5 ] }, "render_bug": { "score": 0.52, "ci": [ 0.34, 0.7 ], "acc": 0.6, "acc_ci": [ 0.45, 0.75 ], "chance": 0.1667, "levels": [ 0.875, 0.5, 0.375, 0.75, 0.5 ] }, "shelf_plan": { "score": 0.9, "ci": [ 0.8, 0.975 ], "acc": 0.9, "acc_ci": [ 0.8, 0.975 ], "chance": 0.0, "levels": [ 0.875, 0.875, 1.0, 1.0, 0.75 ] }, "stack_scene": { "score": 0.55, "ci": [ 0.3625, 0.7375 ], "acc": 0.7, "acc_ci": [ 0.575, 0.825 ], "chance": 0.3333, "levels": [ 1.0, 0.75, 0.75, 0.625, 0.375 ] }, "test_strips": { "score": 0.725, "ci": [ 0.65, 0.825 ], "acc": 0.725, "acc_ci": [ 0.65, 0.825 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.25, 0.375 ] }, "absent_object": { "score": 0.9428, "ci": [ 0.8856, 1.0 ], "acc": 0.9538, "acc_ci": [ 0.9077, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.7692 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 87.95, "cost_label": "$87.95", "runs_locally": false, "seconds_per_answer": 5.5, "open_weights": false }, { "model": "Claude Haiku 5.5", "maker": "Anthropic", "id": "claude-haiku-5-5", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.6351, "overall_ci": [ 0.6207, 0.6498 ], "categories": { "reading": { "score": 0.6249, "ci": [ 0.6018, 0.6484 ], "partial": false }, "counting": { "score": 0.4793, "ci": [ 0.4413, 0.5193 ], "partial": false }, "space": { "score": 0.4753, "ci": [ 0.4397, 0.5175 ], "partial": false }, "changes": { "score": 0.7224, "ci": [ 0.6867, 0.7576 ], "partial": false }, "diagrams": { "score": 0.7867, "ci": [ 0.76, 0.816 ], "partial": false }, "fooled": { "score": 0.7221, "ci": [ 0.6791, 0.7685 ], "partial": false } }, "tests": { "shell_game": { "score": 0.3532, "ci": [ 0.2789, 0.4385 ], "acc": 0.53, "acc_ci": [ 0.476, 0.592 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.25, 0.16, 0.24 ] }, "cant_tell_scene": { "score": 0.958, "ci": [ 0.9044, 1.0 ], "acc": 0.9686, "acc_ci": [ 0.9286, 1.0 ], "chance": 0.2538, "levels": [ 1.0, 0.96, 0.96, 0.9545 ] }, "conveyor_count": { "score": 0.782, "ci": [ 0.726, 0.838 ], "acc": 0.782, "acc_ci": [ 0.726, 0.838 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.95, 0.52, 0.44 ] }, "change_scene": { "score": 0.7583, "ci": [ 0.686, 0.8307 ], "acc": 0.7583, "acc_ci": [ 0.686, 0.8307 ], "chance": 0.0, "levels": [ 0.88, 0.7917, 0.64, 0.92, 0.56 ] }, "match_render": { "score": 0.5351, "ci": [ 0.4284, 0.6329 ], "acc": 0.6513, "acc_ci": [ 0.5713, 0.7247 ], "chance": 0.25, "levels": [ 0.84, 0.84, 0.72, 0.4167, 0.44 ] }, "absent_object": { "score": 0.9519, "ci": [ 0.8947, 1.0 ], "acc": 0.9612, "acc_ci": [ 0.9151, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.8462 ] }, "ai_or_real": { "score": 0.3867, "ci": [ 0.2533, 0.52 ], "acc": 0.6933, "acc_ci": [ 0.6267, 0.76 ], "chance": 0.5, "levels": [ 0.8333, 0.8667, 0.7667, 0.5, 0.5 ] }, "cable_scene": { "score": 0.76, "ci": [ 0.712, 0.808 ], "acc": 0.76, "acc_ci": [ 0.712, 0.808 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.56, 0.24 ] }, "chart_screen": { "score": 0.5, "ci": [ 0.42, 0.58 ], "acc": 0.5, "acc_ci": [ 0.42, 0.58 ], "chance": 0.0, "levels": [ 0.96, 0.52, 0.36, 0.16 ] }, "clock_scene": { "score": 0.808, "ci": [ 0.744, 0.872 ], "acc": 0.808, "acc_ci": [ 0.744, 0.872 ], "chance": 0.0, "levels": [ 0.96, 0.68, 0.88, 0.64, 0.88 ] }, "coin_sum": { "score": 0.4033, "ci": [ 0.3467, 0.4593 ], "acc": 0.4033, "acc_ci": [ 0.3467, 0.4593 ], "chance": 0.0, "levels": [ 1.0, 0.4167, 0.56, 0.0, 0.04 ] }, "count_scene": { "score": 0.288, "ci": [ 0.224, 0.36 ], "acc": 0.288, "acc_ci": [ 0.224, 0.36 ], "chance": 0.0, "levels": [ 0.6, 0.56, 0.2, 0.08, 0.0 ] }, "dashcam_light": { "score": 0.736, "ci": [ 0.652, 0.82 ], "acc": 0.824, "acc_ci": [ 0.768, 0.88 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.96, 0.6, 0.56 ] }, "depth_scene": { "score": 0.4413, "ci": [ 0.31, 0.5727 ], "acc": 0.7207, "acc_ci": [ 0.655, 0.7863 ], "chance": 0.5, "levels": [ 0.96, 0.84, 0.72, 0.3333, 0.75 ] }, "illusion": { "score": 0.592, "ci": [ 0.496, 0.688 ], "acc": 0.728, "acc_ci": [ 0.664, 0.792 ], "chance": 0.3333, "levels": [ 0.96, 0.96, 0.88, 0.52, 0.32 ] }, "label_scene": { "score": 0.784, "ci": [ 0.72, 0.848 ], "acc": 0.784, "acc_ci": [ 0.72, 0.848 ], "chance": 0.0, "levels": [ 0.96, 1.0, 0.84, 0.64, 0.48 ] }, "meter_scene": { "score": 0.76, "ci": [ 0.696, 0.824 ], "acc": 0.76, "acc_ci": [ 0.696, 0.824 ], "chance": 0.0, "levels": [ 0.96, 0.68, 0.84, 0.88, 0.44 ] }, "missing_dim": { "score": 0.76, "ci": [ 0.728, 0.792 ], "acc": 0.76, "acc_ci": [ 0.728, 0.792 ], "chance": 0.0, "levels": [ 0.96, 0.92, 0.96, 0.96, 0.0 ] }, "part_mirror": { "score": 0.0947, "ci": [ 0.0, 0.2567 ], "acc": 0.5473, "acc_ci": [ 0.466, 0.6283 ], "chance": 0.5, "levels": [ 0.84, 0.6, 0.44, 0.4167, 0.44 ] }, "shelf_plan": { "score": 0.84, "ci": [ 0.784, 0.888 ], "acc": 0.84, "acc_ci": [ 0.784, 0.888 ], "chance": 0.0, "levels": [ 0.96, 1.0, 1.0, 0.8, 0.44 ] }, "stack_scene": { "score": 0.3506, "ci": [ 0.2362, 0.4655 ], "acc": 0.5671, "acc_ci": [ 0.4908, 0.6437 ], "chance": 0.3333, "levels": [ 0.88, 0.8, 0.4167, 0.3636, 0.375 ] }, "test_strips": { "score": 0.696, "ci": [ 0.6558, 0.744 ], "acc": 0.696, "acc_ci": [ 0.6558, 0.744 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.28, 0.2 ] }, "render_bug": { "score": 0.3171, "ci": [ 0.2044, 0.4219 ], "acc": 0.4309, "acc_ci": [ 0.337, 0.5182 ], "chance": 0.1667, "levels": [ 0.56, 0.5, 0.3182, 0.3478, 0.4286 ] }, "pallet_count": { "score": 0.368, "ci": [ 0.288, 0.448 ], "acc": 0.368, "acc_ci": [ 0.288, 0.448 ], "chance": 0.0, "levels": [ 0.32, 0.28, 0.64, 0.24, 0.36 ] }, "invoice_check": { "score": 0.9743, "ci": [ 0.9486, 1.0 ], "acc": 0.976, "acc_ci": [ 0.952, 1.0 ], "chance": 0.0666, "levels": [ 1.0, 1.0, 1.0, 0.92, 0.96 ] }, "pipe_follow": { "score": 0.84, "ci": [ 0.776, 0.896 ], "acc": 0.84, "acc_ci": [ 0.776, 0.896 ], "chance": 0.0, "levels": [ 1.0, 0.96, 0.84, 0.8, 0.6 ] }, "facing_dir": { "score": 0.6329, "ci": [ 0.5482, 0.7176 ], "acc": 0.688, "acc_ci": [ 0.616, 0.76 ], "chance": 0.15, "levels": [ 1.0, 0.72, 0.72, 0.68, 0.32 ] }, "parking_bay": { "score": 0.9193, "ci": [ 0.8654, 0.9641 ], "acc": 0.928, "acc_ci": [ 0.88, 0.968 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 0.96, 0.88, 0.8 ] }, "eye_chart": { "score": 0.312, "ci": [ 0.264, 0.36 ], "acc": 0.312, "acc_ci": [ 0.264, 0.36 ], "chance": 0.0, "levels": [ 1.0, 0.36, 0.16, 0.04, 0.0 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 1.01, "cost_label": "$1.01", "runs_locally": false, "seconds_per_answer": 4.2, "open_weights": false }, { "model": "Gemini 3.8 Flash", "maker": "Google", "id": "gemini-3.8-flash-medium", "access": "API", "type": "general", "group": "US labs", "effort": "medium", "overall": 0.6192, "overall_ci": [ 0.6021, 0.6373 ], "categories": { "reading": { "score": 0.5842, "ci": [ 0.5578, 0.609 ], "partial": false }, "counting": { "score": 0.4185, "ci": [ 0.3641, 0.4717 ], "partial": false }, "space": { "score": 0.6568, "ci": [ 0.6072, 0.7073 ], "partial": false }, "changes": { "score": 0.8463, "ci": [ 0.7995, 0.8929 ], "partial": false }, "diagrams": { "score": 0.6787, "ci": [ 0.6467, 0.71 ], "partial": false }, "fooled": { "score": 0.5307, "ci": [ 0.4936, 0.5799 ], "partial": false } }, "tests": { "shell_game": { "score": 0.5569, "ci": [ 0.4716, 0.6423 ], "acc": 0.678, "acc_ci": [ 0.616, 0.74 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.75, 0.2, 0.44 ] }, "cant_tell_scene": { "score": 0.9708, "ci": [ 0.9264, 1.0 ], "acc": 0.9782, "acc_ci": [ 0.9451, 1.0 ], "chance": 0.2538, "levels": [ 0.9583, 1.0, 1.0, 0.9545 ] }, "conveyor_count": { "score": 0.414, "ci": [ 0.334, 0.498 ], "acc": 0.414, "acc_ci": [ 0.334, 0.498 ], "chance": 0.0, "levels": [ 0.75, 0.5, 0.1, 0.32, 0.4 ] }, "ai_or_real": { "score": 0.2, "ci": [ 0.04, 0.3467 ], "acc": 0.6, "acc_ci": [ 0.52, 0.6733 ], "chance": 0.5, "levels": [ 0.6667, 0.6667, 0.6667, 0.5, 0.5 ] }, "coin_sum": { "score": 0.5343, "ci": [ 0.4783, 0.583 ], "acc": 0.5343, "acc_ci": [ 0.4783, 0.583 ], "chance": 0.0, "levels": [ 0.92, 0.7917, 0.88, 0.0, 0.08 ] }, "change_scene": { "score": 0.7987, "ci": [ 0.7257, 0.8637 ], "acc": 0.7987, "acc_ci": [ 0.7257, 0.8637 ], "chance": 0.0, "levels": [ 0.84, 0.8333, 0.76, 0.84, 0.72 ] }, "match_render": { "score": 0.7191, "ci": [ 0.6338, 0.7956 ], "acc": 0.7893, "acc_ci": [ 0.7253, 0.8467 ], "chance": 0.25, "levels": [ 1.0, 0.96, 0.92, 0.6667, 0.4 ] }, "absent_object": { "score": 0.9519, "ci": [ 0.8947, 1.0 ], "acc": 0.9612, "acc_ci": [ 0.9151, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.8462 ] }, "cable_scene": { "score": 0.576, "ci": [ 0.52, 0.632 ], "acc": 0.576, "acc_ci": [ 0.52, 0.632 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.56, 0.24, 0.08 ] }, "chart_screen": { "score": 0.48, "ci": [ 0.39, 0.5602 ], "acc": 0.48, "acc_ci": [ 0.39, 0.5602 ], "chance": 0.0, "levels": [ 0.8, 0.56, 0.36, 0.2 ] }, "clock_scene": { "score": 0.72, "ci": [ 0.656, 0.784 ], "acc": 0.72, "acc_ci": [ 0.656, 0.784 ], "chance": 0.0, "levels": [ 0.92, 0.6, 0.96, 0.32, 0.8 ] }, "count_scene": { "score": 0.304, "ci": [ 0.24, 0.36 ], "acc": 0.304, "acc_ci": [ 0.24, 0.36 ], "chance": 0.0, "levels": [ 0.72, 0.6, 0.16, 0.0, 0.04 ] }, "dashcam_light": { "score": 0.748, "ci": [ 0.664, 0.832 ], "acc": 0.832, "acc_ci": [ 0.776, 0.888 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.96, 0.6, 0.6 ] }, "part_mirror": { "score": 0.2631, "ci": [ 0.0084, 0.5019 ], "acc": 0.6315, "acc_ci": [ 0.5042, 0.7509 ], "chance": 0.5, "levels": [ 0.76, 0.7727, 0.375, 0.625, 0.625 ] }, "depth_scene": { "score": 0.75, "ci": [ 0.65, 0.85 ], "acc": 0.875, "acc_ci": [ 0.825, 0.925 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.5417, 0.8333 ] }, "eye_chart": { "score": 0.48, "ci": [ 0.408, 0.552 ], "acc": 0.48, "acc_ci": [ 0.408, 0.552 ], "chance": 0.0, "levels": [ 0.88, 0.64, 0.4, 0.4, 0.08 ] }, "facing_dir": { "score": 0.7553, "ci": [ 0.68, 0.8306 ], "acc": 0.792, "acc_ci": [ 0.728, 0.856 ], "chance": 0.15, "levels": [ 1.0, 0.84, 0.76, 0.88, 0.48 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.112 ], "acc": 0.328, "acc_ci": [ 0.248, 0.408 ], "chance": 0.3333, "levels": [ 0.48, 0.4, 0.28, 0.2, 0.28 ] }, "invoice_check": { "score": 0.9914, "ci": [ 0.9743, 1.0 ], "acc": 0.992, "acc_ci": [ 0.976, 1.0 ], "chance": 0.0666, "levels": [ 1.0, 1.0, 1.0, 0.96, 1.0 ] }, "label_scene": { "score": 0.824, "ci": [ 0.768, 0.88 ], "acc": 0.824, "acc_ci": [ 0.768, 0.88 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.64, 0.48 ] }, "meter_scene": { "score": 0.512, "ci": [ 0.44, 0.584 ], "acc": 0.512, "acc_ci": [ 0.44, 0.584 ], "chance": 0.0, "levels": [ 1.0, 0.36, 0.56, 0.44, 0.2 ] }, "missing_dim": { "score": 0.808, "ci": [ 0.8, 0.824 ], "acc": 0.808, "acc_ci": [ 0.8, 0.824 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.04 ] }, "pallet_count": { "score": 0.5376, "ci": [ 0.4153, 0.6596 ], "acc": 0.5376, "acc_ci": [ 0.4153, 0.6596 ], "chance": 0.0, "levels": [ 0.68, 0.48, 0.7778, 0.375, 0.375 ] }, "parking_bay": { "score": 0.9282, "ci": [ 0.8744, 0.9731 ], "acc": 0.936, "acc_ci": [ 0.888, 0.976 ], "chance": 0.1083, "levels": [ 0.92, 1.0, 1.0, 0.92, 0.84 ] }, "pipe_follow": { "score": 0.652, "ci": [ 0.5739, 0.722 ], "acc": 0.652, "acc_ci": [ 0.5739, 0.722 ], "chance": 0.0, "levels": [ 0.96, 1.0, 0.7, 0.6, 0.0 ] }, "render_bug": { "score": 0.67, "ci": [ 0.49, 0.82 ], "acc": 0.725, "acc_ci": [ 0.575, 0.85 ], "chance": 0.1667, "levels": [ 0.75, 0.75, 0.75, 0.875, 0.5 ] }, "shelf_plan": { "score": 0.925, "ci": [ 0.85, 1.0 ], "acc": 0.925, "acc_ci": [ 0.85, 1.0 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.625 ] }, "stack_scene": { "score": 0.625, "ci": [ 0.4375, 0.8125 ], "acc": 0.75, "acc_ci": [ 0.625, 0.875 ], "chance": 0.3333, "levels": [ 1.0, 0.875, 0.75, 0.625, 0.5 ] }, "test_strips": { "score": 0.375, "ci": [ 0.3, 0.45 ], "acc": 0.375, "acc_ci": [ 0.3, 0.45 ], "chance": 0.0, "levels": [ 1.0, 0.75, 0.0, 0.125, 0.0 ] } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 12.8, "cost_label": "$12.80", "runs_locally": false, "seconds_per_answer": 11.5, "open_weights": false }, { "model": "GPT-6 Luna", "maker": "OpenAI", "id": "gpt-6-luna", "access": "API", "type": "general", "group": "US labs", "effort": "medium", "overall": 0.6069, "overall_ci": [ 0.5927, 0.6218 ], "categories": { "reading": { "score": 0.5924, "ci": [ 0.569, 0.6165 ], "partial": false }, "counting": { "score": 0.3453, "ci": [ 0.306, 0.3827 ], "partial": false }, "space": { "score": 0.5402, "ci": [ 0.5036, 0.5791 ], "partial": false }, "changes": { "score": 0.7263, "ci": [ 0.6924, 0.7601 ], "partial": false }, "diagrams": { "score": 0.712, "ci": [ 0.6827, 0.7413 ], "partial": false }, "fooled": { "score": 0.7253, "ci": [ 0.6779, 0.7723 ], "partial": false } }, "tests": { "change_scene": { "score": 0.7677, "ci": [ 0.7033, 0.8393 ], "acc": 0.7677, "acc_ci": [ 0.7033, 0.8393 ], "chance": 0.0, "levels": [ 0.92, 0.9583, 0.84, 0.64, 0.48 ] }, "conveyor_count": { "score": 0.332, "ci": [ 0.266, 0.402 ], "acc": 0.332, "acc_ci": [ 0.266, 0.402 ], "chance": 0.0, "levels": [ 0.85, 0.5, 0.15, 0.04, 0.12 ] }, "match_render": { "score": 0.6542, "ci": [ 0.5564, 0.7507 ], "acc": 0.7407, "acc_ci": [ 0.6673, 0.813 ], "chance": 0.25, "levels": [ 0.92, 0.8, 0.88, 0.5833, 0.52 ] }, "shell_game": { "score": 0.4743, "ci": [ 0.411, 0.5459 ], "acc": 0.618, "acc_ci": [ 0.572, 0.67 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.85, 0.12, 0.12 ] }, "absent_object": { "score": 0.9031, "ci": [ 0.8352, 0.9702 ], "acc": 0.9218, "acc_ci": [ 0.8671, 0.976 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 0.92, 0.92, 0.7692 ] }, "ai_or_real": { "score": 0.24, "ci": [ 0.0933, 0.4 ], "acc": 0.62, "acc_ci": [ 0.5467, 0.7 ], "chance": 0.5, "levels": [ 0.7667, 0.6667, 0.6333, 0.5333, 0.5 ] }, "cable_scene": { "score": 0.688, "ci": [ 0.64, 0.736 ], "acc": 0.688, "acc_ci": [ 0.64, 0.736 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.92, 0.4, 0.12 ] }, "cant_tell_scene": { "score": 0.986, "ci": [ 0.9581, 1.0 ], "acc": 0.9896, "acc_ci": [ 0.9688, 1.0 ], "chance": 0.2538, "levels": [ 0.9583, 1.0, 1.0, 1.0 ] }, "chart_screen": { "score": 0.46, "ci": [ 0.38, 0.55 ], "acc": 0.46, "acc_ci": [ 0.38, 0.55 ], "chance": 0.0, "levels": [ 0.8, 0.52, 0.36, 0.16 ] }, "clock_scene": { "score": 0.752, "ci": [ 0.68, 0.8162 ], "acc": 0.752, "acc_ci": [ 0.68, 0.8162 ], "chance": 0.0, "levels": [ 1.0, 0.52, 0.92, 0.6, 0.72 ] }, "coin_sum": { "score": 0.443, "ci": [ 0.3787, 0.5077 ], "acc": 0.443, "acc_ci": [ 0.3787, 0.5077 ], "chance": 0.0, "levels": [ 1.0, 0.375, 0.6, 0.04, 0.2 ] }, "count_scene": { "score": 0.376, "ci": [ 0.312, 0.44 ], "acc": 0.376, "acc_ci": [ 0.312, 0.44 ], "chance": 0.0, "levels": [ 0.84, 0.64, 0.24, 0.12, 0.04 ] }, "dashcam_light": { "score": 0.7, "ci": [ 0.628, 0.772 ], "acc": 0.8, "acc_ci": [ 0.752, 0.848 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.68, 0.32 ] }, "depth_scene": { "score": 0.6507, "ci": [ 0.5507, 0.7517 ], "acc": 0.8253, "acc_ci": [ 0.7753, 0.8759 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.96, 0.375, 0.7917 ] }, "eye_chart": { "score": 0.4, "ci": [ 0.336, 0.464 ], "acc": 0.4, "acc_ci": [ 0.336, 0.464 ], "chance": 0.0, "levels": [ 1.0, 0.48, 0.16, 0.36, 0.0 ] }, "facing_dir": { "score": 0.4901, "ci": [ 0.396, 0.5843 ], "acc": 0.5666, "acc_ci": [ 0.4866, 0.6466 ], "chance": 0.15, "levels": [ 0.913, 0.48, 0.44, 0.64, 0.36 ] }, "illusion": { "score": 0.772, "ci": [ 0.7, 0.844 ], "acc": 0.848, "acc_ci": [ 0.8, 0.896 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.96, 0.76, 0.52 ] }, "invoice_check": { "score": 0.9143, "ci": [ 0.8629, 0.9574 ], "acc": 0.92, "acc_ci": [ 0.872, 0.9602 ], "chance": 0.0666, "levels": [ 1.0, 1.0, 1.0, 0.76, 0.84 ] }, "label_scene": { "score": 0.656, "ci": [ 0.608, 0.712 ], "acc": 0.656, "acc_ci": [ 0.608, 0.712 ], "chance": 0.0, "levels": [ 1.0, 0.88, 1.0, 0.24, 0.16 ] }, "meter_scene": { "score": 0.704, "ci": [ 0.632, 0.776 ], "acc": 0.704, "acc_ci": [ 0.632, 0.776 ], "chance": 0.0, "levels": [ 0.96, 0.64, 0.92, 0.68, 0.32 ] }, "missing_dim": { "score": 0.864, "ci": [ 0.824, 0.904 ], "acc": 0.864, "acc_ci": [ 0.824, 0.904 ], "chance": 0.0, "levels": [ 1.0, 0.96, 1.0, 1.0, 0.36 ] }, "pallet_count": { "score": 0.328, "ci": [ 0.248, 0.4 ], "acc": 0.328, "acc_ci": [ 0.248, 0.4 ], "chance": 0.0, "levels": [ 0.48, 0.52, 0.36, 0.16, 0.12 ] }, "parking_bay": { "score": 0.9193, "ci": [ 0.8744, 0.9641 ], "acc": 0.928, "acc_ci": [ 0.888, 0.968 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.68 ] }, "part_mirror": { "score": 0.08, "ci": [ 0.0, 0.2573 ], "acc": 0.54, "acc_ci": [ 0.4517, 0.6287 ], "chance": 0.5, "levels": [ 0.84, 0.44, 0.4, 0.5, 0.52 ] }, "pipe_follow": { "score": 0.584, "ci": [ 0.528, 0.64 ], "acc": 0.584, "acc_ci": [ 0.528, 0.64 ], "chance": 0.0, "levels": [ 0.96, 0.72, 0.92, 0.32, 0.0 ] }, "render_bug": { "score": 0.4392, "ci": [ 0.3359, 0.5426 ], "acc": 0.5327, "acc_ci": [ 0.4466, 0.6188 ], "chance": 0.1667, "levels": [ 0.76, 0.5833, 0.4091, 0.4348, 0.4762 ] }, "shelf_plan": { "score": 0.784, "ci": [ 0.736, 0.832 ], "acc": 0.784, "acc_ci": [ 0.736, 0.832 ], "chance": 0.0, "levels": [ 1.0, 0.96, 1.0, 0.72, 0.24 ] }, "stack_scene": { "score": 0.513, "ci": [ 0.4085, 0.6164 ], "acc": 0.6753, "acc_ci": [ 0.6056, 0.7443 ], "chance": 0.3333, "levels": [ 1.0, 0.96, 0.4167, 0.5, 0.5 ] }, "test_strips": { "score": 0.624, "ci": [ 0.568, 0.68 ], "acc": 0.624, "acc_ci": [ 0.568, 0.68 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.68, 0.24, 0.2 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 0.74, "cost_label": "$0.74", "runs_locally": false, "seconds_per_answer": 9.2, "open_weights": false }, { "model": "Qwen3.8-27B", "maker": "Alibaba", "id": "qwen3.8-27b", "access": "API", "type": "general", "group": "Open models via API", "effort": "default", "overall": 0.5896, "overall_ci": [ 0.5178, 0.6802 ], "categories": { "reading": { "score": 0.525, "ci": [ 0.3688, 0.675 ], "partial": false }, "counting": { "score": 0.4667, "ci": [ 0.2667, 0.6667 ], "partial": false }, "space": { "score": 0.5158, "ci": [ 0.3869, 0.7134 ], "partial": false }, "changes": { "score": 0.78, "ci": [ 0.61, 0.94 ], "partial": false }, "diagrams": { "score": 0.6, "ci": [ 0.3333, 0.8 ], "partial": false }, "fooled": { "score": 0.65, "ci": [ 0.5, 0.925 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.25 }, "change_scene": { "score": 0.8, "chance": 0.0 }, "chart_screen": { "score": 0.5, "chance": 0.0 }, "clock_scene": { "score": 0.6, "chance": 0.0 }, "coin_sum": { "score": 0.4, "chance": 0.0 }, "conveyor_count": { "score": 0.6, "chance": 0.0 }, "count_scene": { "score": 0.8, "chance": 0.0 }, "dashcam_light": { "score": 0.7, "chance": 0.3333 }, "depth_scene": { "score": 1.0, "chance": 0.5 }, "eye_chart": { "score": 0.2, "chance": 0.0 }, "facing_dir": { "score": 0.29, "chance": 0.15 }, "illusion": { "score": 0.4, "chance": 0.3333 }, "invoice_check": { "score": 1.0, "chance": 0.0635 }, "label_scene": { "score": 0.6, "chance": 0.0 }, "match_render": { "score": 0.47, "chance": 0.25 }, "meter_scene": { "score": 0.4, "chance": 0.0 }, "missing_dim": { "score": 0.8, "chance": 0.0 }, "pallet_count": { "score": 0.0, "chance": 0.0 }, "parking_bay": { "score": 1.0, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.6, "chance": 0.0 }, "render_bug": { "score": 0.52, "chance": 0.1667 }, "shelf_plan": { "score": 0.8, "chance": 0.0 }, "shell_game": { "score": 0.45, "chance": 0.2733 }, "stack_scene": { "score": 0.4, "chance": 0.3333 }, "test_strips": { "score": 0.8, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 7.75, "cost_label": "$7.75", "runs_locally": false, "seconds_per_answer": 27.5, "open_weights": true }, { "model": "Qwen3.8 Flash", "maker": "Alibaba", "id": "qwen3.8-flash", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.5641, "overall_ci": [ 0.5505, 0.5784 ], "categories": { "reading": { "score": 0.5777, "ci": [ 0.5532, 0.6013 ], "partial": false }, "counting": { "score": 0.4067, "ci": [ 0.3687, 0.444 ], "partial": false }, "space": { "score": 0.5521, "ci": [ 0.5246, 0.5799 ], "partial": false }, "changes": { "score": 0.6386, "ci": [ 0.6041, 0.6727 ], "partial": false }, "diagrams": { "score": 0.6587, "ci": [ 0.6293, 0.688 ], "partial": false }, "fooled": { "score": 0.5508, "ci": [ 0.5087, 0.6012 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.648, "ci": [ 0.576, 0.728 ], "acc": 0.648, "acc_ci": [ 0.576, 0.728 ], "chance": 0.0, "levels": [ 0.92, 0.52, 0.84, 0.44, 0.52 ] }, "change_scene": { "score": 0.7357, "ci": [ 0.6713, 0.8 ], "acc": 0.7357, "acc_ci": [ 0.6713, 0.8 ], "chance": 0.0, "levels": [ 1.0, 0.9583, 0.56, 0.76, 0.4 ] }, "conveyor_count": { "score": 0.612, "ci": [ 0.548, 0.672 ], "acc": 0.612, "acc_ci": [ 0.548, 0.672 ], "chance": 0.0, "levels": [ 1.0, 0.85, 0.85, 0.2, 0.16 ] }, "match_render": { "score": 0.5458, "ci": [ 0.46, 0.632 ], "acc": 0.6593, "acc_ci": [ 0.595, 0.724 ], "chance": 0.25, "levels": [ 0.96, 0.92, 0.84, 0.4167, 0.16 ] }, "shell_game": { "score": 0.5321, "ci": [ 0.4632, 0.6037 ], "acc": 0.66, "acc_ci": [ 0.6099, 0.712 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.9, 0.08, 0.32 ] }, "absent_object": { "score": 0.9222, "ci": [ 0.8543, 0.9802 ], "acc": 0.9372, "acc_ci": [ 0.8825, 0.984 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 0.84, 0.8462 ] }, "ai_or_real": { "score": 0.3467, "ci": [ 0.2133, 0.48 ], "acc": 0.6733, "acc_ci": [ 0.6067, 0.74 ], "chance": 0.5, "levels": [ 0.8333, 0.7667, 0.8, 0.4333, 0.5333 ] }, "cable_scene": { "score": 0.568, "ci": [ 0.528, 0.616 ], "acc": 0.568, "acc_ci": [ 0.528, 0.616 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.68, 0.16, 0.0 ] }, "cant_tell_scene": { "score": 0.9185, "ci": [ 0.8509, 0.9732 ], "acc": 0.9392, "acc_ci": [ 0.8887, 0.98 ], "chance": 0.2538, "levels": [ 0.9167, 0.96, 0.88, 1.0 ] }, "chart_screen": { "score": 0.46, "ci": [ 0.38, 0.54 ], "acc": 0.46, "acc_ci": [ 0.38, 0.54 ], "chance": 0.0, "levels": [ 0.92, 0.48, 0.28, 0.16 ] }, "coin_sum": { "score": 0.486, "ci": [ 0.43, 0.5417 ], "acc": 0.486, "acc_ci": [ 0.43, 0.5417 ], "chance": 0.0, "levels": [ 1.0, 0.75, 0.64, 0.0, 0.04 ] }, "count_scene": { "score": 0.336, "ci": [ 0.272, 0.392 ], "acc": 0.336, "acc_ci": [ 0.272, 0.392 ], "chance": 0.0, "levels": [ 0.68, 0.76, 0.2, 0.04, 0.0 ] }, "dashcam_light": { "score": 0.676, "ci": [ 0.58, 0.772 ], "acc": 0.784, "acc_ci": [ 0.72, 0.848 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.92, 0.48, 0.52 ] }, "depth_scene": { "score": 0.7333, "ci": [ 0.6333, 0.8333 ], "acc": 0.8667, "acc_ci": [ 0.8167, 0.9167 ], "chance": 0.5, "levels": [ 1.0, 1.0, 1.0, 0.5, 0.8333 ] }, "eye_chart": { "score": 0.376, "ci": [ 0.312, 0.448 ], "acc": 0.376, "acc_ci": [ 0.312, 0.448 ], "chance": 0.0, "levels": [ 0.76, 0.32, 0.56, 0.24, 0.0 ] }, "facing_dir": { "score": 0.5851, "ci": [ 0.4995, 0.6666 ], "acc": 0.6473, "acc_ci": [ 0.5746, 0.7166 ], "chance": 0.15, "levels": [ 0.9565, 0.68, 0.64, 0.8, 0.16 ] }, "illusion": { "score": 0.016, "ci": [ 0.0, 0.136 ], "acc": 0.344, "acc_ci": [ 0.272, 0.424 ], "chance": 0.3333, "levels": [ 0.52, 0.4, 0.32, 0.2, 0.28 ] }, "invoice_check": { "score": 0.8886, "ci": [ 0.8372, 0.94 ], "acc": 0.896, "acc_ci": [ 0.848, 0.944 ], "chance": 0.0666, "levels": [ 1.0, 1.0, 1.0, 0.8, 0.68 ] }, "label_scene": { "score": 0.768, "ci": [ 0.72, 0.816 ], "acc": 0.768, "acc_ci": [ 0.72, 0.816 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.56, 0.28 ] }, "meter_scene": { "score": 0.552, "ci": [ 0.48, 0.632 ], "acc": 0.552, "acc_ci": [ 0.48, 0.632 ], "chance": 0.0, "levels": [ 0.88, 0.44, 0.76, 0.48, 0.2 ] }, "missing_dim": { "score": 0.872, "ci": [ 0.832, 0.912 ], "acc": 0.872, "acc_ci": [ 0.832, 0.912 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.36 ] }, "pallet_count": { "score": 0.272, "ci": [ 0.2, 0.352 ], "acc": 0.272, "acc_ci": [ 0.2, 0.352 ], "chance": 0.0, "levels": [ 0.44, 0.2, 0.52, 0.12, 0.08 ] }, "parking_bay": { "score": 0.9551, "ci": [ 0.9103, 0.9821 ], "acc": 0.96, "acc_ci": [ 0.92, 0.984 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 0.96, 0.84 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.4023, "acc_ci": [ 0.315, 0.49 ], "chance": 0.5, "levels": [ 0.52, 0.56, 0.32, 0.2917, 0.32 ] }, "pipe_follow": { "score": 0.536, "ci": [ 0.472, 0.592 ], "acc": 0.536, "acc_ci": [ 0.472, 0.592 ], "chance": 0.0, "levels": [ 0.92, 0.64, 0.88, 0.2, 0.04 ] }, "render_bug": { "score": 0.2501, "ci": [ 0.152, 0.3528 ], "acc": 0.3751, "acc_ci": [ 0.2933, 0.4606 ], "chance": 0.1667, "levels": [ 0.64, 0.375, 0.3182, 0.3043, 0.2381 ] }, "shelf_plan": { "score": 0.68, "ci": [ 0.632, 0.728 ], "acc": 0.68, "acc_ci": [ 0.632, 0.728 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.92, 0.44, 0.04 ] }, "stack_scene": { "score": 0.513, "ci": [ 0.4062, 0.6176 ], "acc": 0.6753, "acc_ci": [ 0.6041, 0.745 ], "chance": 0.3333, "levels": [ 1.0, 0.96, 0.4167, 0.5, 0.5 ] }, "test_strips": { "score": 0.656, "ci": [ 0.624, 0.696 ], "acc": 0.656, "acc_ci": [ 0.624, 0.696 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.08, 0.2 ] } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 3.04, "cost_label": "$3.04", "runs_locally": false, "seconds_per_answer": 48.0, "open_weights": false }, { "model": "Grok 4.7", "maker": "xAI", "id": "grok-4.7", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.549, "overall_ci": [ 0.4809, 0.6414 ], "categories": { "reading": { "score": 0.5375, "ci": [ 0.4, 0.6814 ], "partial": false }, "counting": { "score": 0.5333, "ci": [ 0.2667, 0.8 ], "partial": false }, "space": { "score": 0.5015, "ci": [ 0.3785, 0.7045 ], "partial": false }, "changes": { "score": 0.68, "ci": [ 0.4798, 0.89 ], "partial": false }, "diagrams": { "score": 0.4, "ci": [ 0.2, 0.6 ], "partial": false }, "fooled": { "score": 0.6417, "ci": [ 0.4248, 0.9167 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 0.67, "chance": 0.25 }, "change_scene": { "score": 0.6, "chance": 0.0 }, "chart_screen": { "score": 0.5, "chance": 0.0 }, "clock_scene": { "score": 0.4, "chance": 0.0 }, "coin_sum": { "score": 0.6, "chance": 0.0 }, "conveyor_count": { "score": 0.4, "chance": 0.0 }, "count_scene": { "score": 0.6, "chance": 0.0 }, "dashcam_light": { "score": 0.4, "chance": 0.3333 }, "depth_scene": { "score": 1.0, "chance": 0.5 }, "eye_chart": { "score": 0.0, "chance": 0.0 }, "facing_dir": { "score": 0.29, "chance": 0.15 }, "illusion": { "score": 0.7, "chance": 0.3333 }, "invoice_check": { "score": 1.0, "chance": 0.0635 }, "label_scene": { "score": 1.0, "chance": 0.0 }, "match_render": { "score": 0.47, "chance": 0.25 }, "meter_scene": { "score": 0.8, "chance": 0.0 }, "missing_dim": { "score": 0.8, "chance": 0.0 }, "pallet_count": { "score": 0.6, "chance": 0.0 }, "parking_bay": { "score": 1.0, "chance": 0.1083 }, "part_mirror": { "score": 0.2, "chance": 0.5 }, "pipe_follow": { "score": 0.0, "chance": 0.0 }, "render_bug": { "score": 0.52, "chance": 0.1667 }, "shelf_plan": { "score": 0.6, "chance": 0.0 }, "shell_game": { "score": 0.45, "chance": 0.2733 }, "stack_scene": { "score": 0.1, "chance": 0.3333 }, "test_strips": { "score": 0.6, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 27.4, "cost_label": "$27.40", "runs_locally": false, "seconds_per_answer": 43.7, "open_weights": false }, { "model": "Mistral Large 4", "maker": "Mistral", "id": "mistral-large-4-0", "access": "API", "type": "general", "group": "Others", "effort": "default", "overall": 0.5272, "overall_ci": [ 0.4674, 0.6166 ], "categories": { "reading": { "score": 0.475, "ci": [ 0.35, 0.6313 ], "partial": false }, "counting": { "score": 0.4, "ci": [ 0.3333, 0.5333 ], "partial": false }, "space": { "score": 0.3764, "ci": [ 0.2465, 0.5714 ], "partial": false }, "changes": { "score": 0.67, "ci": [ 0.5, 0.85 ], "partial": false }, "diagrams": { "score": 0.6667, "ci": [ 0.4, 0.8667 ], "partial": false }, "fooled": { "score": 0.575, "ci": [ 0.5, 0.85 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.6, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.25 }, "change_scene": { "score": 0.8, "chance": 0.0 }, "chart_screen": { "score": 0.5, "chance": 0.0 }, "clock_scene": { "score": 0.4, "chance": 0.0 }, "coin_sum": { "score": 0.6, "chance": 0.0 }, "conveyor_count": { "score": 0.2, "chance": 0.0 }, "count_scene": { "score": 1.0, "chance": 0.0 }, "dashcam_light": { "score": 0.1, "chance": 0.3333 }, "depth_scene": { "score": 0.6, "chance": 0.5 }, "eye_chart": { "score": 0.2, "chance": 0.0 }, "facing_dir": { "score": 0.29, "chance": 0.15 }, "illusion": { "score": 0.1, "chance": 0.3333 }, "invoice_check": { "score": 1.0, "chance": 0.0635 }, "label_scene": { "score": 0.8, "chance": 0.0 }, "match_render": { "score": 0.47, "chance": 0.25 }, "meter_scene": { "score": 0.6, "chance": 0.0 }, "missing_dim": { "score": 0.8, "chance": 0.0 }, "pallet_count": { "score": 0.0, "chance": 0.0 }, "parking_bay": { "score": 1.0, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.6, "chance": 0.0 }, "render_bug": { "score": 0.28, "chance": 0.1667 }, "shelf_plan": { "score": 0.6, "chance": 0.0 }, "shell_game": { "score": 0.17, "chance": 0.2733 }, "stack_scene": { "score": 0.1, "chance": 0.3333 }, "test_strips": { "score": 0.6, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 9.59, "cost_label": "$9.59", "runs_locally": false, "seconds_per_answer": 52.2, "open_weights": true }, { "model": "Muse Spark (open)", "maker": "Meta", "id": "muse-spark-open", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.4763, "overall_ci": [ 0.4026, 0.5727 ], "categories": { "reading": { "score": 0.5375, "ci": [ 0.3813, 0.6939 ], "partial": false }, "counting": { "score": 0.3333, "ci": [ 0.1333, 0.5333 ], "partial": false }, "space": { "score": 0.4101, "ci": [ 0.2801, 0.6175 ], "partial": false }, "changes": { "score": 0.5266, "ci": [ 0.3098, 0.7466 ], "partial": false }, "diagrams": { "score": 0.4, "ci": [ 0.2, 0.6 ], "partial": false }, "fooled": { "score": 0.65, "ci": [ 0.5, 0.925 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.25 }, "change_scene": { "score": 0.4, "chance": 0.0 }, "chart_screen": { "score": 0.5, "chance": 0.0 }, "clock_scene": { "score": 0.6, "chance": 0.0 }, "coin_sum": { "score": 0.4, "chance": 0.0 }, "conveyor_count": { "score": 0.2, "chance": 0.0 }, "count_scene": { "score": 0.6, "chance": 0.0 }, "dashcam_light": { "score": 0.4, "chance": 0.3333 }, "depth_scene": { "score": 0.6, "chance": 0.5 }, "eye_chart": { "score": 0.2, "chance": 0.0 }, "facing_dir": { "score": 0.53, "chance": 0.15 }, "illusion": { "score": 0.4, "chance": 0.3333 }, "invoice_check": { "score": 0.79, "chance": 0.0635 }, "label_scene": { "score": 0.8, "chance": 0.0 }, "match_render": { "score": 0.47, "chance": 0.25 }, "meter_scene": { "score": 0.8, "chance": 0.0 }, "missing_dim": { "score": 0.6, "chance": 0.0 }, "pallet_count": { "score": 0.2, "chance": 0.0 }, "parking_bay": { "score": 1.0, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.2, "chance": 0.0 }, "render_bug": { "score": 0.52, "chance": 0.1667 }, "shelf_plan": { "score": 0.4, "chance": 0.0 }, "shell_game": { "score": 0.17, "chance": 0.2733 }, "stack_scene": { "score": 0.1, "chance": 0.3333 }, "test_strips": { "score": 0.6, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 19.12, "cost_label": "$19.12", "runs_locally": false, "seconds_per_answer": 29.9, "open_weights": true }, { "model": "GLM-5V-Turbo", "maker": "Zhipu", "id": "glm-5v-turbo", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.4679, "overall_ci": [ 0.4465, 0.4946 ], "categories": { "reading": { "score": 0.5008, "ci": [ 0.4641, 0.5352 ], "partial": false }, "counting": { "score": 0.1917, "ci": [ 0.1333, 0.25 ], "partial": false }, "space": { "score": 0.4867, "ci": [ 0.4322, 0.5586 ], "partial": false }, "changes": { "score": 0.6278, "ci": [ 0.5652, 0.6912 ], "partial": false }, "diagrams": { "score": 0.4667, "ci": [ 0.4165, 0.5167 ], "partial": false }, "fooled": { "score": 0.5334, "ci": [ 0.471, 0.6292 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.7, "ci": [ 0.6, 0.8 ], "acc": 0.7, "acc_ci": [ 0.6, 0.8 ], "chance": 0.0, "levels": [ 1.0, 0.75, 0.875, 0.125, 0.75 ] }, "change_scene": { "score": 0.525, "ci": [ 0.375, 0.65 ], "acc": 0.525, "acc_ci": [ 0.375, 0.65 ], "chance": 0.0, "levels": [ 0.75, 0.625, 0.5, 0.625, 0.125 ] }, "conveyor_count": { "score": 0.1, "ci": [ 0.025, 0.2 ], "acc": 0.1, "acc_ci": [ 0.025, 0.2 ], "chance": 0.0, "levels": [ 0.375, 0.125, 0.0, 0.0, 0.0 ] }, "match_render": { "score": 0.5333, "ci": [ 0.4, 0.6667 ], "acc": 0.65, "acc_ci": [ 0.55, 0.75 ], "chance": 0.25, "levels": [ 0.875, 0.875, 1.0, 0.125, 0.375 ] }, "shell_game": { "score": 0.4151, "ci": [ 0.2775, 0.5528 ], "acc": 0.575, "acc_ci": [ 0.475, 0.675 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.5, 0.25, 0.125 ] }, "absent_object": { "score": 0.938, "ci": [ 0.845, 1.0 ], "acc": 0.95, "acc_ci": [ 0.875, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.75 ] }, "ai_or_real": { "score": 0.1, "ci": [ 0.0, 0.4 ], "acc": 0.55, "acc_ci": [ 0.4, 0.7 ], "chance": 0.5, "levels": [ 0.625, 0.625, 0.625, 0.5, 0.375 ] }, "cable_scene": { "score": 0.475, "ci": [ 0.4, 0.55 ], "acc": 0.475, "acc_ci": [ 0.4, 0.55 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.25, 0.125, 0.0 ] }, "cant_tell_scene": { "score": 0.958, "ci": [ 0.8741, 1.0 ], "acc": 0.9688, "acc_ci": [ 0.9062, 1.0 ], "chance": 0.2552, "levels": [ 1.0, 1.0, 0.875, 1.0 ] }, "chart_screen": { "score": 0.4062, "ci": [ 0.3125, 0.5 ], "acc": 0.4062, "acc_ci": [ 0.3125, 0.5 ], "chance": 0.0, "levels": [ 1.0, 0.375, 0.25, 0.0 ] }, "coin_sum": { "score": 0.5, "ci": [ 0.4, 0.575 ], "acc": 0.5, "acc_ci": [ 0.4, 0.575 ], "chance": 0.0, "levels": [ 1.0, 0.75, 0.75, 0.0, 0.0 ] }, "count_scene": { "score": 0.225, "ci": [ 0.125, 0.325 ], "acc": 0.225, "acc_ci": [ 0.125, 0.325 ], "chance": 0.0, "levels": [ 0.75, 0.25, 0.125, 0.0, 0.0 ] }, "dashcam_light": { "score": 0.7, "ci": [ 0.55, 0.85 ], "acc": 0.8, "acc_ci": [ 0.7, 0.9 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 1.0, 0.625, 0.375 ] }, "depth_scene": { "score": 0.7, "ci": [ 0.5, 0.9 ], "acc": 0.85, "acc_ci": [ 0.75, 0.95 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.875, 0.625, 0.75 ] }, "eye_chart": { "score": 0.125, "ci": [ 0.05, 0.175 ], "acc": 0.125, "acc_ci": [ 0.05, 0.175 ], "chance": 0.0, "levels": [ 0.625, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.4706, "ci": [ 0.3235, 0.6176 ], "acc": 0.55, "acc_ci": [ 0.425, 0.675 ], "chance": 0.15, "levels": [ 1.0, 0.5, 0.375, 0.5, 0.375 ] }, "illusion": { "score": 0.1375, "ci": [ 0.0, 0.3625 ], "acc": 0.425, "acc_ci": [ 0.275, 0.575 ], "chance": 0.3333, "levels": [ 0.5, 0.375, 0.25, 0.5, 0.5 ] }, "invoice_check": { "score": 0.8662, "ci": [ 0.7859, 0.9465 ], "acc": 0.875, "acc_ci": [ 0.8, 0.95 ], "chance": 0.0659, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.5 ] }, "label_scene": { "score": 0.6, "ci": [ 0.6, 0.6 ], "acc": 0.6, "acc_ci": [ 0.6, 0.6 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.0, 0.0 ] }, "meter_scene": { "score": 0.475, "ci": [ 0.35, 0.6 ], "acc": 0.475, "acc_ci": [ 0.35, 0.6 ], "chance": 0.0, "levels": [ 0.875, 0.375, 0.625, 0.5, 0.0 ] }, "missing_dim": { "score": 0.6, "ci": [ 0.5, 0.7 ], "acc": 0.6, "acc_ci": [ 0.5, 0.7 ], "chance": 0.0, "levels": [ 1.0, 0.625, 0.75, 0.625, 0.0 ] }, "pallet_count": { "score": 0.25, "ci": [ 0.125, 0.375 ], "acc": 0.25, "acc_ci": [ 0.125, 0.375 ], "chance": 0.0, "levels": [ 0.25, 0.375, 0.5, 0.0, 0.125 ] }, "parking_bay": { "score": 0.8879, "ci": [ 0.8038, 0.972 ], "acc": 0.9, "acc_ci": [ 0.825, 0.975 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 0.875, 0.625 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.25 ], "acc": 0.5, "acc_ci": [ 0.375, 0.625 ], "chance": 0.5, "levels": [ 0.875, 0.625, 0.375, 0.5, 0.125 ] }, "pipe_follow": { "score": 0.325, "ci": [ 0.225, 0.425 ], "acc": 0.325, "acc_ci": [ 0.225, 0.425 ], "chance": 0.0, "levels": [ 0.75, 0.75, 0.125, 0.0, 0.0 ] }, "render_bug": { "score": 0.52, "ci": [ 0.37, 0.67 ], "acc": 0.6, "acc_ci": [ 0.475, 0.725 ], "chance": 0.1667, "levels": [ 0.875, 0.5, 0.375, 0.625, 0.625 ] }, "shelf_plan": { "score": 0.6, "ci": [ 0.55, 0.675 ], "acc": 0.6, "acc_ci": [ 0.55, 0.675 ], "chance": 0.0, "levels": [ 1.0, 0.875, 1.0, 0.125, 0.0 ] }, "stack_scene": { "score": 0.4, "ci": [ 0.175, 0.625 ], "acc": 0.6, "acc_ci": [ 0.45, 0.75 ], "chance": 0.3333, "levels": [ 0.875, 0.625, 0.5, 0.5, 0.5 ] }, "test_strips": { "score": 0.5, "ci": [ 0.3994, 0.6 ], "acc": 0.5, "acc_ci": [ 0.3994, 0.6 ], "chance": 0.0, "levels": [ 1.0, 0.75, 0.375, 0.375, 0.0 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 12.5, "cost_label": "$12.50", "runs_locally": false, "seconds_per_answer": 19.9, "open_weights": false }, { "model": "DeepSeek V4.1 Flash", "maker": "DeepSeek", "id": "deepseek-flash", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.4441, "overall_ci": [ 0.4319, 0.4577 ], "categories": { "reading": { "score": 0.4064, "ci": [ 0.3818, 0.4317 ], "partial": false }, "counting": { "score": 0.1747, "ci": [ 0.146, 0.2047 ], "partial": false }, "space": { "score": 0.4188, "ci": [ 0.3879, 0.4579 ], "partial": false }, "changes": { "score": 0.6083, "ci": [ 0.5743, 0.6425 ], "partial": false }, "diagrams": { "score": 0.5387, "ci": [ 0.512, 0.568 ], "partial": false }, "fooled": { "score": 0.5175, "ci": [ 0.4713, 0.5619 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.256, "ci": [ 0.176, 0.328 ], "acc": 0.256, "acc_ci": [ 0.176, 0.328 ], "chance": 0.0, "levels": [ 0.32, 0.16, 0.32, 0.12, 0.36 ] }, "change_scene": { "score": 0.6227, "ci": [ 0.542, 0.7023 ], "acc": 0.6227, "acc_ci": [ 0.542, 0.7023 ], "chance": 0.0, "levels": [ 0.84, 0.8333, 0.56, 0.52, 0.36 ] }, "conveyor_count": { "score": 0.22, "ci": [ 0.17, 0.27 ], "acc": 0.22, "acc_ci": [ 0.17, 0.27 ], "chance": 0.0, "levels": [ 0.8, 0.3, 0.0, 0.0, 0.0 ] }, "match_render": { "score": 0.4164, "ci": [ 0.3187, 0.5133 ], "acc": 0.5623, "acc_ci": [ 0.489, 0.635 ], "chance": 0.25, "levels": [ 0.84, 0.84, 0.6, 0.2917, 0.24 ] }, "shell_game": { "score": 0.3725, "ci": [ 0.3037, 0.4413 ], "acc": 0.544, "acc_ci": [ 0.494, 0.594 ], "chance": 0.2733, "levels": [ 1.0, 1.0, 0.6, 0.0, 0.12 ] }, "absent_object": { "score": 0.8642, "ci": [ 0.7864, 0.9413 ], "acc": 0.8905, "acc_ci": [ 0.8277, 0.9526 ], "chance": 0.1933, "levels": [ 0.96, 1.0, 0.96, 0.84, 0.6923 ] }, "ai_or_real": { "score": 0.3067, "ci": [ 0.16, 0.4533 ], "acc": 0.6533, "acc_ci": [ 0.58, 0.7267 ], "chance": 0.5, "levels": [ 0.8, 0.8, 0.5333, 0.5667, 0.5667 ] }, "cable_scene": { "score": 0.52, "ci": [ 0.472, 0.576 ], "acc": 0.52, "acc_ci": [ 0.472, 0.576 ], "chance": 0.0, "levels": [ 1.0, 0.96, 0.48, 0.08, 0.08 ] }, "cant_tell_scene": { "score": 0.899, "ci": [ 0.8249, 0.9708 ], "acc": 0.9247, "acc_ci": [ 0.8693, 0.9782 ], "chance": 0.2538, "levels": [ 0.875, 0.96, 1.0, 0.8636 ] }, "chart_screen": { "score": 0.5, "ci": [ 0.42, 0.59 ], "acc": 0.5, "acc_ci": [ 0.42, 0.59 ], "chance": 0.0, "levels": [ 0.88, 0.52, 0.36, 0.24 ] }, "coin_sum": { "score": 0.3233, "ci": [ 0.2737, 0.3803 ], "acc": 0.3233, "acc_ci": [ 0.2737, 0.3803 ], "chance": 0.0, "levels": [ 0.96, 0.4167, 0.24, 0.0, 0.0 ] }, "count_scene": { "score": 0.288, "ci": [ 0.2318, 0.352 ], "acc": 0.288, "acc_ci": [ 0.2318, 0.352 ], "chance": 0.0, "levels": [ 0.64, 0.56, 0.16, 0.08, 0.0 ] }, "dashcam_light": { "score": 0.484, "ci": [ 0.376, 0.592 ], "acc": 0.656, "acc_ci": [ 0.584, 0.728 ], "chance": 0.3333, "levels": [ 1.0, 0.8, 0.6, 0.52, 0.36 ] }, "depth_scene": { "score": 0.6513, "ci": [ 0.55, 0.7667 ], "acc": 0.8257, "acc_ci": [ 0.775, 0.8833 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.92, 0.4167, 0.7917 ] }, "eye_chart": { "score": 0.2, "ci": [ 0.152, 0.248 ], "acc": 0.2, "acc_ci": [ 0.152, 0.248 ], "chance": 0.0, "levels": [ 0.76, 0.12, 0.08, 0.04, 0.0 ] }, "facing_dir": { "score": 0.3567, "ci": [ 0.2696, 0.4533 ], "acc": 0.4532, "acc_ci": [ 0.3791, 0.5353 ], "chance": 0.15, "levels": [ 0.8261, 0.36, 0.44, 0.36, 0.28 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.028 ], "acc": 0.272, "acc_ci": [ 0.2, 0.352 ], "chance": 0.3333, "levels": [ 0.28, 0.32, 0.28, 0.2, 0.28 ] }, "invoice_check": { "score": 0.9229, "ci": [ 0.88, 0.9657 ], "acc": 0.928, "acc_ci": [ 0.888, 0.968 ], "chance": 0.0666, "levels": [ 1.0, 1.0, 1.0, 0.8, 0.84 ] }, "label_scene": { "score": 0.68, "ci": [ 0.64, 0.728 ], "acc": 0.68, "acc_ci": [ 0.64, 0.728 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.24, 0.16 ] }, "meter_scene": { "score": 0.36, "ci": [ 0.296, 0.432 ], "acc": 0.36, "acc_ci": [ 0.296, 0.432 ], "chance": 0.0, "levels": [ 0.84, 0.24, 0.32, 0.4, 0.0 ] }, "missing_dim": { "score": 0.776, "ci": [ 0.744, 0.808 ], "acc": 0.776, "acc_ci": [ 0.744, 0.808 ], "chance": 0.0, "levels": [ 1.0, 0.92, 1.0, 0.92, 0.04 ] }, "pallet_count": { "score": 0.016, "ci": [ 0.0, 0.04 ], "acc": 0.016, "acc_ci": [ 0.0, 0.04 ], "chance": 0.0, "levels": [ 0.04, 0.04, 0.0, 0.0, 0.0 ] }, "parking_bay": { "score": 0.8923, "ci": [ 0.8385, 0.9372 ], "acc": 0.904, "acc_ci": [ 0.856, 0.944 ], "chance": 0.1083, "levels": [ 0.96, 1.0, 1.0, 0.92, 0.64 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.1333 ], "acc": 0.485, "acc_ci": [ 0.404, 0.5667 ], "chance": 0.5, "levels": [ 0.64, 0.48, 0.32, 0.625, 0.36 ] }, "pipe_follow": { "score": 0.32, "ci": [ 0.256, 0.384 ], "acc": 0.32, "acc_ci": [ 0.256, 0.384 ], "chance": 0.0, "levels": [ 0.76, 0.6, 0.24, 0.0, 0.0 ] }, "render_bug": { "score": 0.3036, "ci": [ 0.2042, 0.4073 ], "acc": 0.4196, "acc_ci": [ 0.3368, 0.5061 ], "chance": 0.1667, "levels": [ 0.68, 0.4583, 0.4091, 0.2174, 0.3333 ] }, "shelf_plan": { "score": 0.584, "ci": [ 0.56, 0.6 ], "acc": 0.584, "acc_ci": [ 0.56, 0.6 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.92, 0.0, 0.0 ] }, "stack_scene": { "score": 0.2424, "ci": [ 0.1289, 0.3571 ], "acc": 0.4949, "acc_ci": [ 0.4193, 0.5714 ], "chance": 0.3333, "levels": [ 1.0, 0.52, 0.2917, 0.4545, 0.2083 ] }, "test_strips": { "score": 0.448, "ci": [ 0.4, 0.496 ], "acc": 0.448, "acc_ci": [ 0.4, 0.496 ], "chance": 0.0, "levels": [ 1.0, 0.88, 0.36, 0.0, 0.0 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 1.43, "cost_label": "$1.43", "runs_locally": false, "seconds_per_answer": 10.9, "open_weights": true }, { "model": "GLM-5.3 Flash", "maker": "Zhipu", "id": "glm-5.3-flash", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.4317, "overall_ci": [ 0.4167, 0.4468 ], "categories": { "reading": { "score": 0.4248, "ci": [ 0.398, 0.4532 ], "partial": false }, "counting": { "score": 0.1627, "ci": [ 0.1313, 0.196 ], "partial": false }, "space": { "score": 0.3918, "ci": [ 0.3568, 0.4268 ], "partial": false }, "changes": { "score": 0.5563, "ci": [ 0.5176, 0.5946 ], "partial": false }, "diagrams": { "score": 0.488, "ci": [ 0.448, 0.5253 ], "partial": false }, "fooled": { "score": 0.5664, "ci": [ 0.5178, 0.6155 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.424, "ci": [ 0.344, 0.496 ], "acc": 0.424, "acc_ci": [ 0.344, 0.496 ], "chance": 0.0, "levels": [ 0.64, 0.24, 0.6, 0.52, 0.12 ] }, "change_scene": { "score": 0.5573, "ci": [ 0.4697, 0.638 ], "acc": 0.5573, "acc_ci": [ 0.4697, 0.638 ], "chance": 0.0, "levels": [ 0.72, 0.6667, 0.52, 0.6, 0.28 ] }, "conveyor_count": { "score": 0.12, "ci": [ 0.07, 0.17 ], "acc": 0.12, "acc_ci": [ 0.07, 0.17 ], "chance": 0.0, "levels": [ 0.5, 0.05, 0.05, 0.0, 0.0 ] }, "match_render": { "score": 0.5258, "ci": [ 0.4289, 0.6236 ], "acc": 0.6443, "acc_ci": [ 0.5717, 0.7177 ], "chance": 0.25, "levels": [ 0.96, 0.8, 0.68, 0.5417, 0.24 ] }, "shell_game": { "score": 0.3037, "ci": [ 0.2046, 0.4055 ], "acc": 0.494, "acc_ci": [ 0.422, 0.568 ], "chance": 0.2733, "levels": [ 0.9, 0.8, 0.45, 0.16, 0.16 ] }, "absent_object": { "score": 0.8932, "ci": [ 0.8161, 0.9603 ], "acc": 0.9138, "acc_ci": [ 0.8517, 0.968 ], "chance": 0.1933, "levels": [ 0.92, 1.0, 0.96, 0.92, 0.7692 ] }, "ai_or_real": { "score": 0.36, "ci": [ 0.2267, 0.48 ], "acc": 0.68, "acc_ci": [ 0.6133, 0.74 ], "chance": 0.5, "levels": [ 0.7667, 0.8667, 0.9333, 0.3667, 0.4667 ] }, "cable_scene": { "score": 0.4, "ci": [ 0.328, 0.464 ], "acc": 0.4, "acc_ci": [ 0.328, 0.464 ], "chance": 0.0, "levels": [ 0.76, 0.8, 0.24, 0.12, 0.08 ] }, "cant_tell_scene": { "score": 0.9003, "ci": [ 0.8285, 0.9587 ], "acc": 0.9256, "acc_ci": [ 0.872, 0.9692 ], "chance": 0.2538, "levels": [ 0.8333, 0.96, 1.0, 0.9091 ] }, "chart_screen": { "score": 0.44, "ci": [ 0.36, 0.53 ], "acc": 0.44, "acc_ci": [ 0.36, 0.53 ], "chance": 0.0, "levels": [ 0.8, 0.6, 0.24, 0.12 ] }, "coin_sum": { "score": 0.2503, "ci": [ 0.1937, 0.308 ], "acc": 0.2503, "acc_ci": [ 0.1937, 0.308 ], "chance": 0.0, "levels": [ 0.76, 0.2917, 0.2, 0.0, 0.0 ] }, "count_scene": { "score": 0.256, "ci": [ 0.2, 0.312 ], "acc": 0.256, "acc_ci": [ 0.2, 0.312 ], "chance": 0.0, "levels": [ 0.76, 0.36, 0.08, 0.08, 0.0 ] }, "dashcam_light": { "score": 0.46, "ci": [ 0.364, 0.556 ], "acc": 0.64, "acc_ci": [ 0.576, 0.704 ], "chance": 0.3333, "levels": [ 1.0, 0.92, 0.76, 0.32, 0.2 ] }, "depth_scene": { "score": 0.536, "ci": [ 0.406, 0.655 ], "acc": 0.768, "acc_ci": [ 0.703, 0.8275 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.84, 0.5, 0.5 ] }, "eye_chart": { "score": 0.264, "ci": [ 0.2, 0.336 ], "acc": 0.264, "acc_ci": [ 0.2, 0.336 ], "chance": 0.0, "levels": [ 0.64, 0.32, 0.24, 0.08, 0.04 ] }, "facing_dir": { "score": 0.4533, "ci": [ 0.367, 0.5388 ], "acc": 0.5353, "acc_ci": [ 0.4619, 0.608 ], "chance": 0.15, "levels": [ 0.9565, 0.48, 0.48, 0.56, 0.2 ] }, "illusion": { "score": 0.112, "ci": [ 0.0, 0.244 ], "acc": 0.408, "acc_ci": [ 0.32, 0.496 ], "chance": 0.3333, "levels": [ 0.56, 0.48, 0.36, 0.28, 0.36 ] }, "invoice_check": { "score": 0.82, "ci": [ 0.7514, 0.8886 ], "acc": 0.832, "acc_ci": [ 0.768, 0.896 ], "chance": 0.0666, "levels": [ 0.76, 0.92, 0.88, 0.84, 0.76 ] }, "label_scene": { "score": 0.608, "ci": [ 0.56, 0.656 ], "acc": 0.608, "acc_ci": [ 0.56, 0.656 ], "chance": 0.0, "levels": [ 1.0, 0.88, 0.92, 0.12, 0.12 ] }, "meter_scene": { "score": 0.432, "ci": [ 0.36, 0.504 ], "acc": 0.432, "acc_ci": [ 0.36, 0.504 ], "chance": 0.0, "levels": [ 0.84, 0.48, 0.48, 0.32, 0.04 ] }, "missing_dim": { "score": 0.696, "ci": [ 0.632, 0.76 ], "acc": 0.696, "acc_ci": [ 0.632, 0.76 ], "chance": 0.0, "levels": [ 0.96, 0.8, 0.84, 0.68, 0.2 ] }, "pallet_count": { "score": 0.112, "ci": [ 0.064, 0.168 ], "acc": 0.112, "acc_ci": [ 0.064, 0.168 ], "chance": 0.0, "levels": [ 0.28, 0.12, 0.12, 0.0, 0.04 ] }, "parking_bay": { "score": 0.7488, "ci": [ 0.6681, 0.8206 ], "acc": 0.776, "acc_ci": [ 0.704, 0.84 ], "chance": 0.1083, "levels": [ 0.8, 0.88, 0.84, 0.88, 0.48 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.2257, "acc_ci": [ 0.1603, 0.2993 ], "chance": 0.5, "levels": [ 0.32, 0.16, 0.24, 0.2083, 0.2 ] }, "pipe_follow": { "score": 0.368, "ci": [ 0.304, 0.432 ], "acc": 0.368, "acc_ci": [ 0.304, 0.432 ], "chance": 0.0, "levels": [ 0.68, 0.68, 0.4, 0.08, 0.0 ] }, "render_bug": { "score": 0.2717, "ci": [ 0.1752, 0.376 ], "acc": 0.393, "acc_ci": [ 0.3127, 0.48 ], "chance": 0.1667, "levels": [ 0.68, 0.375, 0.3636, 0.2609, 0.2857 ] }, "shelf_plan": { "score": 0.576, "ci": [ 0.528, 0.624 ], "acc": 0.576, "acc_ci": [ 0.528, 0.624 ], "chance": 0.0, "levels": [ 0.96, 0.88, 0.88, 0.16, 0.0 ] }, "stack_scene": { "score": 0.175, "ci": [ 0.055, 0.2998 ], "acc": 0.45, "acc_ci": [ 0.37, 0.5332 ], "chance": 0.3333, "levels": [ 0.72, 0.64, 0.3333, 0.1818, 0.375 ] }, "test_strips": { "score": 0.52, "ci": [ 0.448, 0.584 ], "acc": 0.52, "acc_ci": [ 0.448, 0.584 ], "chance": 0.0, "levels": [ 0.8, 0.92, 0.56, 0.24, 0.08 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 1.4, "cost_label": "$1.40", "runs_locally": false, "seconds_per_answer": 9.8, "open_weights": true }, { "model": "Qwen3.6-35B-A3B", "maker": "Alibaba", "id": "qwen3.6-35b-a3b", "access": "API", "type": "general", "group": "Open models via API", "effort": "default", "overall": 0.4301, "overall_ci": [ 0.3689, 0.524 ], "categories": { "reading": { "score": 0.625, "ci": [ 0.475, 0.7687 ], "partial": false }, "counting": { "score": 0.1333, "ci": [ 0.0, 0.2667 ], "partial": false }, "space": { "score": 0.3209, "ci": [ 0.2049, 0.5354 ], "partial": false }, "changes": { "score": 0.4598, "ci": [ 0.2498, 0.6832 ], "partial": false }, "diagrams": { "score": 0.4667, "ci": [ 0.2, 0.7333 ], "partial": false }, "fooled": { "score": 0.575, "ci": [ 0.5, 0.85 ], "partial": false } }, "tests": { "change_scene": { "score": 0.6, "chance": 0.0 }, "conveyor_count": { "score": 0.0, "chance": 0.0 }, "match_render": { "score": 0.47, "chance": 0.25 }, "shell_game": { "score": 0.17, "chance": 0.2733 }, "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.25 }, "chart_screen": { "score": 0.5, "chance": 0.0 }, "clock_scene": { "score": 0.8, "chance": 0.0 }, "coin_sum": { "score": 0.6, "chance": 0.0 }, "test_strips": { "score": 0.6, "chance": 0.0 }, "count_scene": { "score": 0.4, "chance": 0.0 }, "stack_scene": { "score": 0.1, "chance": 0.3333 }, "dashcam_light": { "score": 0.7, "chance": 0.3333 }, "depth_scene": { "score": 0.2, "chance": 0.5 }, "shelf_plan": { "score": 0.6, "chance": 0.0 }, "eye_chart": { "score": 0.2, "chance": 0.0 }, "render_bug": { "score": 0.28, "chance": 0.1667 }, "facing_dir": { "score": 0.53, "chance": 0.15 }, "pipe_follow": { "score": 0.4, "chance": 0.0 }, "illusion": { "score": 0.1, "chance": 0.3333 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "invoice_check": { "score": 0.36, "chance": 0.0635 }, "parking_bay": { "score": 0.78, "chance": 0.1083 }, "label_scene": { "score": 0.8, "chance": 0.0 }, "meter_scene": { "score": 0.8, "chance": 0.0 }, "missing_dim": { "score": 0.6, "chance": 0.0 }, "pallet_count": { "score": 0.0, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 3.77, "cost_label": "$3.77", "runs_locally": false, "seconds_per_answer": 49.7, "open_weights": true }, { "model": "Qwen3-VL-235B", "maker": "Alibaba", "id": "qwen3-vl-235b-a22b-instruct", "access": "API", "type": "general", "group": "Open models via API", "effort": "default", "overall": 0.4122, "overall_ci": [ 0.3502, 0.5025 ], "categories": { "reading": { "score": 0.4625, "ci": [ 0.3312, 0.6064 ], "partial": false }, "counting": { "score": 0.2, "ci": [ 0.0667, 0.4 ], "partial": false }, "space": { "score": 0.3409, "ci": [ 0.1978, 0.5531 ], "partial": false }, "changes": { "score": 0.42, "ci": [ 0.26, 0.59 ], "partial": false }, "diagrams": { "score": 0.4, "ci": [ 0.2, 0.6 ], "partial": false }, "fooled": { "score": 0.65, "ci": [ 0.5, 0.925 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.6, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.25 }, "change_scene": { "score": 1.0, "chance": 0.0 }, "chart_screen": { "score": 0.5, "chance": 0.0 }, "clock_scene": { "score": 0.6, "chance": 0.0 }, "coin_sum": { "score": 0.2, "chance": 0.0 }, "conveyor_count": { "score": 0.0, "chance": 0.0 }, "count_scene": { "score": 0.2, "chance": 0.0 }, "dashcam_light": { "score": 0.4, "chance": 0.3333 }, "depth_scene": { "score": 0.6, "chance": 0.5 }, "eye_chart": { "score": 0.2, "chance": 0.0 }, "facing_dir": { "score": 0.29, "chance": 0.15 }, "illusion": { "score": 0.4, "chance": 0.3333 }, "invoice_check": { "score": 0.0, "chance": 0.0635 }, "label_scene": { "score": 1.0, "chance": 0.0 }, "match_render": { "score": 0.47, "chance": 0.25 }, "meter_scene": { "score": 0.6, "chance": 0.0 }, "missing_dim": { "score": 0.2, "chance": 0.0 }, "pallet_count": { "score": 0.4, "chance": 0.0 }, "parking_bay": { "score": 0.78, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.4, "chance": 0.0 }, "render_bug": { "score": 0.28, "chance": 0.1667 }, "shelf_plan": { "score": 0.4, "chance": 0.0 }, "shell_game": { "score": 0.25, "chance": 0.3333 }, "stack_scene": { "score": 0.0, "chance": 0.3333 }, "test_strips": { "score": 0.2, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 7.96, "cost_label": "$7.96", "runs_locally": false, "seconds_per_answer": 1.8, "open_weights": true }, { "model": "MiniMax M3", "maker": "MiniMax", "id": "minimax-m3", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.4094, "overall_ci": [ 0.3867, 0.4358 ], "categories": { "reading": { "score": 0.4086, "ci": [ 0.368, 0.4484 ], "partial": false }, "counting": { "score": 0.15, "ci": [ 0.1, 0.2083 ], "partial": false }, "space": { "score": 0.3714, "ci": [ 0.3166, 0.4301 ], "partial": false }, "changes": { "score": 0.5422, "ci": [ 0.4805, 0.6035 ], "partial": false }, "diagrams": { "score": 0.5167, "ci": [ 0.4583, 0.5667 ], "partial": false }, "fooled": { "score": 0.4673, "ci": [ 0.3906, 0.5687 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.325, "ci": [ 0.2, 0.45 ], "acc": 0.325, "acc_ci": [ 0.2, 0.45 ], "chance": 0.0, "levels": [ 0.25, 0.125, 0.375, 0.25, 0.625 ] }, "change_scene": { "score": 0.6, "ci": [ 0.475, 0.725 ], "acc": 0.6, "acc_ci": [ 0.475, 0.725 ], "chance": 0.0, "levels": [ 0.875, 0.875, 0.75, 0.25, 0.25 ] }, "conveyor_count": { "score": 0.25, "ci": [ 0.15, 0.35 ], "acc": 0.25, "acc_ci": [ 0.15, 0.35 ], "chance": 0.0, "levels": [ 0.75, 0.25, 0.125, 0.125, 0.0 ] }, "match_render": { "score": 0.4667, "ci": [ 0.3, 0.6333 ], "acc": 0.6, "acc_ci": [ 0.475, 0.725 ], "chance": 0.25, "levels": [ 0.875, 0.75, 0.75, 0.375, 0.25 ] }, "shell_game": { "score": 0.1743, "ci": [ 0.0711, 0.3119 ], "acc": 0.4, "acc_ci": [ 0.325, 0.5 ], "chance": 0.2733, "levels": [ 0.75, 1.0, 0.25, 0.0, 0.0 ] }, "absent_object": { "score": 0.938, "ci": [ 0.845, 1.0 ], "acc": 0.95, "acc_ci": [ 0.875, 1.0 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.75 ] }, "ai_or_real": { "score": 0.05, "ci": [ 0.0, 0.35 ], "acc": 0.525, "acc_ci": [ 0.375, 0.675 ], "chance": 0.5, "levels": [ 0.625, 0.75, 0.625, 0.375, 0.25 ] }, "cable_scene": { "score": 0.5, "ci": [ 0.4, 0.575 ], "acc": 0.5, "acc_ci": [ 0.4, 0.575 ], "chance": 0.0, "levels": [ 0.75, 1.0, 0.75, 0.0, 0.0 ] }, "cant_tell_scene": { "score": 0.7063, "ci": [ 0.4965, 0.8741 ], "acc": 0.7812, "acc_ci": [ 0.625, 0.9062 ], "chance": 0.2552, "levels": [ 0.875, 0.875, 0.625, 0.75 ] }, "chart_screen": { "score": 0.4062, "ci": [ 0.3125, 0.5312 ], "acc": 0.4062, "acc_ci": [ 0.3125, 0.5312 ], "chance": 0.0, "levels": [ 1.0, 0.375, 0.25, 0.0 ] }, "coin_sum": { "score": 0.4, "ci": [ 0.3, 0.5 ], "acc": 0.4, "acc_ci": [ 0.3, 0.5 ], "chance": 0.0, "levels": [ 1.0, 0.625, 0.375, 0.0, 0.0 ] }, "count_scene": { "score": 0.125, "ci": [ 0.05, 0.225 ], "acc": 0.125, "acc_ci": [ 0.05, 0.225 ], "chance": 0.0, "levels": [ 0.375, 0.125, 0.125, 0.0, 0.0 ] }, "dashcam_light": { "score": 0.5125, "ci": [ 0.325, 0.7 ], "acc": 0.675, "acc_ci": [ 0.55, 0.8 ], "chance": 0.3333, "levels": [ 1.0, 0.875, 0.75, 0.5, 0.25 ] }, "depth_scene": { "score": 0.6, "ci": [ 0.4, 0.8 ], "acc": 0.8, "acc_ci": [ 0.7, 0.9 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.875, 0.625, 0.5 ] }, "eye_chart": { "score": 0.225, "ci": [ 0.125, 0.325 ], "acc": 0.225, "acc_ci": [ 0.125, 0.325 ], "chance": 0.0, "levels": [ 0.625, 0.375, 0.125, 0.0, 0.0 ] }, "facing_dir": { "score": 0.5294, "ci": [ 0.3824, 0.6765 ], "acc": 0.6, "acc_ci": [ 0.475, 0.725 ], "chance": 0.15, "levels": [ 1.0, 0.75, 0.625, 0.5, 0.125 ] }, "illusion": { "score": 0.175, "ci": [ 0.0, 0.4 ], "acc": 0.45, "acc_ci": [ 0.3, 0.6 ], "chance": 0.3333, "levels": [ 0.5, 0.375, 0.5, 0.625, 0.25 ] }, "invoice_check": { "score": 0.6788, "ci": [ 0.5985, 0.7859 ], "acc": 0.7, "acc_ci": [ 0.625, 0.8 ], "chance": 0.0659, "levels": [ 1.0, 1.0, 1.0, 0.25, 0.25 ] }, "label_scene": { "score": 0.6, "ci": [ 0.6, 0.6 ], "acc": 0.6, "acc_ci": [ 0.6, 0.6 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.0, 0.0 ] }, "meter_scene": { "score": 0.35, "ci": [ 0.25, 0.475 ], "acc": 0.35, "acc_ci": [ 0.25, 0.475 ], "chance": 0.0, "levels": [ 0.875, 0.25, 0.5, 0.125, 0.0 ] }, "missing_dim": { "score": 0.725, "ci": [ 0.65, 0.825 ], "acc": 0.725, "acc_ci": [ 0.65, 0.825 ], "chance": 0.0, "levels": [ 1.0, 0.5, 1.0, 1.0, 0.125 ] }, "pallet_count": { "score": 0.075, "ci": [ 0.0, 0.175 ], "acc": 0.075, "acc_ci": [ 0.0, 0.175 ], "chance": 0.0, "levels": [ 0.125, 0.0, 0.0, 0.125, 0.125 ] }, "parking_bay": { "score": 0.6916, "ci": [ 0.5795, 0.8038 ], "acc": 0.725, "acc_ci": [ 0.625, 0.825 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 0.75, 0.5, 0.375 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.1 ], "acc": 0.4, "acc_ci": [ 0.275, 0.55 ], "chance": 0.5, "levels": [ 0.5, 0.625, 0.125, 0.5, 0.25 ] }, "pipe_follow": { "score": 0.325, "ci": [ 0.2, 0.45 ], "acc": 0.325, "acc_ci": [ 0.2, 0.45 ], "chance": 0.0, "levels": [ 0.625, 0.5, 0.5, 0.0, 0.0 ] }, "render_bug": { "score": 0.34, "ci": [ 0.16, 0.52 ], "acc": 0.45, "acc_ci": [ 0.3, 0.6 ], "chance": 0.1667, "levels": [ 0.625, 0.625, 0.25, 0.375, 0.375 ] }, "shelf_plan": { "score": 0.55, "ci": [ 0.475, 0.6 ], "acc": 0.55, "acc_ci": [ 0.475, 0.6 ], "chance": 0.0, "levels": [ 1.0, 0.875, 0.875, 0.0, 0.0 ] }, "stack_scene": { "score": 0.1375, "ci": [ 0.0, 0.325 ], "acc": 0.425, "acc_ci": [ 0.3, 0.55 ], "chance": 0.3333, "levels": [ 0.875, 0.5, 0.375, 0.25, 0.125 ] }, "test_strips": { "score": 0.45, "ci": [ 0.35, 0.55 ], "acc": 0.45, "acc_ci": [ 0.35, 0.55 ], "chance": 0.0, "levels": [ 0.875, 0.875, 0.375, 0.125, 0.0 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 3.53, "cost_label": "$3.53", "runs_locally": false, "seconds_per_answer": 10.1, "open_weights": true }, { "model": "Step-5 (preview)", "maker": "StepFun", "id": "step-5-preview", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.4066, "overall_ci": [ 0.332, 0.5034 ], "categories": { "reading": { "score": 0.4188, "ci": [ 0.2748, 0.575 ], "partial": false }, "counting": { "score": 0.2, "ci": [ 0.0667, 0.4 ], "partial": false }, "space": { "score": 0.3243, "ci": [ 0.1814, 0.5424 ], "partial": false }, "changes": { "score": 0.4632, "ci": [ 0.2498, 0.6867 ], "partial": false }, "diagrams": { "score": 0.4667, "ci": [ 0.2, 0.7333 ], "partial": false }, "fooled": { "score": 0.5667, "ci": [ 0.3583, 0.8417 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 0.67, "chance": 0.25 }, "change_scene": { "score": 0.4, "chance": 0.0 }, "chart_screen": { "score": 0.75, "chance": 0.0 }, "clock_scene": { "score": 0.4, "chance": 0.0 }, "coin_sum": { "score": 0.2, "chance": 0.0 }, "conveyor_count": { "score": 0.0, "chance": 0.0 }, "count_scene": { "score": 0.2, "chance": 0.0 }, "dashcam_light": { "score": 0.4, "chance": 0.3333 }, "depth_scene": { "score": 0.6, "chance": 0.5 }, "eye_chart": { "score": 0.0, "chance": 0.0 }, "facing_dir": { "score": 0.29, "chance": 0.15 }, "illusion": { "score": 0.4, "chance": 0.3333 }, "invoice_check": { "score": 0.57, "chance": 0.0635 }, "label_scene": { "score": 0.6, "chance": 0.0 }, "match_render": { "score": 0.0, "chance": 0.25 }, "meter_scene": { "score": 0.4, "chance": 0.0 }, "missing_dim": { "score": 0.6, "chance": 0.0 }, "pallet_count": { "score": 0.4, "chance": 0.0 }, "parking_bay": { "score": 0.78, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.4, "chance": 0.0 }, "render_bug": { "score": 0.28, "chance": 0.1667 }, "shelf_plan": { "score": 0.6, "chance": 0.0 }, "shell_game": { "score": 0.5, "chance": 0.3333 }, "stack_scene": { "score": 0.1, "chance": 0.3333 }, "test_strips": { "score": 0.6, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 12.5, "cost_label": "$12.50", "runs_locally": false, "seconds_per_answer": 65.2, "open_weights": true }, { "model": "MiMo v2.6 Pro", "maker": "Xiaomi", "id": "mimo-v2.6-pro", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.4028, "overall_ci": [ 0.3864, 0.4261 ], "categories": { "reading": { "score": 0.3601, "ci": [ 0.3308, 0.3918 ], "partial": false }, "counting": { "score": 0.1949, "ci": [ 0.1487, 0.241 ], "partial": false }, "space": { "score": 0.3659, "ci": [ 0.3268, 0.4194 ], "partial": false }, "changes": { "score": 0.5794, "ci": [ 0.5326, 0.6263 ], "partial": false }, "diagrams": { "score": 0.4564, "ci": [ 0.4051, 0.5027 ], "partial": false }, "fooled": { "score": 0.4602, "ci": [ 0.4101, 0.538 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.0923, "ci": [ 0.0308, 0.1538 ], "acc": 0.0923, "acc_ci": [ 0.0308, 0.1538 ], "chance": 0.0, "levels": [ 0.2308, 0.0769, 0.1538, 0.0, 0.0 ] }, "change_scene": { "score": 0.5538, "ci": [ 0.4615, 0.6462 ], "acc": 0.5538, "acc_ci": [ 0.4615, 0.6462 ], "chance": 0.0, "levels": [ 0.9231, 0.8462, 0.6154, 0.2308, 0.1538 ] }, "conveyor_count": { "score": 0.1692, "ci": [ 0.1077, 0.2462 ], "acc": 0.1692, "acc_ci": [ 0.1077, 0.2462 ], "chance": 0.0, "levels": [ 0.6154, 0.0769, 0.0, 0.0769, 0.0769 ] }, "match_render": { "score": 0.3231, "ci": [ 0.2, 0.4462 ], "acc": 0.4923, "acc_ci": [ 0.4, 0.5846 ], "chance": 0.25, "levels": [ 0.8462, 0.7692, 0.6154, 0.2308, 0.0 ] }, "shell_game": { "score": 0.2802, "ci": [ 0.1743, 0.3649 ], "acc": 0.4769, "acc_ci": [ 0.4, 0.5385 ], "chance": 0.2733, "levels": [ 1.0, 0.8462, 0.4615, 0.0, 0.0769 ] }, "absent_object": { "score": 0.9237, "ci": [ 0.8665, 0.9809 ], "acc": 0.9385, "acc_ci": [ 0.8923, 0.9846 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 1.0, 0.6923 ] }, "ai_or_real": { "score": 0.0462, "ci": [ 0.0, 0.2923 ], "acc": 0.5231, "acc_ci": [ 0.4, 0.6462 ], "chance": 0.5, "levels": [ 0.6154, 0.6154, 0.6154, 0.3846, 0.3846 ] }, "cable_scene": { "score": 0.5077, "ci": [ 0.4462, 0.5846 ], "acc": 0.5077, "acc_ci": [ 0.4462, 0.5846 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.3846, 0.1538, 0.0 ] }, "cant_tell_scene": { "score": 0.794, "ci": [ 0.6395, 0.9227 ], "acc": 0.8462, "acc_ci": [ 0.7308, 0.9423 ], "chance": 0.2532, "levels": [ 0.8462, 0.8462, 0.8462, 0.8462 ] }, "chart_screen": { "score": 0.3654, "ci": [ 0.2692, 0.4615 ], "acc": 0.3654, "acc_ci": [ 0.2692, 0.4615 ], "chance": 0.0, "levels": [ 0.9231, 0.3077, 0.1538, 0.0769 ] }, "coin_sum": { "score": 0.3231, "ci": [ 0.2615, 0.3846 ], "acc": 0.3231, "acc_ci": [ 0.2615, 0.3846 ], "chance": 0.0, "levels": [ 1.0, 0.4615, 0.1538, 0.0, 0.0 ] }, "count_scene": { "score": 0.1846, "ci": [ 0.1077, 0.2615 ], "acc": 0.1846, "acc_ci": [ 0.1077, 0.2615 ], "chance": 0.0, "levels": [ 0.6154, 0.1538, 0.1538, 0.0, 0.0 ] }, "dashcam_light": { "score": 0.5154, "ci": [ 0.3769, 0.6538 ], "acc": 0.6769, "acc_ci": [ 0.5846, 0.7692 ], "chance": 0.3333, "levels": [ 1.0, 0.8462, 0.5385, 0.3846, 0.6154 ] }, "depth_scene": { "score": 0.6615, "ci": [ 0.5077, 0.8154 ], "acc": 0.8308, "acc_ci": [ 0.7538, 0.9077 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.8462, 0.4615, 0.8462 ] }, "eye_chart": { "score": 0.0769, "ci": [ 0.0308, 0.1231 ], "acc": 0.0769, "acc_ci": [ 0.0308, 0.1231 ], "chance": 0.0, "levels": [ 0.3846, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.2941, "ci": [ 0.1855, 0.4027 ], "acc": 0.4, "acc_ci": [ 0.3077, 0.4923 ], "chance": 0.15, "levels": [ 1.0, 0.3846, 0.3846, 0.1538, 0.0769 ] }, "illusion": { "score": 0.0769, "ci": [ 0.0, 0.2615 ], "acc": 0.3846, "acc_ci": [ 0.2769, 0.5077 ], "chance": 0.3333, "levels": [ 0.5385, 0.5385, 0.3077, 0.1538, 0.3846 ] }, "invoice_check": { "score": 0.8682, "ci": [ 0.7859, 0.9506 ], "acc": 0.8769, "acc_ci": [ 0.8, 0.9538 ], "chance": 0.066, "levels": [ 1.0, 0.9231, 1.0, 0.8462, 0.6154 ] }, "label_scene": { "score": 0.6923, "ci": [ 0.6308, 0.7538 ], "acc": 0.6923, "acc_ci": [ 0.6308, 0.7538 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.3846, 0.0769 ] }, "meter_scene": { "score": 0.3231, "ci": [ 0.2308, 0.4308 ], "acc": 0.3231, "acc_ci": [ 0.2308, 0.4308 ], "chance": 0.0, "levels": [ 0.6154, 0.3077, 0.5385, 0.1538, 0.0 ] }, "missing_dim": { "score": 0.5385, "ci": [ 0.4615, 0.6154 ], "acc": 0.5385, "acc_ci": [ 0.4615, 0.6154 ], "chance": 0.0, "levels": [ 1.0, 0.4615, 0.8462, 0.3846, 0.0 ] }, "pallet_count": { "score": 0.2308, "ci": [ 0.1385, 0.3231 ], "acc": 0.2308, "acc_ci": [ 0.1385, 0.3231 ], "chance": 0.0, "levels": [ 0.3846, 0.3077, 0.2308, 0.0769, 0.1538 ] }, "parking_bay": { "score": 0.8792, "ci": [ 0.8102, 0.9482 ], "acc": 0.8923, "acc_ci": [ 0.8308, 0.9538 ], "chance": 0.1083, "levels": [ 1.0, 1.0, 1.0, 0.7692, 0.6923 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.2308 ], "acc": 0.4923, "acc_ci": [ 0.3692, 0.6154 ], "chance": 0.5, "levels": [ 0.3846, 0.6154, 0.6154, 0.5385, 0.3077 ] }, "pipe_follow": { "score": 0.3231, "ci": [ 0.2308, 0.4154 ], "acc": 0.3231, "acc_ci": [ 0.2308, 0.4154 ], "chance": 0.0, "levels": [ 0.6154, 0.4615, 0.5385, 0.0, 0.0 ] }, "render_bug": { "score": 0.1877, "ci": [ 0.0585, 0.3169 ], "acc": 0.3231, "acc_ci": [ 0.2154, 0.4308 ], "chance": 0.1667, "levels": [ 0.5385, 0.3077, 0.3077, 0.2308, 0.2308 ] }, "shelf_plan": { "score": 0.7077, "ci": [ 0.6462, 0.7692 ], "acc": 0.7077, "acc_ci": [ 0.6462, 0.7692 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.4615, 0.0769 ] }, "stack_scene": { "score": 0.1231, "ci": [ 0.0, 0.3077 ], "acc": 0.4154, "acc_ci": [ 0.3077, 0.5385 ], "chance": 0.3333, "levels": [ 0.5385, 0.5385, 0.3846, 0.2308, 0.3846 ] }, "test_strips": { "score": 0.4923, "ci": [ 0.4154, 0.5846 ], "acc": 0.4923, "acc_ci": [ 0.4154, 0.5846 ], "chance": 0.0, "levels": [ 1.0, 0.8462, 0.3077, 0.2308, 0.0769 ] } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 4.6, "cost_label": "$4.60", "runs_locally": false, "seconds_per_answer": 54.7, "open_weights": true }, { "model": "Gemini 3.5 Flash-Lite", "maker": "Google", "id": "gemini-3.5-flash-lite", "access": "API", "type": "general", "group": "US labs", "effort": "default", "overall": 0.4027, "overall_ci": [ 0.3872, 0.4179 ], "categories": { "reading": { "score": 0.3968, "ci": [ 0.3728, 0.4214 ], "partial": false }, "counting": { "score": 0.2153, "ci": [ 0.1767, 0.2513 ], "partial": false }, "space": { "score": 0.3931, "ci": [ 0.3571, 0.4287 ], "partial": false }, "changes": { "score": 0.4415, "ci": [ 0.3992, 0.4817 ], "partial": false }, "diagrams": { "score": 0.4373, "ci": [ 0.3973, 0.4747 ], "partial": false }, "fooled": { "score": 0.5321, "ci": [ 0.4892, 0.5753 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.232, "ci": [ 0.168, 0.296 ], "acc": 0.232, "acc_ci": [ 0.168, 0.296 ], "chance": 0.0, "levels": [ 0.64, 0.28, 0.16, 0.0, 0.08 ] }, "change_scene": { "score": 0.322, "ci": [ 0.2417, 0.4027 ], "acc": 0.322, "acc_ci": [ 0.2417, 0.4027 ], "chance": 0.0, "levels": [ 0.4, 0.25, 0.28, 0.44, 0.24 ] }, "match_render": { "score": 0.536, "ci": [ 0.4391, 0.6342 ], "acc": 0.652, "acc_ci": [ 0.5793, 0.7257 ], "chance": 0.25, "levels": [ 0.92, 0.84, 0.72, 0.5, 0.28 ] }, "conveyor_count": { "score": 0.182, "ci": [ 0.1159, 0.25 ], "acc": 0.182, "acc_ci": [ 0.1159, 0.25 ], "chance": 0.0, "levels": [ 0.45, 0.25, 0.05, 0.04, 0.12 ] }, "shell_game": { "score": 0.2404, "ci": [ 0.1633, 0.3174 ], "acc": 0.448, "acc_ci": [ 0.392, 0.504 ], "chance": 0.2733, "levels": [ 0.95, 0.9, 0.15, 0.04, 0.2 ] }, "absent_object": { "score": 0.8634, "ci": [ 0.7864, 0.9313 ], "acc": 0.8898, "acc_ci": [ 0.8277, 0.9446 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 1.0, 0.68, 0.7692 ] }, "ai_or_real": { "score": 0.3733, "ci": [ 0.24, 0.5067 ], "acc": 0.6867, "acc_ci": [ 0.62, 0.7533 ], "chance": 0.5, "levels": [ 0.6667, 0.9333, 0.8667, 0.4333, 0.5333 ] }, "cable_scene": { "score": 0.496, "ci": [ 0.432, 0.56 ], "acc": 0.496, "acc_ci": [ 0.432, 0.56 ], "chance": 0.0, "levels": [ 0.92, 0.84, 0.44, 0.24, 0.04 ] }, "cant_tell_scene": { "score": 0.8917, "ci": [ 0.8107, 0.9464 ], "acc": 0.9192, "acc_ci": [ 0.8588, 0.96 ], "chance": 0.2538, "levels": [ 0.9167, 0.88, 0.88, 1.0 ] }, "chart_screen": { "score": 0.53, "ci": [ 0.45, 0.62 ], "acc": 0.53, "acc_ci": [ 0.45, 0.62 ], "chance": 0.0, "levels": [ 0.88, 0.52, 0.4, 0.32 ] }, "coin_sum": { "score": 0.2007, "ci": [ 0.1683, 0.2333 ], "acc": 0.2007, "acc_ci": [ 0.1683, 0.2333 ], "chance": 0.0, "levels": [ 0.92, 0.0833, 0.0, 0.0, 0.0 ] }, "count_scene": { "score": 0.16, "ci": [ 0.104, 0.216 ], "acc": 0.16, "acc_ci": [ 0.104, 0.216 ], "chance": 0.0, "levels": [ 0.56, 0.12, 0.08, 0.04, 0.0 ] }, "dashcam_light": { "score": 0.508, "ci": [ 0.412, 0.616 ], "acc": 0.672, "acc_ci": [ 0.608, 0.744 ], "chance": 0.3333, "levels": [ 1.0, 0.96, 0.56, 0.52, 0.32 ] }, "depth_scene": { "score": 0.474, "ci": [ 0.342, 0.6047 ], "acc": 0.737, "acc_ci": [ 0.671, 0.8023 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.56, 0.5417, 0.5833 ] }, "eye_chart": { "score": 0.04, "ci": [ 0.008, 0.072 ], "acc": 0.04, "acc_ci": [ 0.008, 0.072 ], "chance": 0.0, "levels": [ 0.12, 0.04, 0.04, 0.0, 0.0 ] }, "facing_dir": { "score": 0.4234, "ci": [ 0.3379, 0.5098 ], "acc": 0.5099, "acc_ci": [ 0.4372, 0.5833 ], "chance": 0.15, "levels": [ 0.8696, 0.6, 0.6, 0.36, 0.12 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.028 ], "acc": 0.272, "acc_ci": [ 0.2, 0.352 ], "chance": 0.3333, "levels": [ 0.28, 0.32, 0.28, 0.2, 0.28 ] }, "invoice_check": { "score": 0.5457, "ci": [ 0.4686, 0.6229 ], "acc": 0.576, "acc_ci": [ 0.504, 0.648 ], "chance": 0.0666, "levels": [ 0.84, 0.76, 0.72, 0.36, 0.2 ] }, "label_scene": { "score": 0.8, "ci": [ 0.744, 0.856 ], "acc": 0.8, "acc_ci": [ 0.744, 0.856 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.56, 0.44 ] }, "meter_scene": { "score": 0.376, "ci": [ 0.296, 0.456 ], "acc": 0.376, "acc_ci": [ 0.296, 0.456 ], "chance": 0.0, "levels": [ 0.72, 0.32, 0.32, 0.48, 0.04 ] }, "missing_dim": { "score": 0.624, "ci": [ 0.56, 0.68 ], "acc": 0.624, "acc_ci": [ 0.56, 0.68 ], "chance": 0.0, "levels": [ 1.0, 0.68, 0.76, 0.68, 0.0 ] }, "pallet_count": { "score": 0.304, "ci": [ 0.24, 0.368 ], "acc": 0.304, "acc_ci": [ 0.24, 0.368 ], "chance": 0.0, "levels": [ 0.76, 0.48, 0.24, 0.0, 0.04 ] }, "parking_bay": { "score": 0.7578, "ci": [ 0.686, 0.8295 ], "acc": 0.784, "acc_ci": [ 0.72, 0.848 ], "chance": 0.1083, "levels": [ 1.0, 0.84, 0.84, 0.76, 0.48 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.0813 ], "acc": 0.4517, "acc_ci": [ 0.364, 0.5407 ], "chance": 0.5, "levels": [ 0.44, 0.44, 0.48, 0.4583, 0.44 ] }, "pipe_follow": { "score": 0.192, "ci": [ 0.128, 0.256 ], "acc": 0.192, "acc_ci": [ 0.128, 0.256 ], "chance": 0.0, "levels": [ 0.32, 0.32, 0.24, 0.08, 0.0 ] }, "render_bug": { "score": 0.3944, "ci": [ 0.2955, 0.4982 ], "acc": 0.4954, "acc_ci": [ 0.413, 0.5818 ], "chance": 0.1667, "levels": [ 0.8, 0.4583, 0.4545, 0.4783, 0.2857 ] }, "shelf_plan": { "score": 0.504, "ci": [ 0.4398, 0.568 ], "acc": 0.504, "acc_ci": [ 0.4398, 0.568 ], "chance": 0.0, "levels": [ 0.88, 0.8, 0.64, 0.16, 0.04 ] }, "stack_scene": { "score": 0.3204, "ci": [ 0.1871, 0.4494 ], "acc": 0.5469, "acc_ci": [ 0.4581, 0.633 ], "chance": 0.3333, "levels": [ 0.68, 0.6, 0.5417, 0.4545, 0.4583 ] }, "test_strips": { "score": 0.488, "ci": [ 0.424, 0.56 ], "acc": 0.488, "acc_ci": [ 0.424, 0.56 ], "chance": 0.0, "levels": [ 1.0, 0.68, 0.36, 0.24, 0.16 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 0.34, "cost_label": "$0.34", "runs_locally": false, "seconds_per_answer": 2.6, "open_weights": false }, { "model": "Gemma 4 31B", "maker": "Google", "id": "gemma-4-31b-it", "access": "API", "type": "general", "group": "Open models via API", "effort": "default", "overall": 0.3877, "overall_ci": [ 0.3338, 0.466 ], "categories": { "reading": { "score": 0.3625, "ci": [ 0.2188, 0.525 ], "partial": false }, "counting": { "score": 0.0, "ci": [ 0.0, 0.0 ], "partial": false }, "space": { "score": 0.312, "ci": [ 0.1967, 0.511 ], "partial": false }, "changes": { "score": 0.4765, "ci": [ 0.2998, 0.6734 ], "partial": false }, "diagrams": { "score": 0.4, "ci": [ 0.1333, 0.6667 ], "partial": false }, "fooled": { "score": 0.775, "ci": [ 0.75, 0.925 ], "partial": false } }, "tests": { "change_scene": { "score": 0.8, "chance": 0.0 }, "conveyor_count": { "score": 0.0, "chance": 0.0 }, "match_render": { "score": 0.0, "chance": 0.25 }, "shell_game": { "score": 0.45, "chance": 0.2733 }, "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 1.0, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.25 }, "chart_screen": { "score": 0.5, "chance": 0.0 }, "clock_scene": { "score": 0.2, "chance": 0.0 }, "coin_sum": { "score": 0.2, "chance": 0.0 }, "count_scene": { "score": 0.0, "chance": 0.0 }, "dashcam_light": { "score": 0.4, "chance": 0.3333 }, "depth_scene": { "score": 0.2, "chance": 0.5 }, "eye_chart": { "score": 0.2, "chance": 0.0 }, "facing_dir": { "score": 0.06, "chance": 0.15 }, "illusion": { "score": 0.1, "chance": 0.3333 }, "invoice_check": { "score": 0.15, "chance": 0.0635 }, "label_scene": { "score": 0.6, "chance": 0.0 }, "meter_scene": { "score": 0.4, "chance": 0.0 }, "missing_dim": { "score": 0.4, "chance": 0.0 }, "pallet_count": { "score": 0.0, "chance": 0.0 }, "parking_bay": { "score": 0.78, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.4, "chance": 0.0 }, "render_bug": { "score": 0.76, "chance": 0.1667 }, "shelf_plan": { "score": 0.2, "chance": 0.0 }, "stack_scene": { "score": 0.7, "chance": 0.3333 }, "test_strips": { "score": 0.4, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 1.94, "cost_label": "$1.94", "runs_locally": false, "seconds_per_answer": 2.0, "open_weights": true }, { "model": "Seed 2.1 Turbo", "maker": "ByteDance", "id": "seed-2-1-turbo", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.3832, "overall_ci": [ 0.3232, 0.465 ], "categories": { "reading": { "score": 0.2687, "ci": [ 0.1562, 0.4125 ], "partial": false }, "counting": { "score": 0.2667, "ci": [ 0.1333, 0.3333 ], "partial": false }, "space": { "score": 0.1622, "ci": [ 0.0623, 0.3672 ], "partial": false }, "changes": { "score": 0.4598, "ci": [ 0.25, 0.6766 ], "partial": false }, "diagrams": { "score": 0.4667, "ci": [ 0.2667, 0.6667 ], "partial": false }, "fooled": { "score": 0.675, "ci": [ 0.525, 0.925 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.6, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 1.0, "chance": 0.25 }, "change_scene": { "score": 0.8, "chance": 0.0 }, "chart_screen": { "score": 0.25, "chance": 0.0 }, "clock_scene": { "score": 0.0, "chance": 0.0 }, "coin_sum": { "score": 0.2, "chance": 0.0 }, "conveyor_count": { "score": 0.0, "chance": 0.0 }, "count_scene": { "score": 0.8, "chance": 0.0 }, "dashcam_light": { "score": 0.1, "chance": 0.3333 }, "depth_scene": { "score": 0.2, "chance": 0.5 }, "eye_chart": { "score": 0.4, "chance": 0.0 }, "facing_dir": { "score": 0.06, "chance": 0.15 }, "illusion": { "score": 0.1, "chance": 0.3333 }, "invoice_check": { "score": 0.36, "chance": 0.0635 }, "label_scene": { "score": 0.6, "chance": 0.0 }, "match_render": { "score": 0.0, "chance": 0.25 }, "meter_scene": { "score": 0.4, "chance": 0.0 }, "missing_dim": { "score": 0.8, "chance": 0.0 }, "pallet_count": { "score": 0.0, "chance": 0.0 }, "parking_bay": { "score": 0.33, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.2, "chance": 0.0 }, "render_bug": { "score": 0.28, "chance": 0.1667 }, "shelf_plan": { "score": 0.4, "chance": 0.0 }, "shell_game": { "score": 0.45, "chance": 0.2733 }, "stack_scene": { "score": 0.1, "chance": 0.3333 }, "test_strips": { "score": 0.2, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 10.85, "cost_label": "$10.85", "runs_locally": false, "seconds_per_answer": 229.5, "open_weights": false }, { "model": "Hunyuan HY4", "maker": "Tencent", "id": "hy4", "access": "API", "type": "general", "group": "Chinese labs", "effort": "default", "overall": 0.3792, "overall_ci": [ 0.3116, 0.4652 ], "categories": { "reading": { "score": 0.3312, "ci": [ 0.1936, 0.4813 ], "partial": false }, "counting": { "score": 0.0667, "ci": [ 0.0, 0.2 ], "partial": false }, "space": { "score": 0.2421, "ci": [ 0.111, 0.4329 ], "partial": false }, "changes": { "score": 0.5266, "ci": [ 0.3132, 0.7366 ], "partial": false }, "diagrams": { "score": 0.4667, "ci": [ 0.2667, 0.6667 ], "partial": false }, "fooled": { "score": 0.6417, "ci": [ 0.4333, 0.9167 ], "partial": false } }, "tests": { "absent_object": { "score": 1.0, "chance": 0.1933 }, "ai_or_real": { "score": 0.2, "chance": 0.5 }, "cable_scene": { "score": 0.4, "chance": 0.0 }, "cant_tell_scene": { "score": 0.67, "chance": 0.25 }, "change_scene": { "score": 0.2, "chance": 0.0 }, "chart_screen": { "score": 0.25, "chance": 0.0 }, "clock_scene": { "score": 0.4, "chance": 0.0 }, "coin_sum": { "score": 0.2, "chance": 0.0 }, "conveyor_count": { "score": 0.2, "chance": 0.0 }, "count_scene": { "score": 0.0, "chance": 0.0 }, "dashcam_light": { "score": 0.4, "chance": 0.3333 }, "depth_scene": { "score": 0.2, "chance": 0.5 }, "eye_chart": { "score": 0.4, "chance": 0.0 }, "facing_dir": { "score": 0.29, "chance": 0.15 }, "illusion": { "score": 0.7, "chance": 0.3333 }, "invoice_check": { "score": 0.79, "chance": 0.0635 }, "label_scene": { "score": 0.6, "chance": 0.0 }, "match_render": { "score": 0.2, "chance": 0.25 }, "meter_scene": { "score": 0.4, "chance": 0.0 }, "missing_dim": { "score": 0.8, "chance": 0.0 }, "pallet_count": { "score": 0.0, "chance": 0.0 }, "parking_bay": { "score": 0.55, "chance": 0.1083 }, "part_mirror": { "score": 0.0, "chance": 0.5 }, "pipe_follow": { "score": 0.2, "chance": 0.0 }, "render_bug": { "score": 0.52, "chance": 0.1667 }, "shelf_plan": { "score": 0.6, "chance": 0.0 }, "shell_game": { "score": 0.45, "chance": 0.2733 }, "stack_scene": { "score": 0.0, "chance": 0.3333 }, "test_strips": { "score": 0.0, "chance": 0.0 } }, "unsupported_tests": [], "tested": "2026-11-07", "cost_per_1000": 5.71, "cost_label": "$5.71", "runs_locally": false, "seconds_per_answer": 170.6, "open_weights": false }, { "model": "Qwen3.5-9B", "maker": "Alibaba", "id": "qwen3.5-9b-nothink", "access": "runs locally", "type": "local open model", "group": "Local · open weights", "effort": "no thinking", "overall": 0.2949, "overall_ci": [ 0.2801, 0.3106 ], "categories": { "reading": { "score": 0.3648, "ci": [ 0.3425, 0.3872 ], "partial": false }, "counting": { "score": 0.192, "ci": [ 0.1587, 0.222 ], "partial": false }, "space": { "score": 0.268, "ci": [ 0.2293, 0.3109 ], "partial": false }, "changes": { "score": 0.2369, "ci": [ 0.1986, 0.2754 ], "partial": false }, "diagrams": { "score": 0.2507, "ci": [ 0.2187, 0.2827 ], "partial": false }, "fooled": { "score": 0.4567, "ci": [ 0.4095, 0.5117 ], "partial": false } }, "tests": { "cant_tell_scene": { "score": 0.7335, "ci": [ 0.6196, 0.8423 ], "acc": 0.8011, "acc_ci": [ 0.7162, 0.8824 ], "chance": 0.2538, "levels": [ 0.7917, 0.84, 0.8, 0.7727 ] }, "ai_or_real": { "score": 0.2, "ci": [ 0.04, 0.3467 ], "acc": 0.6, "acc_ci": [ 0.52, 0.6733 ], "chance": 0.5, "levels": [ 0.5333, 0.7667, 0.7, 0.5333, 0.4667 ] }, "cable_scene": { "score": 0.352, "ci": [ 0.288, 0.416 ], "acc": 0.352, "acc_ci": [ 0.288, 0.416 ], "chance": 0.0, "levels": [ 0.76, 0.6, 0.32, 0.08, 0.0 ] }, "change_scene": { "score": 0.477, "ci": [ 0.396, 0.558 ], "acc": 0.477, "acc_ci": [ 0.396, 0.558 ], "chance": 0.0, "levels": [ 0.72, 0.625, 0.48, 0.24, 0.32 ] }, "chart_screen": { "score": 0.47, "ci": [ 0.39, 0.55 ], "acc": 0.47, "acc_ci": [ 0.39, 0.55 ], "chance": 0.0, "levels": [ 0.92, 0.56, 0.24, 0.16 ] }, "clock_scene": { "score": 0.496, "ci": [ 0.448, 0.536 ], "acc": 0.496, "acc_ci": [ 0.448, 0.536 ], "chance": 0.0, "levels": [ 1.0, 0.56, 0.92, 0.0, 0.0 ] }, "conveyor_count": { "score": 0.28, "ci": [ 0.23, 0.34 ], "acc": 0.28, "acc_ci": [ 0.23, 0.34 ], "chance": 0.0, "levels": [ 0.9, 0.35, 0.15, 0.0, 0.0 ] }, "count_scene": { "score": 0.192, "ci": [ 0.136, 0.24 ], "acc": 0.192, "acc_ci": [ 0.136, 0.24 ], "chance": 0.0, "levels": [ 0.6, 0.16, 0.16, 0.04, 0.0 ] }, "depth_scene": { "score": 0.602, "ci": [ 0.486, 0.718 ], "acc": 0.801, "acc_ci": [ 0.743, 0.859 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.88, 0.5, 0.625 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.1 ], "acc": 0.32, "acc_ci": [ 0.24, 0.4 ], "chance": 0.3333, "levels": [ 0.32, 0.28, 0.24, 0.32, 0.44 ] }, "label_scene": { "score": 0.68, "ci": [ 0.624, 0.736 ], "acc": 0.68, "acc_ci": [ 0.624, 0.736 ], "chance": 0.0, "levels": [ 1.0, 0.88, 1.0, 0.4, 0.12 ] }, "meter_scene": { "score": 0.376, "ci": [ 0.304, 0.448 ], "acc": 0.376, "acc_ci": [ 0.304, 0.448 ], "chance": 0.0, "levels": [ 0.8, 0.32, 0.44, 0.32, 0.0 ] }, "part_mirror": { "score": 0.13, "ci": [ 0.0, 0.2936 ], "acc": 0.565, "acc_ci": [ 0.4763, 0.6468 ], "chance": 0.5, "levels": [ 0.56, 0.56, 0.56, 0.625, 0.52 ] }, "shell_game": { "score": 0.0, "ci": [ 0.0, 0.0752 ], "acc": 0.25, "acc_ci": [ 0.17, 0.328 ], "chance": 0.2733, "levels": [ 0.35, 0.35, 0.35, 0.16, 0.04 ] }, "stack_scene": { "score": 0.1024, "ci": [ 0.0, 0.2261 ], "acc": 0.4016, "acc_ci": [ 0.3144, 0.4841 ], "chance": 0.3333, "levels": [ 0.36, 0.36, 0.5, 0.4545, 0.3333 ] }, "absent_object": { "score": 0.8932, "ci": [ 0.8245, 0.9603 ], "acc": 0.9138, "acc_ci": [ 0.8585, 0.968 ], "chance": 0.1933, "levels": [ 0.96, 1.0, 1.0, 0.84, 0.7692 ] }, "match_render": { "score": 0.5027, "ci": [ 0.4156, 0.6004 ], "acc": 0.627, "acc_ci": [ 0.5617, 0.7003 ], "chance": 0.25, "levels": [ 0.8, 0.88, 0.84, 0.375, 0.24 ] }, "dashcam_light": { "score": 0.496, "ci": [ 0.4, 0.592 ], "acc": 0.664, "acc_ci": [ 0.6, 0.728 ], "chance": 0.3333, "levels": [ 1.0, 1.0, 0.6, 0.36, 0.36 ] }, "invoice_check": { "score": 0.0829, "ci": [ 0.0315, 0.1429 ], "acc": 0.144, "acc_ci": [ 0.096, 0.2 ], "chance": 0.0666, "levels": [ 0.48, 0.04, 0.04, 0.04, 0.12 ] }, "pipe_follow": { "score": 0.064, "ci": [ 0.024, 0.112 ], "acc": 0.064, "acc_ci": [ 0.024, 0.112 ], "chance": 0.0, "levels": [ 0.2, 0.04, 0.04, 0.04, 0.0 ] }, "parking_bay": { "score": 0.3092, "ci": [ 0.2462, 0.381 ], "acc": 0.384, "acc_ci": [ 0.3278, 0.448 ], "chance": 0.1083, "levels": [ 0.88, 0.52, 0.52, 0.0, 0.0 ] }, "facing_dir": { "score": 0.2295, "ci": [ 0.1353, 0.326 ], "acc": 0.345, "acc_ci": [ 0.265, 0.4271 ], "chance": 0.15, "levels": [ 0.5652, 0.32, 0.4, 0.32, 0.12 ] }, "eye_chart": { "score": 0.032, "ci": [ 0.008, 0.064 ], "acc": 0.032, "acc_ci": [ 0.008, 0.064 ], "chance": 0.0, "levels": [ 0.12, 0.04, 0.0, 0.0, 0.0 ] }, "render_bug": { "score": 0.3078, "ci": [ 0.2032, 0.4101 ], "acc": 0.4232, "acc_ci": [ 0.336, 0.5084 ], "chance": 0.1667, "levels": [ 0.48, 0.5, 0.3636, 0.3913, 0.381 ] }, "shelf_plan": { "score": 0.08, "ci": [ 0.032, 0.128 ], "acc": 0.08, "acc_ci": [ 0.032, 0.128 ], "chance": 0.0, "levels": [ 0.12, 0.12, 0.08, 0.08, 0.0 ] }, "missing_dim": { "score": 0.336, "ci": [ 0.28, 0.392 ], "acc": 0.336, "acc_ci": [ 0.28, 0.392 ], "chance": 0.0, "levels": [ 0.96, 0.4, 0.32, 0.0, 0.0 ] }, "test_strips": { "score": 0.328, "ci": [ 0.272, 0.384 ], "acc": 0.328, "acc_ci": [ 0.272, 0.384 ], "chance": 0.0, "levels": [ 1.0, 0.16, 0.16, 0.12, 0.2 ] }, "coin_sum": { "score": 0.04, "ci": [ 0.008, 0.072 ], "acc": 0.04, "acc_ci": [ 0.008, 0.072 ], "chance": 0.0, "levels": [ 0.2, 0.0, 0.0, 0.0, 0.0 ] }, "pallet_count": { "score": 0.104, "ci": [ 0.056, 0.1522 ], "acc": 0.104, "acc_ci": [ 0.056, 0.1522 ], "chance": 0.0, "levels": [ 0.2, 0.12, 0.04, 0.0, 0.16 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 0, "cost_label": "$0 (local)", "runs_locally": true, "seconds_per_answer": 1.5, "open_weights": true }, { "model": "Perceptron Mk1.5", "maker": "Perceptron", "id": "perceptron-mk1.5", "access": "API", "type": "general", "group": "Others", "effort": "default", "overall": 0.2881, "overall_ci": [ 0.274, 0.3042 ], "categories": { "reading": { "score": 0.3947, "ci": [ 0.3723, 0.418 ], "partial": false }, "counting": { "score": 0.108, "ci": [ 0.076, 0.14 ], "partial": true }, "space": { "score": 0.3583, "ci": [ 0.3205, 0.401 ], "partial": true }, "changes": { "score": 0.1584, "ci": [ 0.1256, 0.1955 ], "partial": false }, "diagrams": { "score": 0.2373, "ci": [ 0.2027, 0.2693 ], "partial": false }, "fooled": { "score": 0.4717, "ci": [ 0.4237, 0.521 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.512, "ci": [ 0.464, 0.552 ], "acc": 0.512, "acc_ci": [ 0.464, 0.552 ], "chance": 0.0, "levels": [ 1.0, 0.48, 1.0, 0.0, 0.08 ] }, "change_scene": { "score": 0.3227, "ci": [ 0.2577, 0.3947 ], "acc": 0.3227, "acc_ci": [ 0.2577, 0.3947 ], "chance": 0.0, "levels": [ 0.84, 0.3333, 0.12, 0.08, 0.24 ] }, "match_render": { "score": 0.5587, "ci": [ 0.4609, 0.6469 ], "acc": 0.669, "acc_ci": [ 0.5957, 0.7351 ], "chance": 0.25, "levels": [ 0.96, 0.8, 0.68, 0.625, 0.28 ] }, "absent_object": { "score": 0.9138, "ci": [ 0.8467, 0.9802 ], "acc": 0.9305, "acc_ci": [ 0.8763, 0.984 ], "chance": 0.1933, "levels": [ 0.96, 1.0, 1.0, 1.0, 0.6923 ] }, "ai_or_real": { "score": 0.2133, "ci": [ 0.0667, 0.3733 ], "acc": 0.6067, "acc_ci": [ 0.5333, 0.6867 ], "chance": 0.5, "levels": [ 0.5333, 0.8333, 0.7, 0.4667, 0.5 ] }, "cable_scene": { "score": 0.344, "ci": [ 0.272, 0.408 ], "acc": 0.344, "acc_ci": [ 0.272, 0.408 ], "chance": 0.0, "levels": [ 0.68, 0.68, 0.24, 0.12, 0.0 ] }, "cant_tell_scene": { "score": 0.7597, "ci": [ 0.6558, 0.8583 ], "acc": 0.8207, "acc_ci": [ 0.7431, 0.8942 ], "chance": 0.2538, "levels": [ 0.75, 0.88, 0.88, 0.7727 ] }, "chart_screen": { "score": 0.51, "ci": [ 0.43, 0.59 ], "acc": 0.51, "acc_ci": [ 0.43, 0.59 ], "chance": 0.0, "levels": [ 1.0, 0.52, 0.28, 0.24 ] }, "coin_sum": { "score": 0.04, "ci": [ 0.008, 0.072 ], "acc": 0.04, "acc_ci": [ 0.008, 0.072 ], "chance": 0.0, "levels": [ 0.2, 0.0, 0.0, 0.0, 0.0 ] }, "count_scene": { "score": 0.136, "ci": [ 0.088, 0.184 ], "acc": 0.136, "acc_ci": [ 0.088, 0.184 ], "chance": 0.0, "levels": [ 0.52, 0.0, 0.08, 0.04, 0.04 ] }, "dashcam_light": { "score": 0.544, "ci": [ 0.448, 0.652 ], "acc": 0.696, "acc_ci": [ 0.632, 0.768 ], "chance": 0.3333, "levels": [ 1.0, 0.92, 0.8, 0.44, 0.32 ] }, "depth_scene": { "score": 0.618, "ci": [ 0.5007, 0.7347 ], "acc": 0.809, "acc_ci": [ 0.7503, 0.8673 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.92, 0.5, 0.625 ] }, "eye_chart": { "score": 0.008, "ci": [ 0.0, 0.024 ], "acc": 0.008, "acc_ci": [ 0.0, 0.024 ], "chance": 0.0, "levels": [ 0.04, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.2696, "ci": [ 0.1816, 0.3567 ], "acc": 0.3791, "acc_ci": [ 0.3043, 0.4532 ], "chance": 0.15, "levels": [ 0.6957, 0.52, 0.44, 0.12, 0.12 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.028 ], "acc": 0.272, "acc_ci": [ 0.192, 0.352 ], "chance": 0.3333, "levels": [ 0.24, 0.32, 0.32, 0.2, 0.28 ] }, "invoice_check": { "score": 0.0572, "ci": [ 0.0058, 0.1258 ], "acc": 0.12, "acc_ci": [ 0.072, 0.184 ], "chance": 0.0666, "levels": [ 0.24, 0.08, 0.16, 0.04, 0.08 ] }, "label_scene": { "score": 0.688, "ci": [ 0.64, 0.736 ], "acc": 0.688, "acc_ci": [ 0.64, 0.736 ], "chance": 0.0, "levels": [ 1.0, 0.96, 1.0, 0.32, 0.16 ] }, "meter_scene": { "score": 0.448, "ci": [ 0.384, 0.52 ], "acc": 0.448, "acc_ci": [ 0.384, 0.52 ], "chance": 0.0, "levels": [ 0.84, 0.28, 0.52, 0.6, 0.0 ] }, "missing_dim": { "score": 0.232, "ci": [ 0.184, 0.28 ], "acc": 0.232, "acc_ci": [ 0.184, 0.28 ], "chance": 0.0, "levels": [ 0.88, 0.08, 0.16, 0.04, 0.0 ] }, "pallet_count": { "score": 0.08, "ci": [ 0.04, 0.128 ], "acc": 0.08, "acc_ci": [ 0.04, 0.128 ], "chance": 0.0, "levels": [ 0.0, 0.2, 0.08, 0.04, 0.08 ] }, "parking_bay": { "score": 0.4707, "ci": [ 0.3989, 0.5425 ], "acc": 0.528, "acc_ci": [ 0.464, 0.592 ], "chance": 0.1083, "levels": [ 0.76, 0.8, 0.8, 0.24, 0.04 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.128 ], "acc": 0.4757, "acc_ci": [ 0.3943, 0.564 ], "chance": 0.5, "levels": [ 0.48, 0.48, 0.48, 0.4583, 0.48 ] }, "pipe_follow": { "score": 0.136, "ci": [ 0.08, 0.192 ], "acc": 0.136, "acc_ci": [ 0.08, 0.192 ], "chance": 0.0, "levels": [ 0.24, 0.2, 0.2, 0.04, 0.0 ] }, "render_bug": { "score": 0.0696, "ci": [ 0.0, 0.1652 ], "acc": 0.2247, "acc_ci": [ 0.1479, 0.3044 ], "chance": 0.1667, "levels": [ 0.28, 0.2083, 0.2273, 0.2174, 0.1905 ] }, "shelf_plan": { "score": 0.184, "ci": [ 0.12, 0.248 ], "acc": 0.184, "acc_ci": [ 0.12, 0.248 ], "chance": 0.0, "levels": [ 0.4, 0.16, 0.28, 0.04, 0.04 ] }, "stack_scene": { "score": 0.2325, "ci": [ 0.1051, 0.36 ], "acc": 0.4884, "acc_ci": [ 0.4034, 0.5733 ], "chance": 0.3333, "levels": [ 0.36, 0.4, 0.5, 0.6818, 0.5 ] }, "test_strips": { "score": 0.408, "ci": [ 0.336, 0.48 ], "acc": 0.408, "acc_ci": [ 0.336, 0.48 ], "chance": 0.0, "levels": [ 0.96, 0.36, 0.36, 0.2, 0.16 ] } }, "unsupported_tests": [ "conveyor_count", "shell_game" ], "tested": "2026-11-06", "cost_per_1000": 0.2, "cost_label": "$0.20", "runs_locally": false, "seconds_per_answer": 7.5, "open_weights": false }, { "model": "Qwen3-VL-8B", "maker": "Alibaba", "id": "qwen3-vl-8b", "access": "runs locally", "type": "local open model", "group": "Local · open weights", "effort": "default", "overall": 0.2602, "overall_ci": [ 0.2479, 0.2761 ], "categories": { "reading": { "score": 0.327, "ci": [ 0.3042, 0.3508 ], "partial": false }, "counting": { "score": 0.1753, "ci": [ 0.1413, 0.2107 ], "partial": false }, "space": { "score": 0.2235, "ci": [ 0.1933, 0.2665 ], "partial": false }, "changes": { "score": 0.1847, "ci": [ 0.1479, 0.2227 ], "partial": false }, "diagrams": { "score": 0.1867, "ci": [ 0.1547, 0.2187 ], "partial": false }, "fooled": { "score": 0.4638, "ci": [ 0.4239, 0.5113 ], "partial": false } }, "tests": { "cant_tell_scene": { "score": 0.8588, "ci": [ 0.7713, 0.9416 ], "acc": 0.8947, "acc_ci": [ 0.8293, 0.9564 ], "chance": 0.2538, "levels": [ 0.875, 0.96, 0.88, 0.8636 ] }, "clock_scene": { "score": 0.408, "ci": [ 0.352, 0.464 ], "acc": 0.408, "acc_ci": [ 0.352, 0.464 ], "chance": 0.0, "levels": [ 0.88, 0.4, 0.76, 0.0, 0.0 ] }, "coin_sum": { "score": 0.048, "ci": [ 0.016, 0.08 ], "acc": 0.048, "acc_ci": [ 0.016, 0.08 ], "chance": 0.0, "levels": [ 0.24, 0.0, 0.0, 0.0, 0.0 ] }, "conveyor_count": { "score": 0.23, "ci": [ 0.156, 0.308 ], "acc": 0.23, "acc_ci": [ 0.156, 0.308 ], "chance": 0.0, "levels": [ 0.4, 0.15, 0.2, 0.24, 0.16 ] }, "dashcam_light": { "score": 0.412, "ci": [ 0.304, 0.52 ], "acc": 0.608, "acc_ci": [ 0.536, 0.68 ], "chance": 0.3333, "levels": [ 0.84, 0.92, 0.52, 0.36, 0.4 ] }, "depth_scene": { "score": 0.6527, "ci": [ 0.536, 0.768 ], "acc": 0.8263, "acc_ci": [ 0.768, 0.884 ], "chance": 0.5, "levels": [ 1.0, 1.0, 0.84, 0.5417, 0.75 ] }, "eye_chart": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.2053, "ci": [ 0.1268, 0.2908 ], "acc": 0.3245, "acc_ci": [ 0.2577, 0.3972 ], "chance": 0.15, "levels": [ 0.7826, 0.32, 0.2, 0.16, 0.16 ] }, "invoice_check": { "score": 0.04, "ci": [ 0.0, 0.0915 ], "acc": 0.104, "acc_ci": [ 0.056, 0.152 ], "chance": 0.0666, "levels": [ 0.32, 0.08, 0.08, 0.0, 0.04 ] }, "label_scene": { "score": 0.72, "ci": [ 0.672, 0.776 ], "acc": 0.72, "acc_ci": [ 0.672, 0.776 ], "chance": 0.0, "levels": [ 1.0, 1.0, 1.0, 0.4, 0.2 ] }, "meter_scene": { "score": 0.272, "ci": [ 0.2, 0.344 ], "acc": 0.272, "acc_ci": [ 0.2, 0.344 ], "chance": 0.0, "levels": [ 0.44, 0.24, 0.48, 0.2, 0.0 ] }, "missing_dim": { "score": 0.136, "ci": [ 0.088, 0.192 ], "acc": 0.136, "acc_ci": [ 0.088, 0.192 ], "chance": 0.0, "levels": [ 0.44, 0.2, 0.04, 0.0, 0.0 ] }, "pallet_count": { "score": 0.184, "ci": [ 0.136, 0.232 ], "acc": 0.184, "acc_ci": [ 0.136, 0.232 ], "chance": 0.0, "levels": [ 0.68, 0.24, 0.0, 0.0, 0.0 ] }, "parking_bay": { "score": 0.1388, "ci": [ 0.058, 0.2105 ], "acc": 0.232, "acc_ci": [ 0.16, 0.296 ], "chance": 0.1083, "levels": [ 0.2, 0.48, 0.32, 0.16, 0.0 ] }, "pipe_follow": { "score": 0.104, "ci": [ 0.056, 0.152 ], "acc": 0.104, "acc_ci": [ 0.056, 0.152 ], "chance": 0.0, "levels": [ 0.32, 0.2, 0.0, 0.0, 0.0 ] }, "render_bug": { "score": 0.2312, "ci": [ 0.1307, 0.3346 ], "acc": 0.3594, "acc_ci": [ 0.2756, 0.4455 ], "chance": 0.1667, "levels": [ 0.28, 0.3333, 0.3636, 0.3913, 0.4286 ] }, "shelf_plan": { "score": 0.112, "ci": [ 0.064, 0.168 ], "acc": 0.112, "acc_ci": [ 0.064, 0.168 ], "chance": 0.0, "levels": [ 0.24, 0.12, 0.16, 0.04, 0.0 ] }, "shell_game": { "score": 0.1688, "ci": [ 0.045, 0.2845 ], "acc": 0.396, "acc_ci": [ 0.306, 0.48 ], "chance": 0.2733, "levels": [ 0.6, 0.4, 0.5, 0.2, 0.28 ] }, "test_strips": { "score": 0.296, "ci": [ 0.224, 0.368 ], "acc": 0.296, "acc_ci": [ 0.224, 0.368 ], "chance": 0.0, "levels": [ 0.52, 0.36, 0.24, 0.2, 0.16 ] }, "ai_or_real": { "score": 0.0933, "ci": [ 0.0, 0.24 ], "acc": 0.5467, "acc_ci": [ 0.4667, 0.62 ], "chance": 0.5, "levels": [ 0.5, 0.7333, 0.6333, 0.4333, 0.4333 ] }, "cable_scene": { "score": 0.32, "ci": [ 0.256, 0.384 ], "acc": 0.32, "acc_ci": [ 0.256, 0.384 ], "chance": 0.0, "levels": [ 0.68, 0.56, 0.28, 0.08, 0.0 ] }, "change_scene": { "score": 0.3557, "ci": [ 0.2747, 0.4367 ], "acc": 0.3557, "acc_ci": [ 0.2747, 0.4367 ], "chance": 0.0, "levels": [ 0.32, 0.4583, 0.4, 0.44, 0.16 ] }, "chart_screen": { "score": 0.46, "ci": [ 0.38, 0.54 ], "acc": 0.46, "acc_ci": [ 0.38, 0.54 ], "chance": 0.0, "levels": [ 1.0, 0.4, 0.16, 0.28 ] }, "count_scene": { "score": 0.112, "ci": [ 0.064, 0.168 ], "acc": 0.112, "acc_ci": [ 0.064, 0.168 ], "chance": 0.0, "levels": [ 0.28, 0.12, 0.04, 0.12, 0.0 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.028 ], "acc": 0.272, "acc_ci": [ 0.2, 0.352 ], "chance": 0.3333, "levels": [ 0.28, 0.32, 0.28, 0.2, 0.28 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.1447 ], "acc": 0.4847, "acc_ci": [ 0.397, 0.5723 ], "chance": 0.5, "levels": [ 0.52, 0.44, 0.36, 0.5833, 0.52 ] }, "stack_scene": { "score": 0.0147, "ci": [ 0.0, 0.1461 ], "acc": 0.3432, "acc_ci": [ 0.2593, 0.4308 ], "chance": 0.3333, "levels": [ 0.32, 0.32, 0.3333, 0.4091, 0.3333 ] }, "absent_object": { "score": 0.9031, "ci": [ 0.8337, 0.9619 ], "acc": 0.9218, "acc_ci": [ 0.8658, 0.9692 ], "chance": 0.1933, "levels": [ 0.92, 1.0, 1.0, 0.92, 0.7692 ] }, "match_render": { "score": 0.384, "ci": [ 0.2769, 0.4916 ], "acc": 0.538, "acc_ci": [ 0.4577, 0.6187 ], "chance": 0.25, "levels": [ 0.68, 0.76, 0.68, 0.25, 0.32 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 0, "cost_label": "$0 (local)", "runs_locally": true, "seconds_per_answer": 1.3, "open_weights": true }, { "model": "MiniCPM-V 4.5", "maker": "OpenBMB", "id": "minicpm-v-4.5", "access": "runs locally", "type": "local open model", "group": "Local · open weights", "effort": "default", "overall": 0.171, "overall_ci": [ 0.1593, 0.1866 ], "categories": { "reading": { "score": 0.2108, "ci": [ 0.1898, 0.2312 ], "partial": false }, "counting": { "score": 0.122, "ci": [ 0.0927, 0.1533 ], "partial": false }, "space": { "score": 0.0635, "ci": [ 0.0399, 0.1062 ], "partial": false }, "changes": { "score": 0.1595, "ci": [ 0.1234, 0.1958 ], "partial": false }, "diagrams": { "score": 0.072, "ci": [ 0.0507, 0.096 ], "partial": false }, "fooled": { "score": 0.398, "ci": [ 0.351, 0.4594 ], "partial": false } }, "tests": { "absent_object": { "score": 0.8245, "ci": [ 0.7383, 0.9016 ], "acc": 0.8585, "acc_ci": [ 0.7889, 0.9206 ], "chance": 0.1933, "levels": [ 1.0, 1.0, 0.88, 0.72, 0.6923 ] }, "cant_tell_scene": { "score": 0.6196, "ci": [ 0.5016, 0.7354 ], "acc": 0.7162, "acc_ci": [ 0.6281, 0.8026 ], "chance": 0.2538, "levels": [ 0.7083, 0.72, 0.8, 0.6364 ] }, "clock_scene": { "score": 0.016, "ci": [ 0.0, 0.04 ], "acc": 0.016, "acc_ci": [ 0.0, 0.04 ], "chance": 0.0, "levels": [ 0.0, 0.04, 0.04, 0.0, 0.0 ] }, "coin_sum": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "conveyor_count": { "score": 0.134, "ci": [ 0.0759, 0.2 ], "acc": 0.134, "acc_ci": [ 0.0759, 0.2 ], "chance": 0.0, "levels": [ 0.2, 0.05, 0.1, 0.24, 0.08 ] }, "dashcam_light": { "score": 0.376, "ci": [ 0.256, 0.484 ], "acc": 0.584, "acc_ci": [ 0.504, 0.656 ], "chance": 0.3333, "levels": [ 0.88, 0.8, 0.52, 0.36, 0.36 ] }, "depth_scene": { "score": 0.2967, "ci": [ 0.148, 0.4587 ], "acc": 0.6483, "acc_ci": [ 0.574, 0.7293 ], "chance": 0.5, "levels": [ 1.0, 0.68, 0.52, 0.5833, 0.4583 ] }, "eye_chart": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.0944, "ci": [ 0.0089, 0.1738 ], "acc": 0.2303, "acc_ci": [ 0.1575, 0.2977 ], "chance": 0.15, "levels": [ 0.3913, 0.32, 0.2, 0.16, 0.08 ] }, "illusion": { "score": 0.028, "ci": [ 0.0, 0.16 ], "acc": 0.352, "acc_ci": [ 0.264, 0.44 ], "chance": 0.3333, "levels": [ 0.44, 0.36, 0.4, 0.32, 0.24 ] }, "invoice_check": { "score": 0.0572, "ci": [ 0.0058, 0.1172 ], "acc": 0.12, "acc_ci": [ 0.072, 0.176 ], "chance": 0.0666, "levels": [ 0.32, 0.04, 0.12, 0.04, 0.08 ] }, "label_scene": { "score": 0.552, "ci": [ 0.512, 0.584 ], "acc": 0.552, "acc_ci": [ 0.512, 0.584 ], "chance": 0.0, "levels": [ 1.0, 0.8, 0.96, 0.0, 0.0 ] }, "match_render": { "score": 0.0, "ci": [ 0.0, 0.0853 ], "acc": 0.2417, "acc_ci": [ 0.169, 0.314 ], "chance": 0.25, "levels": [ 0.4, 0.2, 0.28, 0.2083, 0.12 ] }, "meter_scene": { "score": 0.16, "ci": [ 0.104, 0.216 ], "acc": 0.16, "acc_ci": [ 0.104, 0.216 ], "chance": 0.0, "levels": [ 0.48, 0.08, 0.16, 0.08, 0.0 ] }, "missing_dim": { "score": 0.008, "ci": [ 0.0, 0.024 ], "acc": 0.008, "acc_ci": [ 0.0, 0.024 ], "chance": 0.0, "levels": [ 0.0, 0.04, 0.0, 0.0, 0.0 ] }, "pallet_count": { "score": 0.136, "ci": [ 0.088, 0.184 ], "acc": 0.136, "acc_ci": [ 0.088, 0.184 ], "chance": 0.0, "levels": [ 0.56, 0.04, 0.08, 0.0, 0.0 ] }, "parking_bay": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.04, "acc_ci": [ 0.008, 0.08 ], "chance": 0.1083, "levels": [ 0.04, 0.04, 0.08, 0.04, 0.0 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.1127 ], "acc": 0.4677, "acc_ci": [ 0.3787, 0.5563 ], "chance": 0.5, "levels": [ 0.48, 0.44, 0.48, 0.4583, 0.48 ] }, "pipe_follow": { "score": 0.024, "ci": [ 0.0, 0.056 ], "acc": 0.024, "acc_ci": [ 0.0, 0.056 ], "chance": 0.0, "levels": [ 0.08, 0.04, 0.0, 0.0, 0.0 ] }, "render_bug": { "score": 0.1213, "ci": [ 0.0279, 0.2219 ], "acc": 0.2677, "acc_ci": [ 0.1899, 0.3516 ], "chance": 0.1667, "levels": [ 0.36, 0.2917, 0.2273, 0.1739, 0.2857 ] }, "shelf_plan": { "score": 0.08, "ci": [ 0.032, 0.128 ], "acc": 0.08, "acc_ci": [ 0.032, 0.128 ], "chance": 0.0, "levels": [ 0.12, 0.12, 0.12, 0.04, 0.0 ] }, "shell_game": { "score": 0.0532, "ci": [ 0.0, 0.1661 ], "acc": 0.312, "acc_ci": [ 0.23, 0.394 ], "chance": 0.2733, "levels": [ 0.4, 0.45, 0.35, 0.2, 0.16 ] }, "stack_scene": { "score": 0.0, "ci": [ 0.0, 0.1257 ], "acc": 0.333, "acc_ci": [ 0.2524, 0.4172 ], "chance": 0.3333, "levels": [ 0.36, 0.32, 0.3333, 0.3182, 0.3333 ] }, "test_strips": { "score": 0.272, "ci": [ 0.2, 0.352 ], "acc": 0.272, "acc_ci": [ 0.2, 0.352 ], "chance": 0.0, "levels": [ 0.36, 0.28, 0.36, 0.2, 0.16 ] }, "ai_or_real": { "score": 0.12, "ci": [ 0.0, 0.2667 ], "acc": 0.56, "acc_ci": [ 0.4867, 0.6333 ], "chance": 0.5, "levels": [ 0.4, 0.7667, 0.7, 0.5333, 0.4 ] }, "cable_scene": { "score": 0.184, "ci": [ 0.128, 0.248 ], "acc": 0.184, "acc_ci": [ 0.128, 0.248 ], "chance": 0.0, "levels": [ 0.48, 0.32, 0.12, 0.0, 0.0 ] }, "change_scene": { "score": 0.3797, "ci": [ 0.2997, 0.4607 ], "acc": 0.3797, "acc_ci": [ 0.2997, 0.4607 ], "chance": 0.0, "levels": [ 0.72, 0.4583, 0.24, 0.2, 0.28 ] }, "chart_screen": { "score": 0.31, "ci": [ 0.24, 0.38 ], "acc": 0.31, "acc_ci": [ 0.24, 0.38 ], "chance": 0.0, "levels": [ 0.72, 0.32, 0.2, 0.0 ] }, "count_scene": { "score": 0.096, "ci": [ 0.048, 0.144 ], "acc": 0.096, "acc_ci": [ 0.048, 0.144 ], "chance": 0.0, "levels": [ 0.28, 0.16, 0.0, 0.0, 0.04 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 0, "cost_label": "$0 (local)", "runs_locally": true, "seconds_per_answer": 1.3, "open_weights": true }, { "model": "Gemma 4 E4B", "maker": "Google", "id": "gemma-4-e4b", "access": "runs locally", "type": "local open model", "group": "Local · open weights", "effort": "default", "overall": 0.137, "overall_ci": [ 0.1254, 0.1519 ], "categories": { "reading": { "score": 0.165, "ci": [ 0.1435, 0.1875 ], "partial": false }, "counting": { "score": 0.0293, "ci": [ 0.016, 0.0453 ], "partial": false }, "space": { "score": 0.0967, "ci": [ 0.0715, 0.1399 ], "partial": false }, "changes": { "score": 0.0735, "ci": [ 0.0459, 0.1069 ], "partial": false }, "diagrams": { "score": 0.0667, "ci": [ 0.0427, 0.0907 ], "partial": false }, "fooled": { "score": 0.3909, "ci": [ 0.3326, 0.4469 ], "partial": false } }, "tests": { "absent_object": { "score": 0.6674, "ci": [ 0.5613, 0.765 ], "acc": 0.7317, "acc_ci": [ 0.6462, 0.8105 ], "chance": 0.1933, "levels": [ 0.92, 0.88, 0.84, 0.48, 0.5385 ] }, "cant_tell_scene": { "score": 0.6295, "ci": [ 0.5126, 0.7414 ], "acc": 0.7236, "acc_ci": [ 0.6363, 0.807 ], "chance": 0.2538, "levels": [ 0.5833, 0.88, 0.84, 0.5909 ] }, "clock_scene": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "coin_sum": { "score": 0.04, "ci": [ 0.008, 0.072 ], "acc": 0.04, "acc_ci": [ 0.008, 0.072 ], "chance": 0.0, "levels": [ 0.2, 0.0, 0.0, 0.0, 0.0 ] }, "conveyor_count": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "count_scene": { "score": 0.088, "ci": [ 0.048, 0.136 ], "acc": 0.088, "acc_ci": [ 0.048, 0.136 ], "chance": 0.0, "levels": [ 0.28, 0.12, 0.04, 0.0, 0.0 ] }, "dashcam_light": { "score": 0.124, "ci": [ 0.004, 0.244 ], "acc": 0.416, "acc_ci": [ 0.336, 0.496 ], "chance": 0.3333, "levels": [ 0.68, 0.4, 0.32, 0.32, 0.36 ] }, "depth_scene": { "score": 0.4413, "ci": [ 0.294, 0.5733 ], "acc": 0.7207, "acc_ci": [ 0.647, 0.7867 ], "chance": 0.5, "levels": [ 1.0, 0.92, 0.6, 0.5, 0.5833 ] }, "eye_chart": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.0756, "ci": [ 0.0, 0.1628 ], "acc": 0.2143, "acc_ci": [ 0.1482, 0.2883 ], "chance": 0.15, "levels": [ 0.3913, 0.24, 0.2, 0.12, 0.12 ] }, "invoice_check": { "score": 0.0229, "ci": [ 0.0, 0.0743 ], "acc": 0.088, "acc_ci": [ 0.048, 0.136 ], "chance": 0.0666, "levels": [ 0.28, 0.08, 0.04, 0.04, 0.0 ] }, "label_scene": { "score": 0.544, "ci": [ 0.504, 0.576 ], "acc": 0.544, "acc_ci": [ 0.504, 0.576 ], "chance": 0.0, "levels": [ 0.92, 0.88, 0.92, 0.0, 0.0 ] }, "meter_scene": { "score": 0.096, "ci": [ 0.048, 0.144 ], "acc": 0.096, "acc_ci": [ 0.048, 0.144 ], "chance": 0.0, "levels": [ 0.28, 0.0, 0.16, 0.04, 0.0 ] }, "missing_dim": { "score": 0.04, "ci": [ 0.008, 0.072 ], "acc": 0.04, "acc_ci": [ 0.008, 0.072 ], "chance": 0.0, "levels": [ 0.2, 0.0, 0.0, 0.0, 0.0 ] }, "pallet_count": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "parking_bay": { "score": 0.049, "ci": [ 0.0, 0.1208 ], "acc": 0.152, "acc_ci": [ 0.096, 0.216 ], "chance": 0.1083, "levels": [ 0.2, 0.2, 0.12, 0.16, 0.08 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.1273 ], "acc": 0.4757, "acc_ci": [ 0.3943, 0.5637 ], "chance": 0.5, "levels": [ 0.48, 0.48, 0.48, 0.4583, 0.48 ] }, "pipe_follow": { "score": 0.056, "ci": [ 0.024, 0.096 ], "acc": 0.056, "acc_ci": [ 0.024, 0.096 ], "chance": 0.0, "levels": [ 0.08, 0.2, 0.0, 0.0, 0.0 ] }, "render_bug": { "score": 0.0538, "ci": [ 0.0, 0.1409 ], "acc": 0.2115, "acc_ci": [ 0.1406, 0.2841 ], "chance": 0.1667, "levels": [ 0.2, 0.0833, 0.1364, 0.3043, 0.3333 ] }, "shelf_plan": { "score": 0.096, "ci": [ 0.048, 0.152 ], "acc": 0.096, "acc_ci": [ 0.048, 0.152 ], "chance": 0.0, "levels": [ 0.16, 0.2, 0.08, 0.0, 0.04 ] }, "shell_game": { "score": 0.0, "ci": [ 0.0, 0.1083 ], "acc": 0.264, "acc_ci": [ 0.182, 0.352 ], "chance": 0.2733, "levels": [ 0.25, 0.25, 0.3, 0.28, 0.24 ] }, "test_strips": { "score": 0.256, "ci": [ 0.184, 0.336 ], "acc": 0.256, "acc_ci": [ 0.184, 0.336 ], "chance": 0.0, "levels": [ 0.4, 0.28, 0.2, 0.16, 0.24 ] }, "ai_or_real": { "score": 0.2667, "ci": [ 0.1067, 0.4133 ], "acc": 0.6333, "acc_ci": [ 0.5533, 0.7067 ], "chance": 0.5, "levels": [ 0.5333, 0.7667, 0.6667, 0.6333, 0.5667 ] }, "cable_scene": { "score": 0.104, "ci": [ 0.056, 0.152 ], "acc": 0.104, "acc_ci": [ 0.056, 0.152 ], "chance": 0.0, "levels": [ 0.2, 0.12, 0.16, 0.0, 0.04 ] }, "change_scene": { "score": 0.1213, "ci": [ 0.0723, 0.178 ], "acc": 0.1213, "acc_ci": [ 0.0723, 0.178 ], "chance": 0.0, "levels": [ 0.16, 0.1667, 0.24, 0.0, 0.04 ] }, "chart_screen": { "score": 0.26, "ci": [ 0.18, 0.3302 ], "acc": 0.26, "acc_ci": [ 0.18, 0.3302 ], "chance": 0.0, "levels": [ 0.52, 0.4, 0.12, 0.0 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.04 ], "acc": 0.28, "acc_ci": [ 0.2, 0.36 ], "chance": 0.3333, "levels": [ 0.28, 0.32, 0.32, 0.2, 0.28 ] }, "stack_scene": { "score": 0.0147, "ci": [ 0.0, 0.1346 ], "acc": 0.3432, "acc_ci": [ 0.2596, 0.423 ], "chance": 0.3333, "levels": [ 0.32, 0.32, 0.3333, 0.4091, 0.3333 ] }, "match_render": { "score": 0.096, "ci": [ 0.0, 0.2036 ], "acc": 0.322, "acc_ci": [ 0.2417, 0.4027 ], "chance": 0.25, "levels": [ 0.28, 0.44, 0.4, 0.25, 0.24 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 0, "cost_label": "$0 (local)", "runs_locally": true, "seconds_per_answer": 0.6, "open_weights": true }, { "model": "Mistral Small 2603", "maker": "Mistral", "id": "mistral-small-2603", "access": "API", "type": "general", "group": "Others", "effort": "default", "overall": 0.1253, "overall_ci": [ 0.1149, 0.1422 ], "categories": { "reading": { "score": 0.1488, "ci": [ 0.136, 0.1708 ], "partial": false }, "counting": { "score": 0.1, "ci": [ 0.068, 0.136 ], "partial": true }, "space": { "score": 0.0658, "ci": [ 0.0332, 0.1087 ], "partial": true }, "changes": { "score": 0.1204, "ci": [ 0.0883, 0.1551 ], "partial": false }, "diagrams": { "score": 0.096, "ci": [ 0.0693, 0.12 ], "partial": false }, "fooled": { "score": 0.2208, "ci": [ 0.1891, 0.2864 ], "partial": false } }, "tests": { "clock_scene": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "change_scene": { "score": 0.3223, "ci": [ 0.2417, 0.4033 ], "acc": 0.3223, "acc_ci": [ 0.2417, 0.4033 ], "chance": 0.0, "levels": [ 0.4, 0.2917, 0.44, 0.16, 0.32 ] }, "match_render": { "score": 0.064, "ci": [ 0.0, 0.1724 ], "acc": 0.298, "acc_ci": [ 0.2173, 0.3793 ], "chance": 0.25, "levels": [ 0.28, 0.4, 0.32, 0.25, 0.24 ] }, "absent_object": { "score": 0.3813, "ci": [ 0.273, 0.4896 ], "acc": 0.5009, "acc_ci": [ 0.4135, 0.5883 ], "chance": 0.1933, "levels": [ 0.72, 0.72, 0.28, 0.4, 0.3846 ] }, "ai_or_real": { "score": 0.0, "ci": [ 0.0, 0.16 ], "acc": 0.5, "acc_ci": [ 0.42, 0.58 ], "chance": 0.5, "levels": [ 0.5, 0.5, 0.5, 0.5, 0.5 ] }, "cable_scene": { "score": 0.104, "ci": [ 0.056, 0.1522 ], "acc": 0.104, "acc_ci": [ 0.056, 0.1522 ], "chance": 0.0, "levels": [ 0.24, 0.16, 0.08, 0.0, 0.04 ] }, "cant_tell_scene": { "score": 0.4981, "ci": [ 0.3716, 0.622 ], "acc": 0.6255, "acc_ci": [ 0.5311, 0.718 ], "chance": 0.2538, "levels": [ 0.5, 0.68, 0.64, 0.6818 ] }, "chart_screen": { "score": 0.29, "ci": [ 0.2198, 0.37 ], "acc": 0.29, "acc_ci": [ 0.2198, 0.37 ], "chance": 0.0, "levels": [ 0.64, 0.28, 0.16, 0.08 ] }, "coin_sum": { "score": 0.0163, "ci": [ 0.0, 0.041 ], "acc": 0.0163, "acc_ci": [ 0.0, 0.041 ], "chance": 0.0, "levels": [ 0.04, 0.0417, 0.0, 0.0, 0.0 ] }, "count_scene": { "score": 0.064, "ci": [ 0.024, 0.104 ], "acc": 0.064, "acc_ci": [ 0.024, 0.104 ], "chance": 0.0, "levels": [ 0.2, 0.04, 0.04, 0.04, 0.0 ] }, "dashcam_light": { "score": 0.004, "ci": [ 0.0, 0.136 ], "acc": 0.336, "acc_ci": [ 0.256, 0.424 ], "chance": 0.3333, "levels": [ 0.48, 0.36, 0.28, 0.32, 0.24 ] }, "depth_scene": { "score": 0.262, "ci": [ 0.098, 0.4107 ], "acc": 0.631, "acc_ci": [ 0.549, 0.7053 ], "chance": 0.5, "levels": [ 0.8, 0.84, 0.64, 0.5, 0.375 ] }, "eye_chart": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.0, "ci": [ 0.0, 0.038 ], "acc": 0.1242, "acc_ci": [ 0.0741, 0.1823 ], "chance": 0.15, "levels": [ 0.2609, 0.12, 0.12, 0.08, 0.04 ] }, "illusion": { "score": 0.004, "ci": [ 0.0, 0.124 ], "acc": 0.336, "acc_ci": [ 0.256, 0.416 ], "chance": 0.3333, "levels": [ 0.2, 0.36, 0.52, 0.28, 0.32 ] }, "invoice_check": { "score": 0.0572, "ci": [ 0.0, 0.1172 ], "acc": 0.12, "acc_ci": [ 0.064, 0.176 ], "chance": 0.0666, "levels": [ 0.32, 0.08, 0.08, 0.08, 0.04 ] }, "label_scene": { "score": 0.608, "ci": [ 0.584, 0.64 ], "acc": 0.608, "acc_ci": [ 0.584, 0.64 ], "chance": 0.0, "levels": [ 1.0, 1.0, 0.96, 0.08, 0.0 ] }, "meter_scene": { "score": 0.056, "ci": [ 0.024, 0.096 ], "acc": 0.056, "acc_ci": [ 0.024, 0.096 ], "chance": 0.0, "levels": [ 0.2, 0.08, 0.0, 0.0, 0.0 ] }, "missing_dim": { "score": 0.16, "ci": [ 0.112, 0.208 ], "acc": 0.16, "acc_ci": [ 0.112, 0.208 ], "chance": 0.0, "levels": [ 0.6, 0.12, 0.08, 0.0, 0.0 ] }, "pallet_count": { "score": 0.136, "ci": [ 0.08, 0.192 ], "acc": 0.136, "acc_ci": [ 0.08, 0.192 ], "chance": 0.0, "levels": [ 0.2, 0.12, 0.36, 0.0, 0.0 ] }, "parking_bay": { "score": 0.0, "ci": [ 0.0, 0.0132 ], "acc": 0.072, "acc_ci": [ 0.032, 0.12 ], "chance": 0.1083, "levels": [ 0.0, 0.16, 0.08, 0.08, 0.04 ] }, "part_mirror": { "score": 0.0, "ci": [ 0.0, 0.1127 ], "acc": 0.4677, "acc_ci": [ 0.3787, 0.5563 ], "chance": 0.5, "levels": [ 0.4, 0.44, 0.44, 0.4583, 0.6 ] }, "pipe_follow": { "score": 0.024, "ci": [ 0.0, 0.0482 ], "acc": 0.024, "acc_ci": [ 0.0, 0.0482 ], "chance": 0.0, "levels": [ 0.0, 0.08, 0.04, 0.0, 0.0 ] }, "render_bug": { "score": 0.062, "ci": [ 0.0, 0.1549 ], "acc": 0.2184, "acc_ci": [ 0.1425, 0.2958 ], "chance": 0.1667, "levels": [ 0.24, 0.2083, 0.2273, 0.1304, 0.2857 ] }, "shelf_plan": { "score": 0.04, "ci": [ 0.008, 0.072 ], "acc": 0.04, "acc_ci": [ 0.008, 0.072 ], "chance": 0.0, "levels": [ 0.04, 0.16, 0.0, 0.0, 0.0 ] }, "stack_scene": { "score": 0.0689, "ci": [ 0.0, 0.195 ], "acc": 0.3793, "acc_ci": [ 0.2964, 0.4633 ], "chance": 0.3333, "levels": [ 0.4, 0.36, 0.25, 0.6364, 0.25 ] }, "test_strips": { "score": 0.216, "ci": [ 0.144, 0.288 ], "acc": 0.216, "acc_ci": [ 0.144, 0.288 ], "chance": 0.0, "levels": [ 0.32, 0.2, 0.24, 0.12, 0.2 ] } }, "unsupported_tests": [ "conveyor_count", "shell_game" ], "tested": "2026-11-06", "cost_per_1000": 0.26, "cost_label": "$0.26", "runs_locally": false, "seconds_per_answer": 3.0, "open_weights": true }, { "model": "LFM2.5-VL 1.6B", "maker": "Liquid", "id": "lfm2.5-vl-1.6b", "access": "runs locally", "type": "local open model", "group": "Local · open weights", "effort": "default", "overall": 0.1244, "overall_ci": [ 0.114, 0.1421 ], "categories": { "reading": { "score": 0.1807, "ci": [ 0.1568, 0.2042 ], "partial": false }, "counting": { "score": 0.0587, "ci": [ 0.0367, 0.0827 ], "partial": false }, "space": { "score": 0.0285, "ci": [ 0.0099, 0.0822 ], "partial": false }, "changes": { "score": 0.1483, "ci": [ 0.1146, 0.1842 ], "partial": false }, "diagrams": { "score": 0.0427, "ci": [ 0.024, 0.0614 ], "partial": false }, "fooled": { "score": 0.2872, "ci": [ 0.2406, 0.3546 ], "partial": false } }, "tests": { "absent_object": { "score": 0.5484, "ci": [ 0.4408, 0.6544 ], "acc": 0.6357, "acc_ci": [ 0.5489, 0.7212 ], "chance": 0.1933, "levels": [ 0.8, 0.92, 0.48, 0.44, 0.5385 ] }, "cant_tell_scene": { "score": 0.5469, "ci": [ 0.4235, 0.6642 ], "acc": 0.6619, "acc_ci": [ 0.5698, 0.7494 ], "chance": 0.2538, "levels": [ 0.4167, 0.8, 0.84, 0.5909 ] }, "clock_scene": { "score": 0.024, "ci": [ 0.0, 0.056 ], "acc": 0.024, "acc_ci": [ 0.0, 0.056 ], "chance": 0.0, "levels": [ 0.04, 0.0, 0.08, 0.0, 0.0 ] }, "coin_sum": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "conveyor_count": { "score": 0.04, "ci": [ 0.01, 0.08 ], "acc": 0.04, "acc_ci": [ 0.01, 0.08 ], "chance": 0.0, "levels": [ 0.2, 0.0, 0.0, 0.0, 0.0 ] }, "dashcam_light": { "score": 0.34, "ci": [ 0.208, 0.472 ], "acc": 0.56, "acc_ci": [ 0.472, 0.648 ], "chance": 0.3333, "levels": [ 0.64, 0.64, 0.56, 0.56, 0.4 ] }, "depth_scene": { "score": 0.1213, "ci": [ 0.0, 0.3 ], "acc": 0.5607, "acc_ci": [ 0.471, 0.65 ], "chance": 0.5, "levels": [ 0.6, 0.56, 0.56, 0.5417, 0.5417 ] }, "eye_chart": { "score": 0.0, "ci": [ 0.0, 0.0 ], "acc": 0.0, "acc_ci": [ 0.0, 0.0 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.0, 0.0, 0.0 ] }, "facing_dir": { "score": 0.0, "ci": [ 0.0, 0.0748 ], "acc": 0.1482, "acc_ci": [ 0.0901, 0.2136 ], "chance": 0.15, "levels": [ 0.2609, 0.12, 0.16, 0.12, 0.08 ] }, "invoice_check": { "score": 0.0486, "ci": [ 0.0, 0.1 ], "acc": 0.112, "acc_ci": [ 0.064, 0.16 ], "chance": 0.0666, "levels": [ 0.44, 0.0, 0.08, 0.0, 0.04 ] }, "label_scene": { "score": 0.408, "ci": [ 0.344, 0.472 ], "acc": 0.408, "acc_ci": [ 0.344, 0.472 ], "chance": 0.0, "levels": [ 0.68, 0.8, 0.48, 0.08, 0.0 ] }, "meter_scene": { "score": 0.064, "ci": [ 0.024, 0.104 ], "acc": 0.064, "acc_ci": [ 0.024, 0.104 ], "chance": 0.0, "levels": [ 0.2, 0.04, 0.04, 0.04, 0.0 ] }, "missing_dim": { "score": 0.024, "ci": [ 0.0, 0.056 ], "acc": 0.024, "acc_ci": [ 0.0, 0.056 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.12, 0.0, 0.0 ] }, "pallet_count": { "score": 0.024, "ci": [ 0.0, 0.048 ], "acc": 0.024, "acc_ci": [ 0.0, 0.048 ], "chance": 0.0, "levels": [ 0.0, 0.0, 0.08, 0.0, 0.04 ] }, "parking_bay": { "score": 0.0042, "ci": [ 0.0, 0.067 ], "acc": 0.112, "acc_ci": [ 0.064, 0.168 ], "chance": 0.1083, "levels": [ 0.08, 0.16, 0.2, 0.08, 0.04 ] }, "part_mirror": { "score": 0.0487, "ci": [ 0.0, 0.2136 ], "acc": 0.5243, "acc_ci": [ 0.4357, 0.6068 ], "chance": 0.5, "levels": [ 0.52, 0.52, 0.52, 0.5417, 0.52 ] }, "pipe_follow": { "score": 0.032, "ci": [ 0.008, 0.064 ], "acc": 0.032, "acc_ci": [ 0.008, 0.064 ], "chance": 0.0, "levels": [ 0.0, 0.08, 0.08, 0.0, 0.0 ] }, "render_bug": { "score": 0.3034, "ci": [ 0.1998, 0.4087 ], "acc": 0.4195, "acc_ci": [ 0.3331, 0.5073 ], "chance": 0.1667, "levels": [ 0.6, 0.5833, 0.3636, 0.2174, 0.3333 ] }, "shelf_plan": { "score": 0.08, "ci": [ 0.04, 0.128 ], "acc": 0.08, "acc_ci": [ 0.04, 0.128 ], "chance": 0.0, "levels": [ 0.2, 0.12, 0.0, 0.0, 0.08 ] }, "shell_game": { "score": 0.0, "ci": [ 0.0, 0.1138 ], "acc": 0.272, "acc_ci": [ 0.192, 0.356 ], "chance": 0.2733, "levels": [ 0.35, 0.35, 0.3, 0.2, 0.16 ] }, "test_strips": { "score": 0.24, "ci": [ 0.168, 0.312 ], "acc": 0.24, "acc_ci": [ 0.168, 0.312 ], "chance": 0.0, "levels": [ 0.36, 0.2, 0.24, 0.28, 0.12 ] }, "ai_or_real": { "score": 0.0533, "ci": [ 0.0, 0.2267 ], "acc": 0.5267, "acc_ci": [ 0.4467, 0.6133 ], "chance": 0.5, "levels": [ 0.5, 0.5667, 0.5333, 0.5333, 0.5 ] }, "cable_scene": { "score": 0.072, "ci": [ 0.032, 0.12 ], "acc": 0.072, "acc_ci": [ 0.032, 0.12 ], "chance": 0.0, "levels": [ 0.2, 0.08, 0.08, 0.0, 0.0 ] }, "change_scene": { "score": 0.1613, "ci": [ 0.0967, 0.2257 ], "acc": 0.1613, "acc_ci": [ 0.0967, 0.2257 ], "chance": 0.0, "levels": [ 0.16, 0.1667, 0.16, 0.16, 0.16 ] }, "chart_screen": { "score": 0.37, "ci": [ 0.29, 0.46 ], "acc": 0.37, "acc_ci": [ 0.29, 0.46 ], "chance": 0.0, "levels": [ 0.64, 0.48, 0.16, 0.2 ] }, "count_scene": { "score": 0.112, "ci": [ 0.064, 0.168 ], "acc": 0.112, "acc_ci": [ 0.064, 0.168 ], "chance": 0.0, "levels": [ 0.24, 0.16, 0.08, 0.08, 0.0 ] }, "illusion": { "score": 0.0, "ci": [ 0.0, 0.1 ], "acc": 0.32, "acc_ci": [ 0.24, 0.4 ], "chance": 0.3333, "levels": [ 0.44, 0.36, 0.44, 0.2, 0.16 ] }, "stack_scene": { "score": 0.0147, "ci": [ 0.0, 0.1361 ], "acc": 0.3432, "acc_ci": [ 0.2601, 0.4241 ], "chance": 0.3333, "levels": [ 0.32, 0.32, 0.3333, 0.4091, 0.3333 ] }, "match_render": { "score": 0.0107, "ci": [ 0.0, 0.1182 ], "acc": 0.258, "acc_ci": [ 0.185, 0.3387 ], "chance": 0.25, "levels": [ 0.28, 0.24, 0.28, 0.25, 0.24 ] } }, "unsupported_tests": [], "tested": "2026-11-06", "cost_per_1000": 0, "cost_label": "$0 (local)", "runs_locally": true, "seconds_per_answer": 0.7, "open_weights": true } ], "method": "Scores are corrected for guessing: see https://aivisionindex.com/method/", "licence": "CC BY 4.0 (aggregate results): reuse with credit to the AI Vision Index, https://aivisionindex.com/" }