{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_production_content_review", "scoring_mode": "automated", "score_status": "scored", "model_results": [ { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 8.38, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-06-13T06:11:55.079398+00:00", "evaluated_item_count": 80, "score_count": 80, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 0.001336, "cost_basis": "actual_cash", "costed_scored_output_count": 43 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "provider": "Google via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 8, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-13T19:27:16.837023+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.011758, "cost_basis": "actual_cash", "costed_scored_output_count": 26 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 7.79, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-23T09:06:03.144283+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.238752, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 27 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 6.83, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-13T07:10:10.465469+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.038264, "cost_basis": "actual_cash", "costed_scored_output_count": 22 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "provider": "Google Gemini", "model_profile_ids": [ "default", "reasoning_high" ], "quality_score_10": 6.79, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T14:58:19.536003+00:00", "evaluated_item_count": 160, "score_count": 160, "scored_run_count": 4, "mean_cost_per_scored_output_usd": 0.002095, "cost_basis": "actual_cash", "costed_scored_output_count": 66 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 6.68, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T13:13:01.074910+00:00", "evaluated_item_count": 120, "score_count": 120, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.001534, "cost_basis": "actual_cash", "costed_scored_output_count": 69 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 6.38, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-09T10:49:21.128753+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.311794, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 30 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 6.25, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-09T10:19:09.204523+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.011322, "cost_basis": "actual_cash", "costed_scored_output_count": 25 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 6.24, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T15:46:13.432140+00:00", "evaluated_item_count": 120, "score_count": 120, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.092892, "cost_basis": "actual_cash", "costed_scored_output_count": 72 }, { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 5.5, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T20:03:46.341112+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.166208, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 24 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "provider": "Meta via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 5.38, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-08T13:21:12.644135+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.009496, "cost_basis": "actual_cash", "costed_scored_output_count": 27 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 5.22, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T14:50:33.928423+00:00", "evaluated_item_count": 120, "score_count": 120, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.027287, "cost_basis": "actual_cash", "costed_scored_output_count": 79 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "provider": "OpenAI", "model_profile_ids": [ "reasoning_xhigh" ], "quality_score_10": 5.18, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T16:35:14.915731+00:00", "evaluated_item_count": 160, "score_count": 160, "scored_run_count": 4, "mean_cost_per_scored_output_usd": 0.039381, "cost_basis": "actual_cash", "costed_scored_output_count": 99 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 5.12, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-21T05:52:55.788644+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.027743, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 23 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 5.12, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-09T10:51:41.921509+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.067096, "cost_basis": "actual_cash", "costed_scored_output_count": 28 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 4.96, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-23T05:57:12.009052+00:00", "evaluated_item_count": 40, "score_count": 40, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.011546, "cost_basis": "actual_cash", "costed_scored_output_count": 26 } ], "scaffold_comparison": { "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 5.0835, "scaffolded_score_10": 5.5005, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 6.375, "scaffolded_score_10": 5.5415, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 4.875, "scaffolded_score_10": 5.125, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 60, "baseline_score_10": 6.2365, "scaffolded_score_10": 5.8892, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 7.792, "scaffolded_score_10": 7.792, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 80, "baseline_score_10": 6.6563, "scaffolded_score_10": 6.7915, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 40, "baseline_score_10": 8, "scaffolded_score_10": 8.375, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 60, "baseline_score_10": 6.625, "scaffolded_score_10": 6.6807, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 5.9585, "scaffolded_score_10": 6.25, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 80, "baseline_score_10": 4.6872, "scaffolded_score_10": 5.177, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 60, "baseline_score_10": 4.9997, "scaffolded_score_10": 5.2218, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 5.083, "scaffolded_score_10": 5.1245, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 4.833, "scaffolded_score_10": 4.958, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 8, "scaffolded_score_10": 7.917, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 5.375, "scaffolded_score_10": 5.375, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 20, "baseline_score_10": 6.8335, "scaffolded_score_10": 6.4585, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/ad-creative-review/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T08:22:38.946Z", "methodology": { "label": "AI ad creative review", "summary": "Models review ten images and ten videos containing known defects. They are asked to check launch readiness without being told that a defect is guaranteed. Scoring measures how much of the answer-key defect set they find.", "scoringLabel": "Answer-key defect coverage / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "Both prompt conditions receive the same twenty media assets. The evaluator's key records the planted issue and observed secondary video defects. A complete run has forty reviews: twenty assets under each condition." ] }, { "heading": "What changes between prompts", "paragraphs": [ "The simple prompt asks for launch QA; the scaffold adds a content-review checklist. The checklist directs the inspection without telling the model which defect is present in an asset." ] }, { "heading": "How the result is scored", "paragraphs": [ "A single-defect asset scores 10 when the expected defect is found, 5 when an issue is flagged without matching that defect, and 0 when it is missed. Multi-defect video scores reflect the share of expected defects matched. Semantic interpretation resolves eligible answer matches. The pack does not establish a clean-ad false-alarm rate." ] } ] }, "content": { "title": "Best LLM for reviewing ad creative", "description": "Benmark tests whether models find known defects in finished image and video ads.", "intro": "Benmark tests whether models find known defects in finished image and video ads. The useful result is a specific issue with evidence, so the next reviewer can decide what needs fixing.", "findingTitle": "Make the review specific enough to act on", "resultInterpretation": "The leading models recover a substantial share of the known defects. Their value is in directing attention to a concrete problem in the asset, rather than producing a general list of things that might be wrong.", "sections": [ { "heading": "A warning needs to identify the defect", "paragraphs": [ "An ad review that says 'check the claims' leaves the work with the reader. A useful review points to the claim, explains the problem and makes the consequence clear. The benchmark rewards matching the actual defect set, so vague caution cannot earn full credit.", "The twenty-asset pack includes both images and videos. Some videos contain more than one known issue, and the score reflects how much of that set the model finds. Finding one problem is useful; it is not the same as completing the inspection." ] }, { "heading": "A longer checklist does not guarantee better detection", "paragraphs": [ "The baseline asks for launch QA without a checklist, while the scaffold adds one. In the reviewed Claude Fable 5.1 comparison, the mean falls from 6.38 to 5.54 across twenty paired observations. The right question is whether the checklist improves detection on your assets.", "Keep the review focused on the dimensions that matter for the creative: product fidelity, visible or spoken claims, copy, offer and format. Ask the model to distinguish a confirmed issue from a question requiring more context, then inspect the evidence it gives." ] }, { "heading": "Use AI to make human review more targeted", "paragraphs": [ "A practical handover is a short list of issues with the affected image region or video moment, the reason each matters and the next action. That makes the model's finding easy to verify and route to the person who can fix it.", "This pack measures defect coverage on assets with known issues. To test a release gate in your own process, include clean assets too: the cost of flagging a good ad matters alongside the cost of missing a bad one." ] } ], "related": [ { "path": "/benchmarks/creative-tagging/", "label": "Structuring creative information" }, { "path": "/benchmarks/ai-video-ad-generation/", "label": "Comparing generation quality" } ] } }