{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_analysis_ad_analysis", "scoring_mode": "hybrid", "score_status": "scored", "model_results": [ { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "provider": "Google via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 9.93, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.677278+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.035188, "cost_basis": "actual_cash", "costed_scored_output_count": 20 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 9.93, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.523227+00:00", "evaluated_item_count": 40, "score_count": 21, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 0.123921, "cost_basis": "actual_cash", "costed_scored_output_count": 21 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 9.9, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.481918+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.414609, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 20 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 9.9, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.493761+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.621772, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 20 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 9.73, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.638680+00:00", "evaluated_item_count": 60, "score_count": 60, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.005449, "cost_basis": "actual_cash", "costed_scored_output_count": 60 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 9.7, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.648295+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.009809, "cost_basis": "actual_cash", "costed_scored_output_count": 20 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "provider": "Meta via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 9.67, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.686833+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.018153, "cost_basis": "actual_cash", "costed_scored_output_count": 20 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 9.43, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.697118+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.081269, "cost_basis": "actual_cash", "costed_scored_output_count": 20 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "provider": "Google Gemini", "model_profile_ids": [ "default" ], "quality_score_10": 9.42, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.608321+00:00", "evaluated_item_count": 60, "score_count": 60, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.005651, "cost_basis": "actual_cash", "costed_scored_output_count": 60 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 9.33, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.658938+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.153645, "cost_basis": "actual_cash", "costed_scored_output_count": 20 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "provider": "Google Gemini", "model_profile_ids": [ "default" ], "quality_score_10": 9.32, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.580219+00:00", "evaluated_item_count": 60, "score_count": 60, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.007526, "cost_basis": "actual_cash", "costed_scored_output_count": 60 }, { "model_id": "gemini-3.1-flash-lite", "model_label": "Gemini 3.1 Flash-Lite", "provider": "Google Gemini", "model_profile_ids": [ "default" ], "quality_score_10": 8.94, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.551907+00:00", "evaluated_item_count": 60, "score_count": 60, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.000795, "cost_basis": "actual_cash", "costed_scored_output_count": 60 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 8, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.667715+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.022192, "cost_basis": "actual_cash", "costed_scored_output_count": 20 } ], "scaffold_comparison": { "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "status": "awaiting_scores", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 30, "unpaired_scaffolded_count": 30, "unscored_comparison_unit_count": 30, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 9.6667, "scaffolded_score_10": 9.9, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "status": "awaiting_scores", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 30, "unpaired_scaffolded_count": 30, "unscored_comparison_unit_count": 30, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "status": "partial", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 9.8333, "scaffolded_score_10": 9.9333, "unpaired_baseline_count": 20, "unpaired_scaffolded_count": 20, "unscored_comparison_unit_count": 20, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 9.7667, "scaffolded_score_10": 9.9, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-flash-lite", "model_label": "Gemini 3.1 Flash-Lite", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 30, "baseline_score_10": 8.3333, "scaffolded_score_10": 8.9444, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "status": "partial", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 30, "baseline_score_10": 9.1444, "scaffolded_score_10": 9.3222, "unpaired_baseline_count": 30, "unpaired_scaffolded_count": 30, "unscored_comparison_unit_count": 30, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 30, "baseline_score_10": 9.0333, "scaffolded_score_10": 9.4222, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 30, "baseline_score_10": 9.0667, "scaffolded_score_10": 9.7333, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 9.1667, "scaffolded_score_10": 9.7, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "status": "awaiting_scores", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 30, "unpaired_scaffolded_count": 30, "unscored_comparison_unit_count": 30, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "status": "awaiting_scores", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 30, "unpaired_scaffolded_count": 30, "unscored_comparison_unit_count": 30, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 8.6667, "scaffolded_score_10": 9.3333, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 7.5, "scaffolded_score_10": 8, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6.1-sol", "model_label": "GPT-6.1 Sol", "status": "awaiting_scores", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 19, "unpaired_scaffolded_count": 18, "unscored_comparison_unit_count": 18, "invalid_comparison_unit_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 9.3, "scaffolded_score_10": 9.9333, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 8.8333, "scaffolded_score_10": 9.6667, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "status": "partial", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 9, "scaffolded_score_10": 9.4333, "unpaired_baseline_count": 9, "unpaired_scaffolded_count": 9, "unscored_comparison_unit_count": 9, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/ad-creative-analysis/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T08:22:38.946Z", "methodology": { "label": "Ad creative analysis", "summary": "Models inspect five image ads and five video ads and explain their creative mechanisms. Ben blind-scores diagnostic accuracy, depth and explanation quality. Performance data is withheld, so the task measures creative diagnosis rather than causal attribution of sales.", "scoringLabel": "Human-rated creative diagnosis / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "The fixed pack contains five static images and five videos. Models see the media, with original filenames and account performance withheld. They receive no spend, conversion, reach or brand-context evidence. Analysis and creative recommendations are generated as separate outputs." ] }, { "heading": "What changes between prompts", "paragraphs": [ "Each asset is tested with a simple and a scaffolded prompt. The analysis scaffold covers attention, product relevance, proof, platform fit and other creative mechanisms. The action scaffold asks what to preserve, what to change and why. The media remains identical between conditions." ] }, { "heading": "How the result is scored", "paragraphs": [ "The explanation is scored on diagnostic accuracy, depth of analysis and quality of explanation. Diagnostic accuracy is a critical dimension: a persuasive explanation with a weak diagnosis does not clear the task's readiness threshold. Recommendations for the next ad have their own score and page." ] } ] }, "content": { "title": "Best LLM for analysing ad creative", "description": "Benmark tests how well models explain the creative mechanisms in image and video ads.", "intro": "Benmark tests how well models explain the creative mechanisms in image and video ads. The strongest results make AI a valuable part of creative diagnosis and a useful input to the next test.", "findingTitle": "Use AI to sharpen the creative diagnosis", "resultInterpretation": "The joint leaders score exceptionally highly on the reviewed creative-analysis task. This is a strong use case for putting a model beside the marketer: it can develop a detailed explanation of how an ad earns attention and makes its selling argument.", "sections": [ { "heading": "Make the explanation traceable to the ad", "paragraphs": [ "A useful diagnosis points to what is actually there: the opening moment, the demonstration, the proof, the sequence of information or the way the product is connected to a buying situation. Those observations give the explanation something you can inspect.", "The test withholds account performance and brand context. That focuses the model on the creative itself. Its explanation can suggest why an ad might work, while campaign data is needed to investigate what drove a measured result." ] }, { "heading": "Structure helps this analysis task", "paragraphs": [ "Google Gemini 4 Argon scores 9.30 with the simple prompt and 9.93 with the scaffold across ten paired observations. The scaffold directs attention to creative mechanisms such as product relevance, proof, barriers and platform fit.", "That is a useful contrast with concept generation, where the tested framework can constrain the idea. Use instructions that fit the job: a diagnostic checklist can help inspect an existing asset even when the same style of structure is less useful for inventing the next one." ] }, { "heading": "Turn the diagnosis into a testable decision", "paragraphs": [ "Ask the model to identify the most important mechanism and the observation that supports its explanation. Then combine that diagnosis with account evidence before choosing what to preserve or change.", "For an illustrative example, a product demonstration may look like the source of the ad's clarity. An iteration can preserve that demonstration while changing the opening, giving you a clearer test of the opening's contribution. Benmark scores the recommendation for that next version separately." ] } ], "related": [ { "path": "/benchmarks/creative-iteration/", "label": "Recommending the next creative version" }, { "path": "/benchmarks/campaign-performance-analysis/", "label": "Checking the performance evidence" } ] } }