{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_ideation_brand_only_concept_generation", "scoring_mode": "hybrid", "score_status": "scored", "model_results": [ { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 8.11, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.019435+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.553003, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 6 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 7.33, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.056253+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.009638, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "provider": "Google via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 7.22, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.132977+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.012145, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "provider": "Meta via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 6.84, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.113262+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.006857, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 6.78, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-23T08:15:43.850000+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.0098, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 6.67, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.049788+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.24972, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 6 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 6.63, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.074244+00:00", "evaluated_item_count": 18, "score_count": 18, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.000984, "cost_basis": "actual_cash", "costed_scored_output_count": 18 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "provider": "Anthropic", "model_profile_ids": [ "thinking_high" ], "quality_score_10": 6.59, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.199370+00:00", "evaluated_item_count": 18, "score_count": 18, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.011073, "cost_basis": "actual_cash", "costed_scored_output_count": 18 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 6.56, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.219679+00:00", "evaluated_item_count": 18, "score_count": 18, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.00119, "cost_basis": "actual_cash", "costed_scored_output_count": 18 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 6.56, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.139021+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.026809, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 6.33, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.090293+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.003521, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 6.33, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.179375+00:00", "evaluated_item_count": 18, "score_count": 18, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.015495, "cost_basis": "actual_cash", "costed_scored_output_count": 18 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "provider": "OpenAI", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 6.3, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.239254+00:00", "evaluated_item_count": 18, "score_count": 18, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.00549, "cost_basis": "actual_cash", "costed_scored_output_count": 18 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 6.11, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.013175+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.08906, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 6, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.043281+00:00", "evaluated_item_count": 18, "score_count": 18, "scored_run_count": 3, "mean_cost_per_scored_output_usd": 0.093626, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 18 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 5.78, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:27.126242+00:00", "evaluated_item_count": 12, "score_count": 12, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 0.029806, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 12 } ], "scaffold_comparison": { "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 9, "baseline_score_10": 5.9629, "scaffolded_score_10": 6, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 6.5553, "scaffolded_score_10": 6.6667, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 9, "baseline_score_10": 6.5926, "scaffolded_score_10": 6.148, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "status": "partial", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 6, "baseline_score_10": 5.7777, "scaffolded_score_10": 5.5, "unpaired_baseline_count": 3, "unpaired_scaffolded_count": 3, "unscored_comparison_unit_count": 3, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 8.111, "scaffolded_score_10": 8.111, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "status": "partial", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 7, "scaffolded_score_10": 7.3337, "unpaired_baseline_count": 6, "unpaired_scaffolded_count": 6, "unscored_comparison_unit_count": 6, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 9, "baseline_score_10": 6.1111, "scaffolded_score_10": 6.5556, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 9, "baseline_score_10": 6.2962, "scaffolded_score_10": 6.6294, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 6.3333, "scaffolded_score_10": 6.111, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 9, "baseline_score_10": 6.2962, "scaffolded_score_10": 6.185, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 9, "baseline_score_10": 5.5924, "scaffolded_score_10": 6.3332, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 6.111, "scaffolded_score_10": 5.889, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 6.7777, "scaffolded_score_10": 6.111, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6.1-sol", "model_label": "GPT-6.1 Sol", "status": "awaiting_scores", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 3, "unpaired_scaffolded_count": 3, "unscored_comparison_unit_count": 3, "invalid_comparison_unit_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 7.2223, "scaffolded_score_10": 7.111, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 15, "baseline_score_10": 6.4443, "scaffolded_score_10": 6.8444, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 6.5557, "scaffolded_score_10": 6.4443, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/ad-concept-generation/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T09:45:42.975Z", "methodology": { "label": "Ad concept generation", "summary": "Models receive the synthetic brand's product and audience context and generate three concepts per prompt condition. Ben scores whether those concepts are commercially promising, distinctive and coherent enough to test.", "scoringLabel": "Human-rated concept quality / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "The model receives brand guidelines for an invented brand without a reference ad. It generates three numbered concepts from that context. The six outputs in a complete baseline/scaffolded run are three concepts per condition, not six brands or campaigns." ] }, { "heading": "What changes between prompts", "paragraphs": [ "Both conditions receive the same brand context and line-specific inputs, with a 150-word concept limit. The simple prompt states the task. The scaffold adds judging criteria, an ad-quality framework and a hook/narrative/visual/conversion structure. It therefore tests that whole package of guidance, not a single wording change." ] }, { "heading": "How the result is scored", "paragraphs": [ "Ben blind-scores commercial potential, creative innovation, and coherence/suitability on a 0–10 scale. A concept clears the test-worthy threshold only when every dimension reaches 8. The displayed mean summarises quality; it does not measure the ad's eventual conversion rate. Scoring hides model labels, although the scaffold's formatting can reveal the prompt condition." ] } ] }, "content": { "title": "Best LLM for ad concept generation", "description": "Benmark tests whether an LLM can turn brand and audience context into advertising ideas worth producing.", "intro": "Benmark tests whether an LLM can turn brand and audience context into advertising ideas worth producing. Coherent concepts are easy to generate; commercial potential and distinctiveness separate the stronger work.", "findingTitle": "Judge the selling idea", "resultInterpretation": "The leading score provides a clear model to start with for this brand-context task. The wider spread of results shows why model selection matters before building concept generation into a routine.", "sections": [ { "heading": "A plausible idea still needs a reason to be made", "paragraphs": [ "An ad concept can name the right audience, mention the product and suggest a sensible format while giving someone very little reason to stop. Benmark's concept scoring includes commercial potential and creative innovation alongside coherence, so tidy output cannot carry the result on its own.", "The benchmark asks for three concepts from the same brand context. That makes the decision concrete: which routes would deserve a place in a production queue, and which merely rephrase the category?" ] }, { "heading": "Start with the strongest model, then improve the input", "paragraphs": [ "Claude Opus 5.5 reaches the same 8.11 score with the simple and scaffolded prompts in the reviewed paired comparison. For this model and pack, the structured framework did not raise the result. Choosing the model and giving it specific commercial context are the first decisions to get right.", "Useful context includes the buying situation, the objection the product resolves and the evidence available to support the claim. Those details let a concept move beyond a generic promise. A long brand document with no clear customer tension can still leave the model guessing about the most important part of the idea." ] }, { "heading": "Make selection part of the workflow", "paragraphs": [ "Use AI to develop a small set of distinct routes and make each route explain its audience tension, product role and opening execution. Then assess the routes against the same criteria. If removing the brand name makes them interchangeable, they need more development.", "This is a useful way to build a repeatable creative process around the model. The benchmark measures the quality of proposed concepts; testing the finished ads is the next stage of learning." ] } ], "related": [ { "path": "/benchmarks/reference-ad-adaptation/", "label": "Working from a reference ad" }, { "path": "/benchmarks/creative-production-planning/", "label": "Choosing what to produce next" } ] } }