{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_briefing_narrative_development", "scoring_mode": "hybrid", "score_status": "scored", "model_results": [ { "model_id": "claude-opus-4-7", "model_label": "Claude Opus 4.7", "provider": "Anthropic", "model_profile_ids": [ "thinking_high" ], "quality_score_10": 8.58, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T06:30:16.583917+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 3, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 8.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-21T08:53:46.442000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "provider": "Anthropic", "model_profile_ids": [ "thinking_high" ], "quality_score_10": 8.17, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T06:30:50.315746+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 3, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 8, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-21T08:40:05.847000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 8, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-05T09:09:29.915000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "provider": "OpenAI", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 8, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-21T00:39:56.665397+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 3, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 8, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-21T08:51:33.792000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 7.83, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-21T00:39:43.470365+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 3, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 7.75, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-23T11:52:37.118000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.1-flash-lite", "model_label": "Gemini 3.1 Flash-Lite", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 7.75, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T17:49:51.901253+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 3, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 7.75, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-21T08:42:19.867000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 7.75, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-05T13:36:10.380000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "provider": "Google via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 7.75, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-10-14T09:02:51.912000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-5.4-mini", "model_label": "GPT-5.4 Mini", "provider": "OpenAI", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 7.75, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-21T00:39:47.319539+00:00", "evaluated_item_count": 6, "score_count": 6, "scored_run_count": 3, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 7.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-05T13:38:36.465000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 7.5, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-23T08:20:59.537000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 7.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-08T16:18:03.858000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "provider": "Google Gemini", "model_profile_ids": [], "quality_score_10": 6.86, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-05-25T12:35:19.963058+00:00", "evaluated_item_count": 14, "score_count": 14, "scored_run_count": 2, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 6.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-10-14T05:25:02.380000+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 } ], "scaffold_comparison": { "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 8, "scaffolded_score_10": 8, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.5, "scaffolded_score_10": 8, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-7", "model_label": "Claude Opus 4.7", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 8.5833, "scaffolded_score_10": 8, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 8.1667, "scaffolded_score_10": 8.0833, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 8, "scaffolded_score_10": 8.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.75, "scaffolded_score_10": 7.75, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-flash-lite", "model_label": "Gemini 3.1 Flash-Lite", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 7.6667, "scaffolded_score_10": 7.75, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 7.8333, "scaffolded_score_10": 7.6667, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 7, "baseline_score_10": 6.3214, "scaffolded_score_10": 6.8571, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.75, "scaffolded_score_10": 7.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.75, "scaffolded_score_10": 7, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.4-mini", "model_label": "GPT-5.4 Mini", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 7.75, "scaffolded_score_10": 7.25, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 3, "baseline_score_10": 8, "scaffolded_score_10": 7.8333, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.5, "scaffolded_score_10": 8, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.25, "scaffolded_score_10": 7.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.5, "scaffolded_score_10": 7.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 7.75, "scaffolded_score_10": 7.75, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 6.75, "scaffolded_score_10": 7.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 6.25, "scaffolded_score_10": 6.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/ad-narrative-development/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T08:22:38.946Z", "methodology": { "label": "Ad narrative development", "summary": "Models develop an ad's story from the fixed creative input and brand context. Ben blind-scores coherence, commercial persuasion, emotional levers and product relevance.", "scoringLabel": "Human-rated narrative quality / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "The concept-to-brief pack fixes the creative starting point so models can be compared on developing the narrative. Coverage differs by model and repeat count; repeated versions of this pack do not represent independent audiences or brands." ] }, { "heading": "What changes between prompts", "paragraphs": [ "The same fixed creative input and brand context are used with plain and scaffolded-context prompts. The structured condition adds guidance for developing the output. The publisher keeps hooks, narratives and compliant briefs as separate tasks, with their own scores." ] }, { "heading": "How the result is scored", "paragraphs": [ "The narrative score averages coherence, commercial persuasion, use of emotional levers and product relevance. This measures the written narrative. The hook and the production handover have separate task scores, so a high narrative result is not inherited by the whole workflow." ] } ] }, "content": { "title": "Best LLM for developing ad narratives", "description": "Benmark tests whether a model can develop an ad's story with a clear progression, persuasive product role and useful emotional tension.", "intro": "Benmark tests whether a model can develop an ad's story with a clear progression, persuasive product role and useful emotional tension. A strong opening needs a convincing route to the payoff.", "findingTitle": "Make the product necessary to the story", "resultInterpretation": "The strongest narrative result reaches a useful standard on the current creative pack. The opportunity is to turn an agreed direction into a developed story whose product role holds together from setup to payoff.", "sections": [ { "heading": "Watch the transition into the product", "paragraphs": [ "The weak point in many ad narratives is the moment the product arrives. The opening sets up an interesting situation, then the script switches into a list of benefits that could belong to another ad.", "A stronger narrative makes the product relevant to the tension already established. The demonstration, proof or change in the character's situation should resolve something the opening made the viewer care about. That is why product relevance is scored alongside coherence and persuasion." ] }, { "heading": "Separate the story decision from the wording", "paragraphs": [ "Before polishing individual lines, make the model explain the progression: what the viewer recognises, what changes their understanding, where the product becomes useful and what supports the final action.", "This gives you a way to reject a weak route early. If the model needs several paragraphs to explain why the product belongs, improving the dialogue is unlikely to fix the underlying connection." ] }, { "heading": "Carry the narrative into production", "paragraphs": [ "Once the route works, develop the scenes or beats around it. Keep the intended emotional change and product role explicit so a tighter edit does not accidentally remove the persuasion.", "The narrative benchmark and production-brief benchmark measure different jobs. Use the former to choose a model for developing the story, then use the latter for the structured handover. A well-organised brief can preserve a story, but its formatting cannot make a weak one compelling." ] } ], "related": [ { "path": "/benchmarks/ad-hook-writing/", "label": "Writing the opening hook" }, { "path": "/benchmarks/production-brief-generation/", "label": "Preparing a production brief" } ] } }