{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_briefing_production_brief_generation", "scoring_mode": "hybrid", "score_status": "scored", "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 9.5, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-23T10:14:09+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 9.5, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-09T08:42:27+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 9.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-23T10:13:17+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 9.5, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-23T13:46:25+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 9.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-09T08:43:34+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 9.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-23T10:37:31+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 9.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-09T08:45:08+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 9.5, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-23T11:39:44+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 9.5, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-09-09T08:45:57+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 9.5, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-10-14T10:52:29+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "provider": "Anthropic", "model_profile_ids": [ "thinking_high" ], "quality_score_10": 9.39, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T20:41:48.051868+00:00", "evaluated_item_count": 12, "score_count": 12, "scored_run_count": 6, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "provider": "OpenAI", "model_profile_ids": [], "quality_score_10": 9.38, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-05-26T13:36:10.887543+00:00", "evaluated_item_count": 20, "score_count": 20, "scored_run_count": 2, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "provider": "Google Gemini", "model_profile_ids": [], "quality_score_10": 9.37, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-05-26T08:39:56.453177+00:00", "evaluated_item_count": 10, "score_count": 10, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 9.17, "selected_prompt_arm": "scaffolded", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-08-23T10:39:34+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "provider": "Google via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 9.17, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-10-14T10:51:32+00:00", "evaluated_item_count": 2, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "provider": "Google Gemini", "model_profile_ids": [], "quality_score_10": 8.81, "selected_prompt_arm": "baseline", "publication_status": "published", "evidence_maturity": "published", "last_tested_at": "2026-07-29T19:41:38.110882+00:00", "evaluated_item_count": 14, "score_count": 14, "scored_run_count": 2, "mean_cost_per_scored_output_usd": null, "costed_scored_output_count": 0 } ], "scaffold_comparison": { "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.5, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.5, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 6, "baseline_score_10": 9.17, "scaffolded_score_10": 9.39, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 6.83, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.5, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 5, "baseline_score_10": 9.102, "scaffolded_score_10": 9.368, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 7, "baseline_score_10": 8.8086, "scaffolded_score_10": 8.6671, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 8.5, "scaffolded_score_10": 9.17, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 8.5, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 10, "baseline_score_10": 9.351, "scaffolded_score_10": 9.384, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.17, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.17, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.5, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.17, "scaffolded_score_10": 8, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.17, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.5, "scaffolded_score_10": 9.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/production-brief-generation/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T08:22:38.946Z", "methodology": { "label": "Production brief generation", "summary": "Models turn a fixed concept and brand/production constraints into a structured brief. The public task score measures brief quality with a compliance gate; hook and narrative quality are assessed separately.", "scoringLabel": "Compliance-gated brief quality / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "The input supplies the creative starting point, brand context and production constraints. The output should give production a grounded, actionable handover. Recorded model rows can combine more than one completed cycle; the results table preserves their output counts." ] }, { "heading": "What changes between prompts", "paragraphs": [ "The same fixed creative input and brand context are used with plain and scaffolded-context prompts. The structured condition adds guidance for developing the output. The publisher keeps hooks, narratives and compliant briefs as separate tasks, with their own scores." ] }, { "heading": "How the result is scored", "paragraphs": [ "The published compliant-brief task uses the brief-quality component and applies compliance as a gate. Its scorer is AI-assisted within the reviewed scoring process. Human hook and narrative scores exist in the wider brief evaluation but are deliberately excluded from this task's headline. A 9.5 here therefore describes the handover, not a 9.5 advertising idea." ] } ] }, "content": { "title": "Best LLM for production brief generation", "description": "Benmark tests whether AI can turn a creative starting point and brand constraints into a useful production handover.", "intro": "Benmark tests whether AI can turn a creative starting point and brand constraints into a useful production handover. This is one of the clearest places to separate the quality of the idea from the quality of its execution brief.", "findingTitle": "AI is strong at the production handover", "resultInterpretation": "The joint leaders score highly on compliant brief quality. Once the creative direction is chosen, those setups are useful candidates for turning it into instructions a production team can act on.", "sections": [ { "heading": "The brief should resolve production decisions", "paragraphs": [ "A brief earns its place when it makes the next person's work clearer. It should establish the intended audience, product role, proof, opening execution and practical requirements well enough that production can begin.", "An apparently complete document can still leave the most important choice unresolved. If the opening is central to the idea, saying 'start with an engaging hook' passes that work straight to production. Ask for the actual opening or an execution specific enough to make it." ] }, { "heading": "Keep the creative judgment visible", "paragraphs": [ "The public score for this task measures brief quality with a compliance gate. Hook and narrative quality are assessed separately. That distinction makes the result useful: a strong handover score tells you where AI can help without treating the document's completeness as evidence that the ad idea will sell.", "Choose the route first, then use the model to develop the handover around it. Carry the source evidence for claims and the production constraints into the prompt so the model can resolve detail without inventing proof." ] }, { "heading": "Check the decisions that cost money to reverse", "paragraphs": [ "Focus the final review on claims, required assets, production feasibility and any instruction that would lead to a different execution if misunderstood. Cosmetic wording is much cheaper to fix than a shoot built around an unsupported promise.", "The benchmark provides a shortlist of strong brief-writing setups. A useful next step in your own workflow is to have the production team identify the questions the brief leaves open, then improve the shared briefing instructions around those recurring gaps." ] } ], "related": [ { "path": "/benchmarks/ad-concept-generation/", "label": "Choosing the creative idea" }, { "path": "/benchmarks/ad-narrative-development/", "label": "Developing its narrative" } ] } }