{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_analysis_diversity_coverage", "scoring_mode": "hybrid", "score_status": "scored", "model_results": [ { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 10, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-09T14:25:41.012247+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.506977, "cost_basis": "actual_cash", "costed_scored_output_count": 1 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T08:11:49.522451+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.542426, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "provider": "Meta via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-08T14:25:26.920216+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.556868, "cost_basis": "actual_cash", "costed_scored_output_count": 1 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 10, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T08:02:24.508652+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 1.283119, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 10, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T07:50:38.440014+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 1.303566, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-23T06:03:15.666712+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 1.673627, "cost_basis": "actual_cash", "costed_scored_output_count": 1 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "provider": "Google via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 10, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-13T19:39:55.926205+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 1.928141, "cost_basis": "actual_cash", "costed_scored_output_count": 1 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "provider": "OpenAI", "model_profile_ids": [ "reasoning_medium" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T07:49:09.807595+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 2.363958, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 10, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-09T14:40:45.540056+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 3.12888, "cost_basis": "actual_cash", "costed_scored_output_count": 1 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T07:52:36.561210+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 3.461027, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T08:09:39.843811+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 3.534129, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T08:13:53.968078+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 4.053496, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 10, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-13T07:31:27.940540+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 4.088888, "cost_basis": "actual_cash", "costed_scored_output_count": 1 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-09T14:23:29.734040+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 7.21519, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-29T08:51:49.928688+00:00", "evaluated_item_count": 35, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 12.939499, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 10, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-23T09:08:30.553374+00:00", "evaluated_item_count": 1, "score_count": 1, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 14.538813, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 1 } ], "scaffold_comparison": { "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 8.335, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 6.67, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 8.335, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "status": "complete", "publication_status": "published", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 6.67, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 8.335, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 10, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "sealed_paired_aggregate", "comparison_unit_count": 2, "baseline_score_10": 8.335, "scaffolded_score_10": 10, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/creative-diversity-analysis/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T08:22:38.946Z", "methodology": { "label": "Creative diversity analysis", "summary": "Models analyse the same fixed account's reviewed creative labels and identify concentrations, gaps and useful next tests. The benchmark compares the analysis layer; it does not combine tagging accuracy with the headline analysis score.", "scoringLabel": "Reviewed diversity-analysis task score / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "The account pack contains 35 files, including one exact duplicate, representing 34 unique assets. Both analysis conditions receive the fixed reviewed taxonomy records. One account is one case even when the model inspects many asset rows." ] }, { "heading": "What changes between prompts", "paragraphs": [ "The simple and scaffolded analysis prompts use the same labelled portfolio. The scaffold structures the concentration, risk, gap and recommendation assessment. The earlier classification/instrument workflow is separate from this comparison." ] }, { "heading": "How the result is scored", "paragraphs": [ "The evaluation checks analysis against an answer key, including numerical consistency, real risks, unsupported risks, gaps and deliberately absent categories. Recommendation quality and operator readiness have a separate judged layer. Read the published score as performance on this fixed analysis task, rather than proof that an arbitrary portfolio can be classified and planned end to end." ] } ] }, "content": { "title": "Best LLM for creative diversity analysis", "description": "Benmark tests whether models can turn a labelled creative portfolio into a useful view of concentrations, gaps and next tests.", "intro": "Benmark tests whether models can turn a labelled creative portfolio into a useful view of concentrations, gaps and next tests. The value comes from deciding which differences matter.", "findingTitle": "Use the portfolio view to find useful questions", "resultInterpretation": "Many models reach the top score on the fixed analysis case. That makes the supplied labels and the interpretation of the gaps central to using this capability well.", "sections": [ { "heading": "More variation is not automatically better coverage", "paragraphs": [ "A portfolio can contain many different-looking ads that repeat the same promise, proof and buyer situation. Conversely, a small set of assets can test meaningfully different propositions.", "Analyse dimensions that represent actual creative choices. Changing the background colour creates visual variation; changing the demonstration or objection addressed can test a different reason to buy. The taxonomy needs to preserve that distinction." ] }, { "heading": "Keep classification and analysis separate", "paragraphs": [ "In this benchmark, both prompt conditions receive the same reviewed labels for one account. The test measures the analysis built on those labels. This makes it possible to inspect the model's reasoning without mixing in a different set of tagging errors for every model.", "In your workflow, establish whether the important labels are right before acting on the gaps. A missing category in the report might be a genuine absence or a classification problem. Those require different next steps." ] }, { "heading": "Take the gap back to performance", "paragraphs": [ "Use the diversity report to identify a small number of questions: which proposition is repeated, which buying situation is absent and which creative mechanism has little coverage. Then compare those questions with performance and the existing insight library.", "Benmark's production-planning task tests that reconciliation explicitly. It is the next useful step because a gap alone is not a sufficient reason to commission an ad. The plan should explain what filling it would help the team learn." ] } ], "related": [ { "path": "/benchmarks/creative-tagging/", "label": "Building the labels" }, { "path": "/benchmarks/creative-production-planning/", "label": "Turning gaps into a production plan" } ] } }