{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_closing_insight_codification", "scoring_mode": "automated", "score_status": "scored", "model_results": [ { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 8.25, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-07-04T00:31:02.639442+00:00", "evaluated_item_count": 0, "score_count": 6, "scored_run_count": 5, "mean_cost_per_scored_output_usd": 0.01054, "cost_basis": "actual_cash", "costed_scored_output_count": 6 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 7.71, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-12T18:48:25.354669+00:00", "evaluated_item_count": 0, "score_count": 4, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 0.385612, "cost_basis": "actual_cash", "costed_scored_output_count": 4 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 7.57, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T13:38:55.398669+00:00", "evaluated_item_count": 0, "score_count": 12, "scored_run_count": 6, "mean_cost_per_scored_output_usd": 0.008742, "cost_basis": "actual_cash", "costed_scored_output_count": 12 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 7.56, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-23T09:02:43.366734+00:00", "evaluated_item_count": 0, "score_count": 4, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 2.021688, "cost_basis": "mixed_actual_cash_and_api_equivalent_estimate", "costed_scored_output_count": 4 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "provider": "Meta via OpenRouter", "model_profile_ids": [ "max" ], "quality_score_10": 7.44, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-08T12:00:10.811210+00:00", "evaluated_item_count": 0, "score_count": 4, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 0.092736, "cost_basis": "actual_cash", "costed_scored_output_count": 4 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "provider": "Google via OpenRouter", "model_profile_ids": [ "high" ], "quality_score_10": 7.44, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-13T19:46:14.988385+00:00", "evaluated_item_count": 0, "score_count": 4, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 0.282347, "cost_basis": "actual_cash", "costed_scored_output_count": 4 }, { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 7.44, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T14:24:17.450500+00:00", "evaluated_item_count": 0, "score_count": 12, "scored_run_count": 6, "mean_cost_per_scored_output_usd": 1.079908, "cost_basis": "api_equivalent_estimate", "costed_scored_output_count": 12 }, { "model_id": "gemini-3.6-flash", "model_label": "Gemini 3.6 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 6.99, "selected_prompt_arm": "pooled", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-15T14:34:34.344030+00:00", "evaluated_item_count": 0, "score_count": 12, "scored_run_count": 12, "mean_cost_per_scored_output_usd": 0.021017, "cost_basis": "actual_cash", "costed_scored_output_count": 11 }, { "model_id": "gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "provider": "Google Gemini", "model_profile_ids": [], "quality_score_10": 6.76, "selected_prompt_arm": "pooled", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-20T09:11:32.358890+00:00", "evaluated_item_count": 12, "score_count": 12, "scored_run_count": 6, "mean_cost_per_scored_output_usd": null, "cost_basis": null, "costed_scored_output_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 6.65, "selected_prompt_arm": "baseline", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-23T05:44:06.472637+00:00", "evaluated_item_count": 0, "score_count": 4, "scored_run_count": 2, "mean_cost_per_scored_output_usd": 0.097501, "cost_basis": "actual_cash", "costed_scored_output_count": 4 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 6.51, "selected_prompt_arm": "pooled", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-15T22:11:19.570566+00:00", "evaluated_item_count": 0, "score_count": 12, "scored_run_count": 12, "mean_cost_per_scored_output_usd": 0.497965, "cost_basis": "actual_cash", "costed_scored_output_count": 12 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 6.5, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-05T12:13:26.213649+00:00", "evaluated_item_count": 0, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.59542, "cost_basis": "actual_cash", "costed_scored_output_count": 2 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "provider": "OpenAI", "model_profile_ids": [ "xhigh" ], "quality_score_10": 5.77, "selected_prompt_arm": "pooled", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-08-15T21:38:20.879683+00:00", "evaluated_item_count": 0, "score_count": 12, "scored_run_count": 12, "mean_cost_per_scored_output_usd": 0.219021, "cost_basis": "actual_cash", "costed_scored_output_count": 12 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "provider": "OpenAI", "model_profile_ids": [ "reasoning_high", "reasoning_medium" ], "quality_score_10": 5.73, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-07-03T21:32:39.431310+00:00", "evaluated_item_count": 0, "score_count": 8, "scored_run_count": 7, "mean_cost_per_scored_output_usd": 0.03656, "cost_basis": "actual_cash", "costed_scored_output_count": 2 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "provider": "Anthropic", "model_profile_ids": [ "thinking_high" ], "quality_score_10": 5.64, "selected_prompt_arm": "pooled", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-07-03T20:19:22.268658+00:00", "evaluated_item_count": 0, "score_count": 6, "scored_run_count": 6, "mean_cost_per_scored_output_usd": 0.242553, "cost_basis": "actual_cash", "costed_scored_output_count": 2 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "provider": "Anthropic", "model_profile_ids": [ "thinking_max" ], "quality_score_10": 5.62, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-01T21:23:49.144274+00:00", "evaluated_item_count": 0, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "cost_basis": null, "costed_scored_output_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "provider": "Google Gemini", "model_profile_ids": [ "reasoning_high" ], "quality_score_10": 5.25, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-09-04T05:39:41.192217+00:00", "evaluated_item_count": 0, "score_count": 2, "scored_run_count": 1, "mean_cost_per_scored_output_usd": 0.060869, "cost_basis": "actual_cash", "costed_scored_output_count": 2 } ], "scaffold_comparison": { "model_results": [ { "model_id": "claude-fable-5", "model_label": "Claude Fable 5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 6, "baseline_score_10": 5.8963, "scaffolded_score_10": 7.4442, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-fable-5-1", "model_label": "Claude Fable 5.1", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 4.792, "scaffolded_score_10": 5.625, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-4-8", "model_label": "Claude Opus 4.8", "status": "invalid", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "no_stable_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 3, "unpaired_scaffolded_count": 3, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5", "model_label": "Claude Opus 5", "status": "invalid", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "no_stable_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 6, "unpaired_scaffolded_count": 6, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "claude-opus-5-5", "model_label": "Claude Opus 5.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 2, "baseline_score_10": 5.865, "scaffolded_score_10": 7.5625, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.5-flash", "model_label": "Gemini 3.5 Flash", "status": "partial", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 9.25, "scaffolded_score_10": 10, "unpaired_baseline_count": 2, "unpaired_scaffolded_count": 2, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.6-flash", "model_label": "Gemini 3.6 Flash", "status": "invalid", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "no_stable_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 6, "unpaired_scaffolded_count": 6, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 6, "baseline_score_10": 6.896, "scaffolded_score_10": 7.5693, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 4.792, "scaffolded_score_10": 5.25, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.5", "model_label": "GPT-5.5", "status": "partial", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 3.834, "scaffolded_score_10": 4.834, "unpaired_baseline_count": 3, "unpaired_scaffolded_count": 3, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "status": "invalid", "publication_status": "withheld", "comparison_metric_key": "quality_score_10", "pairing_basis": "no_stable_shared_identity", "comparison_unit_count": 0, "baseline_score_10": null, "scaffolded_score_10": null, "unpaired_baseline_count": 6, "unpaired_scaffolded_count": 6, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-astra", "model_label": "GPT-6 Astra", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 1, "baseline_score_10": 3.23, "scaffolded_score_10": 6.5, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "gpt-6-sol", "model_label": "GPT-6 Sol", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 2, "baseline_score_10": 6.646, "scaffolded_score_10": 5.9795, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-google-gemini-4-argon", "model_label": "Google Gemini 4 Argon", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 2, "baseline_score_10": 4.521, "scaffolded_score_10": 7.4375, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-spark-1-3", "model_label": "Meta Muse Spark 1.3", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 2, "baseline_score_10": 7.125, "scaffolded_score_10": 7.4375, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "or-meta-muse-wonder", "model_label": "Meta Muse Wonder", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 2, "baseline_score_10": 5.271, "scaffolded_score_10": 7.7075, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/campaign-insight-memory/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T08:22:38.946Z", "methodology": { "label": "Campaign insight memory", "summary": "Models extract reusable learning from synthetic campaign results and update an existing insight library when new evidence arrives. The scorer checks genuine signals, misleading patterns, confidence and the correct memory-update decisions.", "scoringLabel": "Deterministic insight fidelity / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "The signal-extraction pack contains 22 ads with known signals, decoys, low-evidence traps and a data-quality issue. The memory-update pack contains six existing insights and a ten-ad follow-up digest with eight decision points. These are two controlled cases; the model rows have different coverage and repeat counts." ] }, { "heading": "What changes between prompts", "paragraphs": [ "The baseline states the task and output shape. The scaffold adds evidence floors, a confound checklist, calibration rules and an update vocabulary. Both use the same campaign evidence; the evaluator-only answers are withheld." ] }, { "heading": "How the result is scored", "paragraphs": [ "The deterministic composite checks signal recovery, false insights, confidence, traps and memory updates against the case key. It rewards retaining, narrowing or replacing a learning when the evidence warrants it. It does not score fluent summary writing or measure a downstream campaign lift." ] } ] }, "content": { "title": "Best LLM for building campaign insight memory", "description": "Benmark tests whether models can extract genuine campaign learning and revise it when new evidence arrives.", "intro": "Benmark tests whether models can extract genuine campaign learning and revise it when new evidence arrives. Useful memory keeps the evidence, confidence and conditions attached to each insight.", "findingTitle": "Build memory that can change its mind", "resultInterpretation": "The strongest published result is a useful starting point for maintaining campaign learning. The important capability is deciding what to keep, narrow, update or reject, so yesterday's conclusion does not become tomorrow's unquestioned instruction.", "sections": [ { "heading": "A learning needs a scope", "paragraphs": [ "A sentence such as 'demonstrations work best' can travel much further than the evidence behind it. It may have come from one audience, one offer or an unusually strong execution. Once copied into future prompts, the broad version can steer every new concept.", "Store the claim with its supporting evidence, the situation in which it applies and the condition that would trigger a retest. That makes the learning useful to another model without requiring it to reconstruct the original campaign discussion." ] }, { "heading": "Updating is part of the task", "paragraphs": [ "Benmark tests both extracting signals from a campaign cycle and updating an existing memory from a new digest. The cases include genuine patterns, confounds, weak evidence and stale entries, so a fluent summary is not enough to score well.", "The scaffold supplies evidence floors, calibration rules and explicit update actions. In the reviewed Google Gemini 4 Argon comparison, its score rises from 4.52 to 7.44 across two paired observations. Those observations cover the tested cases, and show why evidence-handling instructions are worth testing for this workflow." ] }, { "heading": "Make the next creative decision use the memory", "paragraphs": [ "An insight library becomes valuable when it changes the work. Ask a new creative proposal to identify the learning it uses, and make a production plan explain when it is departing from previous evidence.", "Keep the original source link and the update history with the insight. If new evidence contradicts an earlier result, the useful action may be to narrow the claim rather than delete it or defend it unchanged. That preserves what was learned while making room for a better decision." ] } ], "related": [ { "path": "/benchmarks/campaign-insights-to-creative/", "label": "Using learning in new creative" }, { "path": "/benchmarks/campaign-performance-analysis/", "label": "Finding the campaign evidence" } ] } }