{ "schemaVersion": "benmark-task-evidence.v1", "task": { "task_id": "os_production_video_generation", "scoring_mode": "hybrid", "score_status": "scored", "model_results": [ { "model_id": "google-gemini-omni-flash", "model_label": "Gemini Omni Flash", "provider": "Google Gemini", "model_profile_ids": [ "default" ], "quality_score_10": 6.68, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.923989+00:00", "evaluated_item_count": 10, "score_count": 10, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "cost_basis": null, "costed_scored_output_count": 0 }, { "model_id": "runway-gen-4-5", "model_label": "Gen-4.5", "provider": "Runway", "model_profile_ids": [ "default" ], "quality_score_10": 5.24, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.931396+00:00", "evaluated_item_count": 10, "score_count": 10, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "cost_basis": null, "costed_scored_output_count": 0 }, { "model_id": "runway-seedance-2", "model_label": "Seedance 2", "provider": "Runway / ByteDance", "model_profile_ids": [ "default" ], "quality_score_10": 4.56, "selected_prompt_arm": "scaffolded", "publication_status": "provisional", "evidence_maturity": "provisional", "last_tested_at": "2026-10-16T00:32:26.938575+00:00", "evaluated_item_count": 10, "score_count": 10, "scored_run_count": 1, "mean_cost_per_scored_output_usd": null, "cost_basis": null, "costed_scored_output_count": 0 } ], "scaffold_comparison": { "model_results": [ { "model_id": "google-gemini-omni-flash", "model_label": "Gemini Omni Flash", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 5, "baseline_score_10": 6, "scaffolded_score_10": 6.68, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "runway-gen-4-5", "model_label": "Gen-4.5", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 5, "baseline_score_10": 4.52, "scaffolded_score_10": 5.24, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 }, { "model_id": "runway-seedance-2", "model_label": "Seedance 2", "status": "complete", "publication_status": "provisional", "comparison_metric_key": "quality_score_10", "pairing_basis": "explicit_shared_identity", "comparison_unit_count": 5, "baseline_score_10": 4.4, "scaffolded_score_10": 4.56, "unpaired_baseline_count": 0, "unpaired_scaffolded_count": 0, "unscored_comparison_unit_count": 0, "invalid_comparison_unit_count": 0 } ] } }, "path": "/benchmarks/ai-video-ad-generation/", "sourcePublication": "abfb9707d19b695fb05c1694d46222ca", "datePublished": "2026-10-16T08:22:38.946Z", "reviewedAt": "2026-10-16T08:22:38.946Z", "methodology": { "label": "AI video ad generation", "summary": "Video models create complete eight-second vertical ads from five briefs. Both prompt conditions use text only, and Ben scores the unedited video; audio is excluded.", "scoringLabel": "Human-rated video quality / 10", "scoreUnit": "score", "sections": [ { "heading": "What the model receives", "paragraphs": [ "The briefs span UGC, founder, lifestyle, explainer and product-demonstration formats. The shared contract requests an eight-second 9:16 video at 720×1280. No reference images are supplied because providers interpret them differently. Five briefs in two conditions produce ten outputs per run." ] }, { "heading": "What changes between prompts", "paragraphs": [ "The simple and scaffolded prompts share the same task. The scaffold adds concise ad-anatomy guidance within the common prompt limit. Video duration, geometry and decodability are checked before the result is assessed; scoring playback is muted." ] }, { "heading": "How the result is scored", "paragraphs": [ "Raw outputs are human-scored for product fidelity, copy accuracy, claims safety, brief compliance and craft quality. Each dimension must reach 8 for an output to clear the usability threshold. The displayed mean is a comparison measure; it can conceal a weak individual dimension. No editing pass is added before scoring." ] } ] }, "content": { "title": "Best AI models for video ad generation", "description": "Benmark compares complete eight-second vertical ads from five fixed briefs.", "intro": "Benmark compares complete eight-second vertical ads from five fixed briefs. The test makes the production challenge visible: a model has to sustain the product, message and execution across the whole clip.", "findingTitle": "Choose by the complete sequence", "resultInterpretation": "The leading model is the strongest starting point in this reviewed panel. The scores also show why video generation needs a different production plan from image generation: the whole sequence has to hold together.", "sections": [ { "heading": "Motion multiplies the decisions", "paragraphs": [ "A static image has to resolve one composition. A video has to preserve the product and message while the scene, camera or person changes. A strong opening frame cannot rescue a sequence that loses the product or fails to deliver the intended demonstration.", "The benchmark therefore scores the complete unedited output. It covers UGC, founder, lifestyle, explainer and product-demonstration briefs, giving a more useful comparison than selecting each model's most impressive frame." ] }, { "heading": "Use a genuinely shared input contract", "paragraphs": [ "The tested video setup is text only. Reference images were excluded because providers use them differently: a first frame and a general visual reference impose different tasks. All models receive a short common brief for an eight-second vertical output.", "Audio is excluded from scoring. The result should guide model choice for the visual sequence under that contract. If your process depends on a fixed first frame, dialogue or several stitched shots, evaluate that production setup on its own terms." ] }, { "heading": "Build the production process around the failure", "paragraphs": [ "Use an initial output to assess which part of the brief the model can execute: the product demonstration, the setting, the motion or the transition. Then decide whether the next step is a clearer brief, a narrower shot or conventional editing.", "The benchmark has not measured a multi-shot edited workflow. It gives you evidence for choosing the generation model and a concrete set of review dimensions, which is a better starting point than assuming a strong demo clip will reproduce reliably on your brief." ] } ], "related": [ { "path": "/benchmarks/ai-static-ad-generation/", "label": "Comparing static image generation" }, { "path": "/benchmarks/ad-creative-review/", "label": "Checking the generated asset" } ] } }