{ "comparison_policy": "newest_exact_scenario_cohort", "fleet": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "tiers": [ "core", "full" ] } ], "generated_from": "20260924.0.0", "profiles": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "records": [ { "cost_per_repair_usd": 0.002234795724, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=001-nginx-502-host&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 51337, "known_cost_usd": 0.006704387172, "median_duration_seconds": 57, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 31, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 64, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 57, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 14727, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "001-nginx-502-host", "scenario_label": "001-nginx-502-host", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.003945461184, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=013-sidekiq-wrong-redis&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 1, "failure_categories": { "agent_runtime_error": 1 }, "input_mode": "text", "input_tokens": 56672, "known_cost_usd": 0.007890922368, "median_duration_seconds": 78, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 86, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 67, "failure_category": "agent_runtime_error", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 78, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 16096, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "013-sidekiq-wrong-redis", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.0109148485464, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=014-missing-rails-migration&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 625375, "known_cost_usd": 0.0327445456392, "median_duration_seconds": 165, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 256, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 165, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 140, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 19239, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "014-missing-rails-migration", "scenario_label": "014-missing-rails-migration", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.015131050157999999, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=015-sidekiq-poison-pill&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 1, "failure_categories": { "backlog_not_recovered": 1 }, "input_mode": "text", "input_tokens": 251227, "known_cost_usd": 0.030262100315999997, "median_duration_seconds": 206, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 86, "failure_category": "backlog_not_recovered", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 248, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 206, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 59855, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "015-sidekiq-poison-pill", "scenario_label": "015-sidekiq-poison-pill", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.009839326245333332, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=016-rails-pool-exhaustion&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 748301, "known_cost_usd": 0.029517978735999997, "median_duration_seconds": 229, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 234, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 226, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 229, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 24332, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "016-rails-pool-exhaustion", "scenario_label": "016-rails-pool-exhaustion", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.00609388647, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=017-partial-rails-rollout&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 79157, "known_cost_usd": 0.01218777294, "median_duration_seconds": 104, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 104, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 138, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 19, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 24236, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "017-partial-rails-rollout", "scenario_label": "017-partial-rails-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.02251468666666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=018-node-event-loop-blocking&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 845852, "known_cost_usd": 0.06754406, "median_duration_seconds": 348, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 406, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 348, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 243, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 27036, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "018-node-event-loop-blocking", "scenario_label": "018-node-event-loop-blocking", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.007688253467999999, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=019-rust-fd-leak&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 93292, "known_cost_usd": 0.015376506935999998, "median_duration_seconds": 110, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 97, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 118, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 110, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 33683, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "019-rust-fd-leak", "scenario_label": "019-rust-fd-leak", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.006941929666666666, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=020-python-gunicorn-saturation&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 678392, "known_cost_usd": 0.020825788999999997, "median_duration_seconds": 244, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 263, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 244, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 126, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 25799, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "020-python-gunicorn-saturation", "scenario_label": "020-python-gunicorn-saturation", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.0093130233, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=021-discourse-shared-uploads&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 132939, "known_cost_usd": 0.027939069900000002, "median_duration_seconds": 163, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 163, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 229, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 102, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 66729, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "021-discourse-shared-uploads", "scenario_label": "021-discourse-shared-uploads", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.006527065000000001, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=022-discourse-multisite-migration&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 480764, "known_cost_usd": 0.019581195000000003, "median_duration_seconds": 112, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 112, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 83, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 124, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 13451, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "022-discourse-multisite-migration", "scenario_label": "022-discourse-multisite-migration", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.0131230928, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=023-auth-secret-rollout&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 838112, "known_cost_usd": 0.0393692784, "median_duration_seconds": 165, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 224, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 165, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 114, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 23926, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "023-auth-secret-rollout", "scenario_label": "023-auth-secret-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.010458549906, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=024-discourse-interrupted-deploy&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 1, "failure_categories": { "release_not_converged": 1 }, "input_mode": "text", "input_tokens": 124627, "known_cost_usd": 0.020917099812, "median_duration_seconds": 98, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 96, "failure_category": "release_not_converged", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 98, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 118, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 31434, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "024-discourse-interrupted-deploy", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.030954798000000002, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 1751314, "known_cost_usd": 0.030954798000000002, "median_duration_seconds": 1752, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 1752, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 27737, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.020622831700000004, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=026-discourse-plugin-boot-loop&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 1, "failure_categories": { "core_api_rolled_back": 1 }, "input_mode": "text", "input_tokens": 747731, "known_cost_usd": 0.04124566340000001, "median_duration_seconds": 149, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 181, "failure_category": "core_api_rolled_back", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 149, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 137, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 32645, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "026-discourse-plugin-boot-loop", "scenario_label": "026-discourse-plugin-boot-loop", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.009406546600000001, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=027-partial-service-rollout&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 702330, "known_cost_usd": 0.0282196398, "median_duration_seconds": 228, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 228, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 170, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 236, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 21265, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "027-partial-service-rollout", "scenario_label": "027-partial-service-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.008676288095999999, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=028-nix-store-disk-pressure&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 126860, "known_cost_usd": 0.026028864287999998, "median_duration_seconds": 155, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 167, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 149, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 155, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 63126, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "028-nix-store-disk-pressure", "scenario_label": "028-nix-store-disk-pressure", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} } ], "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "records": [ { "cost_per_repair_usd": 0.23538495, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=001-nginx-502-host&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 566289, "known_cost_usd": 0.70615485, "median_duration_seconds": 130, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 119, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 167, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 130, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 28346, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "001-nginx-502-host", "scenario_label": "001-nginx-502-host", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.21407542500000004, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=013-sidekiq-wrong-redis&model=google%2Fgemini-3.7-flash", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 396226, "known_cost_usd": 0.4281508500000001, "median_duration_seconds": 123, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 85, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 134, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 123, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 16541, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "013-sidekiq-wrong-redis", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.152430725, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=014-missing-rails-migration&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 4571848, "known_cost_usd": 0.45729217499999997, "median_duration_seconds": 185, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 191, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 185, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 152, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 25941, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "014-missing-rails-migration", "scenario_label": "014-missing-rails-migration", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.38200247499999995, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=015-sidekiq-poison-pill&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 745566, "known_cost_usd": 1.1460074249999999, "median_duration_seconds": 344, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 344, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 457, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 330, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 73013, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "015-sidekiq-poison-pill", "scenario_label": "015-sidekiq-poison-pill", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.20002136249999994, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=016-rails-pool-exhaustion&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 5187756, "known_cost_usd": 0.6000640874999998, "median_duration_seconds": 252, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 252, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 217, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 323, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 39516, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "016-rails-pool-exhaustion", "scenario_label": "016-rails-pool-exhaustion", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.5986506, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=017-partial-rails-rollout&model=google%2Fgemini-3.7-flash", "failed": 2, "failure_categories": { "provider_interrupted": 2 }, "input_mode": "text", "input_tokens": 453431, "known_cost_usd": 0.5986506, "median_duration_seconds": 98, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 98, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 67, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 288, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 21845, "pass_rate": 0.3333333333333333, "pass_rate_95_high": 0.7923403991979522, "pass_rate_95_low": 0.06149194472039621, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "017-partial-rails-rollout", "scenario_label": "017-partial-rails-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.16651658749999998, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=018-node-event-loop-blocking&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 4226702, "known_cost_usd": 0.49954976249999994, "median_duration_seconds": 257, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 363, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 257, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 224, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 54367, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "018-node-event-loop-blocking", "scenario_label": "018-node-event-loop-blocking", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.3034527, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=019-rust-fd-leak&model=google%2Fgemini-3.7-flash", "failed": 2, "failure_categories": { "provider_interrupted": 2 }, "input_mode": "text", "input_tokens": 242349, "known_cost_usd": 0.3034527, "median_duration_seconds": 110, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 35, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 154, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 110, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 19810, "pass_rate": 0.3333333333333333, "pass_rate_95_high": 0.7923403991979522, "pass_rate_95_low": 0.06149194472039621, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "019-rust-fd-leak", "scenario_label": "019-rust-fd-leak", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.36277005, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=020-python-gunicorn-saturation&model=google%2Fgemini-3.7-flash", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 7582037, "known_cost_usd": 0.7255401, "median_duration_seconds": 503, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 232, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 562, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 503, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 65725, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "020-python-gunicorn-saturation", "scenario_label": "020-python-gunicorn-saturation", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 1.248207225, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=021-discourse-shared-uploads&model=google%2Fgemini-3.7-flash", "failed": 2, "failure_categories": { "provider_interrupted": 2 }, "input_mode": "text", "input_tokens": 868597, "known_cost_usd": 1.248207225, "median_duration_seconds": 276, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 294, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 276, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 188, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 72394, "pass_rate": 0.3333333333333333, "pass_rate_95_high": 0.7923403991979522, "pass_rate_95_low": 0.06149194472039621, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "021-discourse-shared-uploads", "scenario_label": "021-discourse-shared-uploads", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.14459302500000001, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=022-discourse-multisite-migration&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 3553624, "known_cost_usd": 0.433779075, "median_duration_seconds": 182, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 170, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 182, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 193, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 34114, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "022-discourse-multisite-migration", "scenario_label": "022-discourse-multisite-migration", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.18134868750000002, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=023-auth-secret-rollout&model=google%2Fgemini-3.7-flash", "failed": 1, "failure_categories": { "current_sessions_unavailable": 1 }, "input_mode": "text", "input_tokens": 2595978, "known_cost_usd": 0.36269737500000004, "median_duration_seconds": 178, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 185, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 178, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 177, "failure_category": "current_sessions_unavailable", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 38923, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "023-auth-secret-rollout", "scenario_label": "023-auth-secret-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.38071972500000006, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=024-discourse-interrupted-deploy&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 882979, "known_cost_usd": 1.1421591750000002, "median_duration_seconds": 177, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 173, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 177, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 189, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 41704, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "024-discourse-interrupted-deploy", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.323668725, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 3871305, "known_cost_usd": 0.323668725, "median_duration_seconds": 865, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 865, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 18440, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.14563270000000003, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=026-discourse-plugin-boot-loop&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 3583351, "known_cost_usd": 0.43689810000000007, "median_duration_seconds": 188, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 187, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 235, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 188, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 40253, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "026-discourse-plugin-boot-loop", "scenario_label": "026-discourse-plugin-boot-loop", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.23268285000000002, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=027-partial-service-rollout&model=google%2Fgemini-3.7-flash", "failed": 2, "failure_categories": { "provider_protocol_error": 2 }, "input_mode": "text", "input_tokens": 1817620, "known_cost_usd": 0.23268285000000002, "median_duration_seconds": 85, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 85, "failure_category": "provider_protocol_error", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 25, "failure_category": "provider_protocol_error", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 207, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 18961, "pass_rate": 0.3333333333333333, "pass_rate_95_high": 0.7923403991979522, "pass_rate_95_low": 0.06149194472039621, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "027-partial-service-rollout", "scenario_label": "027-partial-service-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.37500425, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=028-nix-store-disk-pressure&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 825363, "known_cost_usd": 1.12501275, "median_duration_seconds": 220, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 220, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 247, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 158, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 38582, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "028-nix-store-disk-pressure", "scenario_label": "028-nix-store-disk-pressure", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.1140424375, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260825.0.1&scenario=031-visual-deployment-timeline&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "visual", "input_tokens": 554539, "known_cost_usd": 0.3421273125, "median_duration_seconds": 178, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 219, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 170, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 178, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 30577, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260825.0.1", "scenario": "031-visual-deployment-timeline", "scenario_label": "Visual deployment timeline", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.17239057499999996, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260822.0.4&scenario=030-visual-metrics-regression&model=google%2Fgemini-3.7-flash", "failed": 1, "failure_categories": { "provider_protocol_error": 1 }, "input_mode": "visual", "input_tokens": 546477, "known_cost_usd": 0.3447811499999999, "median_duration_seconds": 219, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 219, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 59, "failure_category": "provider_protocol_error", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 236, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 34699, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260822.0.4", "scenario": "030-visual-metrics-regression", "scenario_label": "Visual metrics regression", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.1349796375, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260822.0.4&scenario=029-visual-topology-drift&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "visual", "input_tokens": 628461, "known_cost_usd": 0.4049389125, "median_duration_seconds": 180, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 146, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 187, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 180, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 33494, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260822.0.4", "scenario": "029-visual-topology-drift", "scenario_label": "Visual topology drift", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} } ], "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "records": [ { "cost_per_repair_usd": 0.02383183333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=001-nginx-502-host&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 4126, "known_cost_usd": 0.07149549999999999, "median_duration_seconds": 204, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 84, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 204, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 204, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 13847, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "001-nginx-502-host", "scenario_label": "001-nginx-502-host", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.02373603666666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=013-sidekiq-wrong-redis&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 147, "known_cost_usd": 0.07120811, "median_duration_seconds": 86, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 86, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 163, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 82, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 18056, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "013-sidekiq-wrong-redis", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.030913713333333332, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=014-missing-rails-migration&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 1661822, "known_cost_usd": 0.09274114, "median_duration_seconds": 122, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 127, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 122, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 80, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 20332, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "014-missing-rails-migration", "scenario_label": "014-missing-rails-migration", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.061030705000000005, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=015-sidekiq-poison-pill&model=openai%2Fgpt-5.6-luna", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 1108, "known_cost_usd": 0.12206141000000001, "median_duration_seconds": 183, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 339, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 150, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 183, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 29631, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "015-sidekiq-poison-pill", "scenario_label": "015-sidekiq-poison-pill", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.019856973333333337, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=016-rails-pool-exhaustion&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 990840, "known_cost_usd": 0.059570920000000006, "median_duration_seconds": 83, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 83, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 56, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 96, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 17713, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "016-rails-pool-exhaustion", "scenario_label": "016-rails-pool-exhaustion", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.034490865, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=017-partial-rails-rollout&model=openai%2Fgpt-5.6-luna", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 21816, "known_cost_usd": 0.06898173, "median_duration_seconds": 130, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 130, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 146, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 58, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 13233, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "017-partial-rails-rollout", "scenario_label": "017-partial-rails-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.03912562666666666, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=018-node-event-loop-blocking&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 2391787, "known_cost_usd": 0.11737687999999999, "median_duration_seconds": 240, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 278, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 240, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 228, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 26681, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "018-node-event-loop-blocking", "scenario_label": "018-node-event-loop-blocking", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.028723753333333334, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=019-rust-fd-leak&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 159, "known_cost_usd": 0.08617126, "median_duration_seconds": 126, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 126, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 223, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 107, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 20413, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "019-rust-fd-leak", "scenario_label": "019-rust-fd-leak", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.03179777, "cost_reported_trials": 2, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=020-python-gunicorn-saturation&model=openai%2Fgpt-5.6-luna", "failed": 1, "failure_categories": { "agent_timeout": 1 }, "input_mode": "text", "input_tokens": 1028180, "known_cost_usd": 0.06359554, "median_duration_seconds": 498, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 498, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 247, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 931, "failure_category": "agent_timeout", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 17544, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "020-python-gunicorn-saturation", "scenario_label": "020-python-gunicorn-saturation", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.06692405500000001, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=021-discourse-shared-uploads&model=openai%2Fgpt-5.6-luna", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 204, "known_cost_usd": 0.13384811000000002, "median_duration_seconds": 159, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 159, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 122, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 166, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 35972, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "021-discourse-shared-uploads", "scenario_label": "021-discourse-shared-uploads", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.023928006666666668, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=022-discourse-multisite-migration&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 1123728, "known_cost_usd": 0.07178402, "median_duration_seconds": 96, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 95, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 108, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 96, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 18271, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "022-discourse-multisite-migration", "scenario_label": "022-discourse-multisite-migration", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.03233095333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=023-auth-secret-rollout&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 1750522, "known_cost_usd": 0.09699285999999999, "median_duration_seconds": 130, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 138, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 130, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 105, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 25544, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "023-auth-secret-rollout", "scenario_label": "023-auth-secret-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.05000849, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=024-discourse-interrupted-deploy&model=openai%2Fgpt-5.6-luna", "failed": 1, "failure_categories": { "release_not_converged": 1 }, "input_mode": "text", "input_tokens": 159, "known_cost_usd": 0.10001698, "median_duration_seconds": 103, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 112, "failure_category": "release_not_converged", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 103, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 97, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 24691, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "024-discourse-interrupted-deploy", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.026975900000000008, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 1275087, "known_cost_usd": 0.026975900000000008, "median_duration_seconds": 602, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 602, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 10901, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.0239323, "cost_reported_trials": 2, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=026-discourse-plugin-boot-loop&model=openai%2Fgpt-5.6-luna", "failed": 1, "failure_categories": { "agent_timeout": 1 }, "input_mode": "text", "input_tokens": 766388, "known_cost_usd": 0.0478646, "median_duration_seconds": 101, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 101, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 88, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 931, "failure_category": "agent_timeout", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 11444, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "026-discourse-plugin-boot-loop", "scenario_label": "026-discourse-plugin-boot-loop", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.036988106666666666, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=027-partial-service-rollout&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 2043334, "known_cost_usd": 0.11096431999999999, "median_duration_seconds": 135, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 135, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 138, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 116, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 26485, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "027-partial-service-rollout", "scenario_label": "027-partial-service-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.031202233333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=028-nix-store-disk-pressure&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 141, "known_cost_usd": 0.0936067, "median_duration_seconds": 134, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 138, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 124, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 134, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 27153, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "028-nix-store-disk-pressure", "scenario_label": "028-nix-store-disk-pressure", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.027466479999999998, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260825.0.1&scenario=031-visual-deployment-timeline&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "visual", "input_tokens": 9697, "known_cost_usd": 0.08239943999999999, "median_duration_seconds": 110, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 101, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 116, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 110, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 19927, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260825.0.1", "scenario": "031-visual-deployment-timeline", "scenario_label": "Visual deployment timeline", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.02323753, "cost_reported_trials": 2, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260822.0.4&scenario=030-visual-metrics-regression&model=openai%2Fgpt-5.6-luna", "failed": 1, "failure_categories": { "agent_timeout": 1 }, "input_mode": "visual", "input_tokens": 5021, "known_cost_usd": 0.04647506, "median_duration_seconds": 84, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 84, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 931, "failure_category": "agent_timeout", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 73, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 11279, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260822.0.4", "scenario": "030-visual-metrics-regression", "scenario_label": "Visual metrics regression", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.026767816666666666, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260822.0.4&scenario=029-visual-topology-drift&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "visual", "input_tokens": 153, "known_cost_usd": 0.08030345, "median_duration_seconds": 93, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 90, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 99, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 93, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 21826, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260822.0.4", "scenario": "029-visual-topology-drift", "scenario_label": "Visual topology drift", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} } ], "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "records": [ { "cost_per_repair_usd": 0.03261741666666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260827.0.1&scenario=024-discourse-interrupted-deploy&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 142641, "known_cost_usd": 0.09785225, "median_duration_seconds": 133, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 133, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 253, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 108, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 24369, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "Vercel AI Gateway", "reasoning_effort": "high", "release": "20260827.0.1", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "Interrupted Discourse deploy", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.024328226666666664, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260827.0.1&scenario=001-nginx-502-host&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 149304, "known_cost_usd": 0.07298468, "median_duration_seconds": 73, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 65, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 73, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 74, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 13696, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "Vercel AI Gateway", "reasoning_effort": "high", "release": "20260827.0.1", "scenario": "001-nginx-502-host", "scenario_label": "Nginx 502 host failure", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.030621036666666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260827.0.1&scenario=013-sidekiq-wrong-redis&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 172738, "known_cost_usd": 0.09186311, "median_duration_seconds": 107, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 130, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 94, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 107, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 20992, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "Vercel AI Gateway", "reasoning_effort": "high", "release": "20260827.0.1", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "Sidekiq connected to the wrong Redis", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} } ], "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "records": [ { "cost_per_repair_usd": 0.07562893333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=001-nginx-502-host&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 66, "known_cost_usd": 0.2268868, "median_duration_seconds": 63, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 73, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 63, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 60, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 10641, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "001-nginx-502-host", "scenario_label": "001-nginx-502-host", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.10063046666666665, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=013-sidekiq-wrong-redis&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 76, "known_cost_usd": 0.3018914, "median_duration_seconds": 86, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 86, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 95, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 65, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 12472, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "013-sidekiq-wrong-redis", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.7644920000000002, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260815.0.1&scenario=014-missing-rails-migration&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 348661, "known_cost_usd": 0.7644920000000002, "median_duration_seconds": 112, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 112, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 6717, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260815.0.1", "scenario": "014-missing-rails-migration", "scenario_label": "014-missing-rails-migration", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.2343724, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=015-sidekiq-poison-pill&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 118, "known_cost_usd": 0.7031172, "median_duration_seconds": 194, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 194, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 166, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 201, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 31849, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "015-sidekiq-poison-pill", "scenario_label": "015-sidekiq-poison-pill", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.506758, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260815.0.1&scenario=016-rails-pool-exhaustion&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 225564, "known_cost_usd": 0.506758, "median_duration_seconds": 111, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 111, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 5563, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260815.0.1", "scenario": "016-rails-pool-exhaustion", "scenario_label": "016-rails-pool-exhaustion", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.2250975333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=017-partial-rails-rollout&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 94, "known_cost_usd": 0.6752925999999999, "median_duration_seconds": 125, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 125, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 124, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 198, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 19742, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "017-partial-rails-rollout", "scenario_label": "017-partial-rails-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.389914, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260815.0.1&scenario=018-node-event-loop-blocking&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 160762, "known_cost_usd": 0.389914, "median_duration_seconds": 123, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 123, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 6839, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260815.0.1", "scenario": "018-node-event-loop-blocking", "scenario_label": "018-node-event-loop-blocking", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.1958167333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=019-rust-fd-leak&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 124, "known_cost_usd": 0.5874501999999999, "median_duration_seconds": 148, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 120, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 148, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 185, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 26063, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "019-rust-fd-leak", "scenario_label": "019-rust-fd-leak", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.5880019999999999, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260815.0.1&scenario=020-python-gunicorn-saturation&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 254451, "known_cost_usd": 0.5880019999999999, "median_duration_seconds": 129, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 129, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 7910, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260815.0.1", "scenario": "020-python-gunicorn-saturation", "scenario_label": "020-python-gunicorn-saturation", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.4619854333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=021-discourse-shared-uploads&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 184, "known_cost_usd": 1.3859563, "median_duration_seconds": 313, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 313, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 303, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 355, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 63177, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "021-discourse-shared-uploads", "scenario_label": "021-discourse-shared-uploads", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.557208, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260815.0.1&scenario=022-discourse-multisite-migration&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 244384, "known_cost_usd": 0.557208, "median_duration_seconds": 111, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 111, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 6844, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260815.0.1", "scenario": "022-discourse-multisite-migration", "scenario_label": "022-discourse-multisite-migration", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.5876560000000001, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260815.0.1&scenario=023-auth-secret-rollout&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 254483, "known_cost_usd": 0.5876560000000001, "median_duration_seconds": 123, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 123, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 7869, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260815.0.1", "scenario": "023-auth-secret-rollout", "scenario_label": "023-auth-secret-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": null, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=024-discourse-interrupted-deploy&model=anthropic%2Fclaude-sonnet-5", "failed": 3, "failure_categories": { "release_not_converged": 3 }, "input_mode": "text", "input_tokens": 114, "known_cost_usd": 0.9510542, "median_duration_seconds": 152, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 122, "failure_category": "release_not_converged", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 152, "failure_category": "release_not_converged", "reward": 0, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 321, "failure_category": "release_not_converged", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 41737, "pass_rate": 0.0, "pass_rate_95_high": 0.5614970317550454, "pass_rate_95_low": 5.551115123125783e-17, "passed": 0, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "024-discourse-interrupted-deploy", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.24632686666666662, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=028-nix-store-disk-pressure&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 96, "known_cost_usd": 0.7389805999999999, "median_duration_seconds": 148, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 165, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 148, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 145, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 31253, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "028-nix-store-disk-pressure", "scenario_label": "028-nix-store-disk-pressure", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.1698881, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260825.0.1&scenario=031-visual-deployment-timeline&model=anthropic%2Fclaude-sonnet-5", "failed": 0, "failure_categories": {}, "input_mode": "visual", "input_tokens": 94, "known_cost_usd": 0.5096643, "median_duration_seconds": 131, "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "outcomes": [ { "attempt": 1, "duration_seconds": 131, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 145, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 112, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 19545, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260825.0.1", "scenario": "031-visual-deployment-timeline", "scenario_label": "Visual deployment timeline", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 0, "unavailable_categories": {} } ], "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "records": [ { "cost_per_repair_usd": 0.063868, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=001-nginx-502-host&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 42384, "known_cost_usd": 0.191604, "median_duration_seconds": 101, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 76, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 135, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 101, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 8990, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "001-nginx-502-host", "scenario_label": "001-nginx-502-host", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.08761000000000001, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=013-sidekiq-wrong-redis&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 53684, "known_cost_usd": 0.26283, "median_duration_seconds": 136, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 157, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 95, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 136, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 12913, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "013-sidekiq-wrong-redis", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.1500786666666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=014-missing-rails-migration&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 774829, "known_cost_usd": 0.4502360000000001, "median_duration_seconds": 122, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 186, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 122, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 99, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 23307, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "014-missing-rails-migration", "scenario_label": "014-missing-rails-migration", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.18190266666666666, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=015-sidekiq-poison-pill&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 85552, "known_cost_usd": 0.545708, "median_duration_seconds": 332, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 291, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 332, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 341, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 31218, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "015-sidekiq-poison-pill", "scenario_label": "015-sidekiq-poison-pill", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.10498066666666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=016-rails-pool-exhaustion&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 560176, "known_cost_usd": 0.314942, "median_duration_seconds": 96, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 96, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 100, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 82, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 19269, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "016-rails-pool-exhaustion", "scenario_label": "016-rails-pool-exhaustion", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.10550599999999999, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=017-partial-rails-rollout&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 66671, "known_cost_usd": 0.31651799999999997, "median_duration_seconds": 164, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 164, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 326, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 164, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 14492, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "017-partial-rails-rollout", "scenario_label": "017-partial-rails-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.13378533333333334, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=018-node-event-loop-blocking&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 451115, "known_cost_usd": 0.401356, "median_duration_seconds": 94, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 121, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 90, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 94, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 22601, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "018-node-event-loop-blocking", "scenario_label": "018-node-event-loop-blocking", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.13336066666666668, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=019-rust-fd-leak&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 68307, "known_cost_usd": 0.40008200000000005, "median_duration_seconds": 187, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 187, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 177, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 290, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 22690, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "019-rust-fd-leak", "scenario_label": "019-rust-fd-leak", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.12676733333333334, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=020-python-gunicorn-saturation&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 559319, "known_cost_usd": 0.38030200000000003, "median_duration_seconds": 116, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 185, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 91, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 116, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 27328, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "020-python-gunicorn-saturation", "scenario_label": "020-python-gunicorn-saturation", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.4356966666666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=021-discourse-shared-uploads&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 84550, "known_cost_usd": 1.30709, "median_duration_seconds": 456, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 605, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 242, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 456, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 97929, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "021-discourse-shared-uploads", "scenario_label": "021-discourse-shared-uploads", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.15142333333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=022-discourse-multisite-migration&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 757937, "known_cost_usd": 0.45427, "median_duration_seconds": 112, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 69, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 112, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 121, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 20874, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "022-discourse-multisite-migration", "scenario_label": "022-discourse-multisite-migration", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.18736266666666665, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=023-auth-secret-rollout&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 987187, "known_cost_usd": 0.5620879999999999, "median_duration_seconds": 143, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 112, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 143, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 148, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 28883, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "023-auth-secret-rollout", "scenario_label": "023-auth-secret-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.14368799999999998, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=024-discourse-interrupted-deploy&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 80580, "known_cost_usd": 0.43106399999999995, "median_duration_seconds": 197, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 256, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 194, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 197, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 19128, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "024-discourse-interrupted-deploy", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.309992, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 677161, "known_cost_usd": 0.309992, "median_duration_seconds": 690, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 690, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 20321, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.22082533333333332, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=026-discourse-plugin-boot-loop&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 967139, "known_cost_usd": 0.662476, "median_duration_seconds": 147, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 147, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 122, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 186, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 28585, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "026-discourse-plugin-boot-loop", "scenario_label": "026-discourse-plugin-boot-loop", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.13333733333333334, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=027-partial-service-rollout&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 607877, "known_cost_usd": 0.400012, "median_duration_seconds": 109, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 82, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 136, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 109, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 19731, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "027-partial-service-rollout", "scenario_label": "027-partial-service-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.202588, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=028-nix-store-disk-pressure&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 102677, "known_cost_usd": 0.607764, "median_duration_seconds": 367, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 373, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 267, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 367, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 34359, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "028-nix-store-disk-pressure", "scenario_label": "028-nix-store-disk-pressure", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": null, "cost_reported_trials": 0, "evaluated": 0, "evidence_url": "benchmark-explorer.html?release=20260822.0.2&scenario=029-visual-topology-drift&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "visual", "input_tokens": 0, "known_cost_usd": 0, "median_duration_seconds": null, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 2, "failure_category": "agent_runtime_error", "reward": 0, "trial_status": "unavailable" }, { "attempt": 2, "duration_seconds": 2, "failure_category": "agent_runtime_error", "reward": 0, "trial_status": "unavailable" }, { "attempt": 3, "duration_seconds": 2, "failure_category": "agent_runtime_error", "reward": 0, "trial_status": "unavailable" } ], "output_tokens": 0, "pass_rate": null, "pass_rate_95_high": null, "pass_rate_95_low": null, "passed": 0, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260822.0.2", "scenario": "029-visual-topology-drift", "scenario_label": "Visual topology drift", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "unclassified", "trials": 3, "unavailable": 3, "unavailable_categories": { "agent_runtime_error": 3 } } ], "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "records": [ { "cost_per_repair_usd": 0.15932785333333332, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=001-nginx-502-host&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 153075, "known_cost_usd": 0.47798356, "median_duration_seconds": 146, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 353, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 112, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 146, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 28349, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "001-nginx-502-host", "scenario_label": "001-nginx-502-host", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.08706796, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=013-sidekiq-wrong-redis&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 51295, "known_cost_usd": 0.26120388, "median_duration_seconds": 150, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 150, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 122, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 203, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 15542, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "013-sidekiq-wrong-redis", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.09706993333333332, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=014-missing-rails-migration&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 683423, "known_cost_usd": 0.29120979999999996, "median_duration_seconds": 189, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 189, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 166, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 221, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 12228, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "014-missing-rails-migration", "scenario_label": "014-missing-rails-migration", "scenario_version": 2, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.256728324, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=015-sidekiq-poison-pill&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 246493, "known_cost_usd": 0.770184972, "median_duration_seconds": 292, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 262, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 599, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 292, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 39705, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "015-sidekiq-poison-pill", "scenario_label": "015-sidekiq-poison-pill", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.05620229333333334, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=016-rails-pool-exhaustion&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 332780, "known_cost_usd": 0.16860688000000001, "median_duration_seconds": 109, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 102, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 136, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 109, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 11232, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "016-rails-pool-exhaustion", "scenario_label": "016-rails-pool-exhaustion", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.12826153333333334, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=017-partial-rails-rollout&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 71551, "known_cost_usd": 0.38478460000000003, "median_duration_seconds": 341, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 340, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 341, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 354, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 18679, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "017-partial-rails-rollout", "scenario_label": "017-partial-rails-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.10987948000000002, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=018-node-event-loop-blocking&model=z-ai%2Fglm-5.3", "failed": 1, "failure_categories": { "provider_interrupted": 1 }, "input_mode": "text", "input_tokens": 433544, "known_cost_usd": 0.21975896000000003, "median_duration_seconds": 118, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 131, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 114, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 118, "failure_category": "provider_interrupted", "reward": 0, "trial_status": "evaluated" } ], "output_tokens": 13049, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "018-node-event-loop-blocking", "scenario_label": "018-node-event-loop-blocking", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.16213272133333334, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=019-rust-fd-leak&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 142261, "known_cost_usd": 0.486398164, "median_duration_seconds": 141, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 314, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 140, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 141, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 25315, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "019-rust-fd-leak", "scenario_label": "019-rust-fd-leak", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.10923059999999998, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=020-python-gunicorn-saturation&model=z-ai%2Fglm-5.3", "failed": 1, "failure_categories": { "host_reboot_failed": 1 }, "input_mode": "text", "input_tokens": 389950, "known_cost_usd": 0.21846119999999997, "median_duration_seconds": 154, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 154, "failure_category": "host_reboot_failed", "reward": 0, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 120, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 175, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 17024, "pass_rate": 0.6666666666666666, "pass_rate_95_high": 0.9385080552796037, "pass_rate_95_low": 0.2076596008020477, "passed": 2, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "020-python-gunicorn-saturation", "scenario_label": "020-python-gunicorn-saturation", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.5880829941666667, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=021-discourse-shared-uploads&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 909467, "known_cost_usd": 1.7642489825, "median_duration_seconds": 759, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 759, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 729, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 806, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 86059, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "021-discourse-shared-uploads", "scenario_label": "021-discourse-shared-uploads", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.10762982666666665, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=022-discourse-multisite-migration&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 662505, "known_cost_usd": 0.32288947999999995, "median_duration_seconds": 119, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 119, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 354, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 90, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 21391, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "022-discourse-multisite-migration", "scenario_label": "022-discourse-multisite-migration", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.09924885333333335, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=023-auth-secret-rollout&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 662506, "known_cost_usd": 0.29774656000000005, "median_duration_seconds": 123, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 123, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 125, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 119, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 13471, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "023-auth-secret-rollout", "scenario_label": "023-auth-secret-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.19236610799999998, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=024-discourse-interrupted-deploy&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 312198, "known_cost_usd": 0.5770983239999999, "median_duration_seconds": 326, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 347, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 326, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 199, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 17786, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "024-discourse-interrupted-deploy", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.09544488, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=026-discourse-plugin-boot-loop&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 609626, "known_cost_usd": 0.28633464, "median_duration_seconds": 126, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 126, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 145, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 117, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 14056, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "026-discourse-plugin-boot-loop", "scenario_label": "026-discourse-plugin-boot-loop", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.07705273333333333, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260819.0.0&scenario=027-partial-service-rollout&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 485549, "known_cost_usd": 0.2311582, "median_duration_seconds": 127, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 152, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 127, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 123, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 13950, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260819.0.0", "scenario": "027-partial-service-rollout", "scenario_label": "027-partial-service-rollout", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "full", "trials": 3, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.14912578666666668, "cost_reported_trials": 3, "evaluated": 3, "evidence_url": "benchmark-explorer.html?release=20260906.0.0&scenario=028-nix-store-disk-pressure&model=z-ai%2Fglm-5.3", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 109375, "known_cost_usd": 0.44737736, "median_duration_seconds": 143, "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "outcomes": [ { "attempt": 1, "duration_seconds": 88, "reward": 1, "trial_status": "evaluated" }, { "attempt": 2, "duration_seconds": 143, "reward": 1, "trial_status": "evaluated" }, { "attempt": 3, "duration_seconds": 199, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 29776, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.4385029682449546, "passed": 3, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260906.0.0", "scenario": "028-nix-store-disk-pressure", "scenario_label": "028-nix-store-disk-pressure", "scenario_version": 1, "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "core", "trials": 3, "unavailable": 0, "unavailable_categories": {} } ], "tiers": [ "core", "full" ] } ], "scenarios": [ { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "d4c06ab5021d607f8f78ece8851ec467da317e7c35fa30f0e025aecfdcd9b721", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=001-nginx-502-host", "release": "20260924.0.0", "scenario": "001-nginx-502-host", "scenario_label": "001-nginx-502-host", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "efe880f278828ff2d399533399ff4bca60b46b1987c5a0c980d6d4d715f35fb6", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=013-sidekiq-wrong-redis", "release": "20260924.0.0", "scenario": "013-sidekiq-wrong-redis", "scenario_label": "013-sidekiq-wrong-redis", "scenario_version": 2 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "3f717ba6ce09a00943eb32d6ae18fc0b63588b53703cdcdce2b249e1d63e8f6d", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=014-missing-rails-migration", "release": "20260821.0.0", "scenario": "014-missing-rails-migration", "scenario_label": "014-missing-rails-migration", "scenario_version": 2 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "39b4c40a3c0e25fea8d66a64dfba96137ff9c32e033678a89b649e702638ae4b", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=015-sidekiq-poison-pill", "release": "20260924.0.0", "scenario": "015-sidekiq-poison-pill", "scenario_label": "015-sidekiq-poison-pill", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "d26159dc01afee8679ca1fb9b1f766cc5df44881ff8d3be442a62909652b4256", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=016-rails-pool-exhaustion", "release": "20260821.0.0", "scenario": "016-rails-pool-exhaustion", "scenario_label": "016-rails-pool-exhaustion", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "4c1f59557a89386eca6f78478a26a2a410145f6e9a9f5fd8c212cec044fbd8ce", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=017-partial-rails-rollout", "release": "20260924.0.0", "scenario": "017-partial-rails-rollout", "scenario_label": "017-partial-rails-rollout", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "deadd0eaaa1d86754f5109eabe17287a6ed9f0cd19b659170859e36c20d48d78", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=018-node-event-loop-blocking", "release": "20260821.0.0", "scenario": "018-node-event-loop-blocking", "scenario_label": "018-node-event-loop-blocking", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "c688839fe58ea027d2fcf5711b6dc4ee84196080ccb25bfb0fdb96c043e64bee", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=019-rust-fd-leak", "release": "20260924.0.0", "scenario": "019-rust-fd-leak", "scenario_label": "019-rust-fd-leak", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "e4818d58a594382f0dfa4b334e319f8ede2a27867b20d97f0af128e1d6a127aa", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=020-python-gunicorn-saturation", "release": "20260821.0.0", "scenario": "020-python-gunicorn-saturation", "scenario_label": "020-python-gunicorn-saturation", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "12987993a9d64e8984abbf0ab5ee603deb73056bc25652096a96e2e6321f9d2f", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=021-discourse-shared-uploads", "release": "20260924.0.0", "scenario": "021-discourse-shared-uploads", "scenario_label": "021-discourse-shared-uploads", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "a64858520a1751f5f71bb40a7660c68af760a631d9e4f27a8ef239916d68d732", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=022-discourse-multisite-migration", "release": "20260821.0.0", "scenario": "022-discourse-multisite-migration", "scenario_label": "022-discourse-multisite-migration", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "234282e2269eb3b838a17a19a60ad30b2b74d1c519b28d57daa7a771f2a15169", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=023-auth-secret-rollout", "release": "20260821.0.0", "scenario": "023-auth-secret-rollout", "scenario_label": "023-auth-secret-rollout", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "c9dfdd5a1a9dae87fb9b94aabcb52b75889d631432d040a3a4a67c6651d33c7b", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=024-discourse-interrupted-deploy", "release": "20260924.0.0", "scenario": "024-discourse-interrupted-deploy", "scenario_label": "024-discourse-interrupted-deploy", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260815.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 1800, "attempts": 1, "claux_binary_sha256": null, "claux_release": "v20260815.0.0", "harness_versions": [ 19 ], "host_harness_sha256": "b937dcc748a811bb06e00a494ca63874ff9008d8409a33a151122d20de8cfe70", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260815.0.0" }, "scenario_snapshot_sha256": null, "tier": "frontier" }, "cells": [ { "cost_per_repair_usd": 0.030954798000000002, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=deepseek%2Fdeepseek-v4-flash-0731", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 1751314, "known_cost_usd": 0.030954798000000002, "median_duration_seconds": 1752, "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "outcomes": [ { "attempt": 1, "duration_seconds": 1752, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 27737, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "status": "covered", "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.323668725, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=google%2Fgemini-3.7-flash", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 3871305, "known_cost_usd": 0.323668725, "median_duration_seconds": 865, "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "outcomes": [ { "attempt": 1, "duration_seconds": 865, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 18440, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "status": "covered", "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "cost_per_repair_usd": 0.026975900000000008, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=openai%2Fgpt-5.6-luna", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 1275087, "known_cost_usd": 0.026975900000000008, "median_duration_seconds": 602, "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "outcomes": [ { "attempt": 1, "duration_seconds": 602, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 10901, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "status": "covered", "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "cost_per_repair_usd": 0.309992, "cost_reported_trials": 1, "evaluated": 1, "evidence_url": "benchmark-explorer.html?release=20260816.0.0&scenario=025-discourse-bootstrap&model=qwen%2Fqwen3.8-2.4t-a95b", "failed": 0, "failure_categories": {}, "input_mode": "text", "input_tokens": 677161, "known_cost_usd": 0.309992, "median_duration_seconds": 690, "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "outcomes": [ { "attempt": 1, "duration_seconds": 690, "reward": 1, "trial_status": "evaluated" } ], "output_tokens": 20321, "pass_rate": 1.0, "pass_rate_95_high": 1.0, "pass_rate_95_low": 0.20654931437723745, "passed": 1, "provider": "OpenRouter", "reasoning_effort": "high", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1, "status": "covered", "subscription_usage_reported_trials": 0, "subscription_usage_usd": 0, "tier": "frontier", "trials": 1, "unavailable": 0, "unavailable_categories": {} }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=025-discourse-bootstrap", "release": "20260816.0.0", "scenario": "025-discourse-bootstrap", "scenario_label": "025-discourse-bootstrap", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "7bceaaf21df61d7a8a8b05c6ee77478ace16b1a86ce1d14707a2f897c278b8aa", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=026-discourse-plugin-boot-loop", "release": "20260821.0.0", "scenario": "026-discourse-plugin-boot-loop", "scenario_label": "026-discourse-plugin-boot-loop", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.0" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": null, "claux_release": "v20260821.0.0", "harness_versions": [ 23 ], "host_harness_sha256": "920d6f594c715f6412ce67ae299a12fa5d9a679ae35ccea9adf167beeca267c8", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260817.1.0" }, "scenario_snapshot_sha256": "0389d64b69ed2dba434edad67282a29c3484792fd4f2e6dd1b09a7b3a1b499b9", "tier": "full" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=027-partial-service-rollout", "release": "20260821.0.0", "scenario": "027-partial-service-rollout", "scenario_label": "027-partial-service-rollout", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "1ea2027179bd35dd7dbb08bb986c4208eae3d74fa5e418e40d5377ee6f25f0ec", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "39e10e2d46bda3d3873578156561f83edd047b0b1a21242f584b0cb35fc80ed7", "tier": "core" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=028-nix-store-disk-pressure", "release": "20260924.0.0", "scenario": "028-nix-store-disk-pressure", "scenario_label": "028-nix-store-disk-pressure", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "693da498a20a39a10546da6db6d94adfa20c8d99d26663d6c0df742db115fc5a", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "9e43c4c58db2c9c1c9f45e43aebfbd2be937a31f4cb8f564aea59a13625ae6e1", "tier": "unclassified" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=029-visual-topology-drift", "release": "20260905.0.0", "scenario": "029-visual-topology-drift", "scenario_label": "029-visual-topology-drift", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "693da498a20a39a10546da6db6d94adfa20c8d99d26663d6c0df742db115fc5a", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "dbf1c44f354b22207d06de7d4336d243afd553bc52aca7aca0ab147d7abb9f31", "tier": "unclassified" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=030-visual-metrics-regression", "release": "20260905.0.0", "scenario": "030-visual-metrics-regression", "scenario_label": "030-visual-metrics-regression", "scenario_version": 1 }, { "boundary": { "agent_adapter": "builtin:claux", "agent_adapter_sha256": null, "agent_harness": { "billing": "provider-reported", "id": "claux", "label": "Claux", "provider": "OpenRouter", "version": "v20260821.0.2" }, "agent_name": "claux", "agent_timeout_seconds": 900, "attempts": 3, "claux_binary_sha256": "693da498a20a39a10546da6db6d94adfa20c8d99d26663d6c0df742db115fc5a", "claux_release": "v20260821.0.2", "harness_versions": [ 23 ], "host_harness_sha256": "9d3d63a934d46f875badbe519f07ac9cb9f50d9a8646d7cd9a2153fd87d8e401", "scenario_pack": { "id": "ducks/replaybook-infra", "version": "20260824.0.0" }, "scenario_snapshot_sha256": "7f0961a1711e582841c568e8c8a389aa309afdfadec13a3cb349ec00167c5de5", "tier": "unclassified" }, "cells": [ { "model": "deepseek/deepseek-v4-flash-0731", "model_label": "DeepSeek V4 Flash 0731", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "google/gemini-3.7-flash", "model_label": "Gemini 3.7 Flash", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "provider": "Vercel AI Gateway", "reasoning_effort": "high", "status": "missing", "tiers": [ "unclassified" ] }, { "model": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "unclassified" ] }, { "model": "qwen/qwen3.8-2.4t-a95b", "model_label": "Qwen3.8 2.4T A95B", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full", "frontier", "unclassified" ] }, { "model": "z-ai/glm-5.3", "model_label": "GLM 5.3", "provider": "OpenRouter", "reasoning_effort": "high", "status": "missing", "tiers": [ "core", "full" ] } ], "evidence_url": "benchmark-scenario.html?scenario=031-visual-deployment-timeline", "release": "20260905.0.0", "scenario": "031-visual-deployment-timeline", "scenario_label": "031-visual-deployment-timeline", "scenario_version": 1 } ], "schema_version": 1, "totals": { "covered_cells": 4, "missing_cells": 136, "model_lanes": 7, "possible_cells": 140, "scenarios": 20 } }