{
  "meta": {
    "title": "Bug Hunt Bench",
    "subtitle": "105 real bugs hidden in two production repos. Frontier coding models find and fix what they can, graded blind.",
    "planted_total": 105,
    "repo1_total": 45,
    "repo2_total": 60,
    "survivors": 50,
    "models": 15,
    "runs": 25,
    "updated": "2026-08-25",
    "source_repo": "https://github.com/phuryn/experiments/tree/main/bug-hunt-bench",
    "author": "Pawel Huryn",
    "author_url": "https://www.productcompass.pm",
    "author_x": "https://x.com/PawelHuryn"
  },
  "method": [
    "Two unrelated production codebases: a ~28K-line TypeScript VS Code extension (45 bugs) and a ~60K-line React/Supabase LMS (60 bugs). Most bugs are real regressions reverted from the repos' own fix history, each with its fix commit.",
    "The test suites stay green, so nothing points at the answers. Every model gets the identical prompt, one round per repo, in its own native agentic CLI.",
    "Grading is blind: the model's diff is the ground truth, not its report. An independent judge scores against a withheld answer key and never learns which model produced the diff.",
    "The headline score counts PLANTED bugs fixed, and nothing else. Extra fixes are tracked separately and never added to the score - some models report large numbers of unrelated issues, and a benchmark that rewards volume rewards noise."
  ],
  "glossary": {
    "fixed": "Planted bugs fixed and verified against the withheld answer key. The only number in the score.",
    "partial": "The defect was addressed but not fully closed. Not counted in the score.",
    "claimed_only": "The model reported a fix that its diff does not deliver.",
    "extras": "Genuine defects fixed that were never planted. Real work, deliberately kept out of the score.",
    "cost_bill": "An actual invoice or credits delta.",
    "cost_list": "A token-count estimate at published list rates, not a bill.",
    "cost_floor": "Reconstructed lower bound - this CLI reports context occupancy, not billing.",
    "cost_free": "Served free at the time of the run.",
    "effort_verified_ceiling": "The reasoning tier was verified to bind, and it is this model's top tier.",
    "effort_verified": "The reasoning tier was verified to bind.",
    "effort_first_party": "A documented tier on the vendor's own CLI.",
    "effort_clamped": "The requested tier was silently replaced by the CLI - published as a correction.",
    "effort_inert_default": "Served through a gateway whose effort parameter provably does nothing, so the run used the model's default."
  },
  "caveats": [
    "One round per model per repo. Same-setting variance is real and measured: three Grok 4.5 runs on identical settings scored 16, 13 and 17.",
    "Wall clock is not comparable across all rows - some waves ran their two repo legs concurrently, some sequentially, and one row excludes a harness stall. Where it matters the row says so.",
    "Cost is not one kind of number. Each row is tagged: an actual bill, a list-rate estimate, a reconstructed floor, or free.",
    "50 of the 105 bugs have survived every model in every run to date. The count moves rarely: three bugs have ever been taken off it."
  ],
  "runs": [
    {
      "id": "GPT-5.6 Sol (max effort)",
      "model": "GPT-5.6 Sol",
      "vendor": "OpenAI",
      "harness": "Codex CLI",
      "effort": "max",
      "effort_status": "first_party",
      "date": "2026-08-01",
      "color": "#2E9C81",
      "fixed": 42,
      "repo1_fixed": 19,
      "repo2_fixed": 23,
      "partial": 3,
      "claimed_only": 0,
      "extras": 40,
      "false_positive_fixes": 0,
      "wall_min": 163.8,
      "cost_usd": 69.61,
      "cost_kind": "list",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "GPT-5.6 Sol (re-run Jul 31)",
      "model": "GPT-5.6 Sol",
      "vendor": "OpenAI",
      "harness": "Codex CLI",
      "effort": "high",
      "effort_status": "first_party",
      "date": "2026-07-31",
      "color": "#82C4B2",
      "fixed": 34,
      "repo1_fixed": 13,
      "repo2_fixed": 21,
      "partial": 4,
      "claimed_only": 0,
      "extras": 28,
      "false_positive_fixes": 0,
      "wall_min": 66.7,
      "cost_usd": 33.92,
      "cost_kind": "list",
      "ceiling": false,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "GPT-5.6 Luna (max effort)",
      "model": "GPT-5.6 Luna",
      "vendor": "OpenAI",
      "harness": "Codex CLI",
      "effort": "max",
      "effort_status": "first_party",
      "date": "2026-07-31",
      "color": "#2596BE",
      "fixed": 33,
      "repo1_fixed": 17,
      "repo2_fixed": 16,
      "partial": 4,
      "claimed_only": 0,
      "extras": 31,
      "false_positive_fixes": 0,
      "wall_min": 85.7,
      "cost_usd": 1.8,
      "cost_kind": "list",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "GPT-5.6 Sol",
      "model": "GPT-5.6 Sol",
      "vendor": "OpenAI",
      "harness": "Codex CLI",
      "effort": "high",
      "effort_status": "first_party",
      "date": "2026-07-26",
      "color": "#82C4B2",
      "fixed": 31,
      "repo1_fixed": 13,
      "repo2_fixed": 18,
      "partial": 4,
      "claimed_only": 1,
      "extras": 38,
      "false_positive_fixes": 0,
      "wall_min": 70.2,
      "cost_usd": 29.16,
      "cost_kind": "list",
      "ceiling": false,
      "featured": false,
      "superseded": "Superseded by the Jul 31 re-run on the same setting (serving-side change, not a new model).",
      "caveat": null,
      "note": null
    },
    {
      "id": "Fable 5 (max effort)",
      "model": "Fable 5",
      "vendor": "Anthropic",
      "harness": "Claude Code",
      "effort": "max",
      "effort_status": "first_party",
      "date": "2026-08-01",
      "color": "#D97757",
      "fixed": 29,
      "repo1_fixed": 12,
      "repo2_fixed": 17,
      "partial": 0,
      "claimed_only": 0,
      "extras": 5,
      "false_positive_fixes": 0,
      "wall_min": 57.3,
      "cost_usd": 104.49,
      "cost_kind": "list",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Grok 4.6 (xhigh)",
      "model": "Grok 4.6",
      "vendor": "xAI",
      "harness": "Grok Build CLI (ACP)",
      "effort": "xhigh",
      "effort_status": "verified_ceiling",
      "date": "2026-08-12",
      "color": "#1f242b",
      "fixed": 27,
      "repo1_fixed": 10,
      "repo2_fixed": 17,
      "partial": 2,
      "claimed_only": 0,
      "extras": 15,
      "false_positive_fixes": 0,
      "wall_min": 34.4,
      "cost_usd": 22.73,
      "cost_kind": "floor",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Opus 5 (max effort)",
      "model": "Opus 5",
      "vendor": "Anthropic",
      "harness": "Claude Code",
      "effort": "max",
      "effort_status": "first_party",
      "date": "2026-08-01",
      "color": "#C88A2E",
      "fixed": 27,
      "repo1_fixed": 13,
      "repo2_fixed": 14,
      "partial": 2,
      "claimed_only": 5,
      "extras": 2,
      "false_positive_fixes": 0,
      "wall_min": 60.0,
      "cost_usd": 51.33,
      "cost_kind": "list",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Fable 5",
      "model": "Fable 5",
      "vendor": "Anthropic",
      "harness": "Claude Code",
      "effort": "high",
      "effort_status": "first_party",
      "date": "2026-07-26",
      "color": "#ECAE97",
      "fixed": 24,
      "repo1_fixed": 9,
      "repo2_fixed": 15,
      "partial": 0,
      "claimed_only": 1,
      "extras": 3,
      "false_positive_fixes": 0,
      "wall_min": 31.4,
      "cost_usd": 68.07,
      "cost_kind": "list",
      "ceiling": false,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Gemini 3.7 Flash (high)",
      "model": "Gemini 3.7 Flash",
      "vendor": "Google",
      "harness": "Gemini CLI (retired) + model-pinning gateway",
      "effort": "high",
      "effort_status": "verified_ceiling",
      "date": "2026-08-24",
      "color": "#9DC0F7",
      "fixed": 22,
      "repo1_fixed": 8,
      "repo2_fixed": 14,
      "partial": 0,
      "claimed_only": 0,
      "extras": 4,
      "false_positive_fixes": 0,
      "wall_min": 96.8,
      "cost_usd": 8.43,
      "cost_kind": "list",
      "ceiling": false,
      "featured": false,
      "superseded": null,
      "caveat": "Gemini CLI was retired on Jun 18, 2026; it silently served 3.5 Flash until the model was pinned on the wire. Wall clock excludes a 35-minute harness stall.",
      "note": null
    },
    {
      "id": "Kimi K3",
      "model": "Kimi K3",
      "vendor": "Moonshot AI",
      "harness": "Claude Code / OpenRouter",
      "effort": "default",
      "effort_status": "inert_default",
      "date": "2026-07-26",
      "color": "#5B5EA6",
      "fixed": 21,
      "repo1_fixed": 4,
      "repo2_fixed": 17,
      "partial": 0,
      "claimed_only": 6,
      "extras": 6,
      "false_positive_fixes": 0,
      "wall_min": 107.8,
      "cost_usd": 25.27,
      "cost_kind": "list",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Opus 5",
      "model": "Opus 5",
      "vendor": "Anthropic",
      "harness": "Claude Code",
      "effort": "high",
      "effort_status": "first_party",
      "date": "2026-07-26",
      "color": "#DDBA77",
      "fixed": 21,
      "repo1_fixed": 11,
      "repo2_fixed": 10,
      "partial": 0,
      "claimed_only": 3,
      "extras": 6,
      "false_positive_fixes": 0,
      "wall_min": 37.4,
      "cost_usd": 38.77,
      "cost_kind": "list",
      "ceiling": false,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Qwen3.8-Max",
      "model": "Qwen3.8-Max",
      "vendor": "Alibaba",
      "harness": "Claude Code / Alibaba API",
      "effort": "xhigh",
      "effort_status": "verified_ceiling",
      "date": "2026-08-03",
      "color": "#7C3AED",
      "fixed": 19,
      "repo1_fixed": 5,
      "repo2_fixed": 14,
      "partial": 0,
      "claimed_only": 1,
      "extras": 6,
      "false_positive_fixes": 0,
      "wall_min": 148.1,
      "cost_usd": 31.1,
      "cost_kind": "list",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Gemini 3.7 Flash (high; Antigravity CLI retest)",
      "model": "Gemini 3.7 Flash",
      "vendor": "Google",
      "harness": "Antigravity CLI",
      "effort": "high",
      "effort_status": "verified_ceiling",
      "date": "2026-08-25",
      "color": "#4285F4",
      "fixed": 18,
      "repo1_fixed": 4,
      "repo2_fixed": 14,
      "partial": 3,
      "claimed_only": 0,
      "extras": 11,
      "false_positive_fixes": 0,
      "wall_min": 37.5,
      "cost_usd": 8.57,
      "cost_kind": "list",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": "Google's current CLI."
    },
    {
      "id": "Grok 4.5 (re-run Aug 6)",
      "model": "Grok 4.5",
      "vendor": "xAI",
      "harness": "Grok Build CLI (ACP)",
      "effort": "high",
      "effort_status": "verified_ceiling",
      "date": "2026-08-06",
      "color": "#5A6472",
      "fixed": 17,
      "repo1_fixed": 5,
      "repo2_fixed": 12,
      "partial": 0,
      "claimed_only": 1,
      "extras": 10,
      "false_positive_fixes": 0,
      "wall_min": 27.9,
      "cost_usd": 8.5,
      "cost_kind": "floor",
      "ceiling": true,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Muse Spark 1.2 (xhigh, Meta API)",
      "model": "Muse Spark 1.2",
      "vendor": "Meta",
      "harness": "Claude Code / Meta API",
      "effort": "xhigh",
      "effort_status": "verified_ceiling",
      "date": "2026-08-06",
      "color": "#C2255C",
      "fixed": 17,
      "repo1_fixed": 6,
      "repo2_fixed": 11,
      "partial": 0,
      "claimed_only": 0,
      "extras": 12,
      "false_positive_fixes": 0,
      "wall_min": 35.7,
      "cost_usd": 13.99,
      "cost_kind": "list",
      "ceiling": true,
      "featured": false,
      "superseded": null,
      "caveat": "Mixed serving path: repo 1 ran Meta-direct at verified xhigh, repo 2 through OpenRouter at default effort. The single-path number for this model is the 14/105 row.",
      "note": null
    },
    {
      "id": "Ox Alpha (stealth; free)",
      "model": "Ox Alpha (stealth)",
      "vendor": "unidentified - fingerprints as Z.ai GLM",
      "harness": "Claude Code / OpenRouter",
      "effort": "default",
      "effort_status": "inert_default",
      "date": "2026-08-25",
      "color": "#0B6E6E",
      "fixed": 16,
      "repo1_fixed": 8,
      "repo2_fixed": 8,
      "partial": 1,
      "claimed_only": 1,
      "extras": 3,
      "false_positive_fixes": 0,
      "wall_min": 59.4,
      "cost_usd": 0.0,
      "cost_kind": "free",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": "OpenRouter stealth slug, free at the time of the run. Wire fingerprint (tokenizer, special tokens, model card) matches Z.ai's GLM-5.3 serving stack. Fixed H3 - a repo-1 bug that had survived every model in every prior run."
    },
    {
      "id": "Grok 4.5",
      "model": "Grok 4.5",
      "vendor": "xAI",
      "harness": "Grok Build CLI (ACP)",
      "effort": "high",
      "effort_status": "verified_ceiling",
      "date": "2026-07-26",
      "color": "#5A6472",
      "fixed": 16,
      "repo1_fixed": 5,
      "repo2_fixed": 11,
      "partial": 1,
      "claimed_only": 3,
      "extras": 7,
      "false_positive_fixes": 0,
      "wall_min": 24.9,
      "cost_usd": 2.58,
      "cost_kind": "floor",
      "ceiling": false,
      "featured": false,
      "superseded": "One of three runs at the same setting (16 / 13 / 17) - the spread is this board's variance evidence.",
      "caveat": null,
      "note": null
    },
    {
      "id": "DeepSeek V4-Flash 0731",
      "model": "DeepSeek V4-Flash",
      "vendor": "DeepSeek",
      "harness": "Claude Code / OpenRouter",
      "effort": "default",
      "effort_status": "inert_default",
      "date": "2026-08-01",
      "color": "#4D6BFE",
      "fixed": 14,
      "repo1_fixed": 6,
      "repo2_fixed": 8,
      "partial": 1,
      "claimed_only": 2,
      "extras": 0,
      "false_positive_fixes": 0,
      "wall_min": 48.5,
      "cost_usd": 1.74,
      "cost_kind": "bill",
      "ceiling": true,
      "featured": true,
      "superseded": null,
      "caveat": null,
      "note": "The Jul 31 re-post-trained revision."
    },
    {
      "id": "Muse Spark 1.2 (default effort, OpenRouter)",
      "model": "Muse Spark 1.2",
      "vendor": "Meta",
      "harness": "Claude Code / OpenRouter",
      "effort": "default",
      "effort_status": "inert_default",
      "date": "2026-08-06",
      "color": "#E88AA6",
      "fixed": 14,
      "repo1_fixed": 3,
      "repo2_fixed": 11,
      "partial": 0,
      "claimed_only": 0,
      "extras": 3,
      "false_positive_fixes": 0,
      "wall_min": 65.2,
      "cost_usd": 19.52,
      "cost_kind": "bill",
      "ceiling": false,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "GPT-5.6 Luna (high effort)",
      "model": "GPT-5.6 Luna",
      "vendor": "OpenAI",
      "harness": "Codex CLI",
      "effort": "high",
      "effort_status": "first_party",
      "date": "2026-07-31",
      "color": "#7FC0D8",
      "fixed": 13,
      "repo1_fixed": 5,
      "repo2_fixed": 8,
      "partial": 2,
      "claimed_only": 0,
      "extras": 22,
      "false_positive_fixes": 0,
      "wall_min": 64.1,
      "cost_usd": 0.57,
      "cost_kind": "list",
      "ceiling": false,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Grok 4.5 (high re-run - CLI has no max)",
      "model": "Grok 4.5",
      "vendor": "xAI",
      "harness": "Grok Build CLI (ACP)",
      "effort": "high",
      "effort_status": "clamped",
      "date": "2026-08-01",
      "color": "#5A6472",
      "fixed": 13,
      "repo1_fixed": 5,
      "repo2_fixed": 8,
      "partial": 3,
      "claimed_only": 1,
      "extras": 5,
      "false_positive_fixes": 0,
      "wall_min": 25.4,
      "cost_usd": 10.94,
      "cost_kind": "floor",
      "ceiling": false,
      "featured": false,
      "superseded": "Same setting as the other Grok 4.5 runs.",
      "caveat": "Commissioned as a max run; the CLI silently clamps unknown tiers, so this is a second high run - published as a correction.",
      "note": null
    },
    {
      "id": "DeepSeek V4-Pro",
      "model": "DeepSeek V4-Pro",
      "vendor": "DeepSeek",
      "harness": "Claude Code / OpenRouter",
      "effort": "default",
      "effort_status": "inert_default",
      "date": "2026-08-01",
      "color": "#2F3E9E",
      "fixed": 10,
      "repo1_fixed": 5,
      "repo2_fixed": 5,
      "partial": 0,
      "claimed_only": 1,
      "extras": 1,
      "false_positive_fixes": 0,
      "wall_min": 27.6,
      "cost_usd": 1.26,
      "cost_kind": "bill",
      "ceiling": true,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Sonnet 5",
      "model": "Sonnet 5",
      "vendor": "Anthropic",
      "harness": "Claude Code",
      "effort": "high",
      "effort_status": "first_party",
      "date": "2026-07-26",
      "color": "#7E6586",
      "fixed": 9,
      "repo1_fixed": 1,
      "repo2_fixed": 8,
      "partial": 0,
      "claimed_only": 0,
      "extras": 4,
      "false_positive_fixes": 0,
      "wall_min": 32.8,
      "cost_usd": 15.12,
      "cost_kind": "list",
      "ceiling": true,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "Opus 4.8",
      "model": "Opus 4.8",
      "vendor": "Anthropic",
      "harness": "Claude Code",
      "effort": "high",
      "effort_status": "first_party",
      "date": "2026-07-26",
      "color": "#9C4630",
      "fixed": 9,
      "repo1_fixed": 2,
      "repo2_fixed": 7,
      "partial": 1,
      "claimed_only": 0,
      "extras": 1,
      "false_positive_fixes": 0,
      "wall_min": 34.8,
      "cost_usd": 19.35,
      "cost_kind": "list",
      "ceiling": true,
      "featured": false,
      "superseded": null,
      "caveat": null,
      "note": null
    },
    {
      "id": "DeepSeek V4-Flash",
      "model": "DeepSeek V4-Flash",
      "vendor": "DeepSeek",
      "harness": "Claude Code / OpenRouter",
      "effort": "default",
      "effort_status": "inert_default",
      "date": "2026-07-31",
      "color": "#4D6BFE",
      "fixed": 8,
      "repo1_fixed": 4,
      "repo2_fixed": 4,
      "partial": 1,
      "claimed_only": 0,
      "extras": 4,
      "false_positive_fixes": 0,
      "wall_min": 24.9,
      "cost_usd": 0.61,
      "cost_kind": "list",
      "ceiling": false,
      "featured": false,
      "superseded": "This slug resolved to the April snapshot; the Jul 31 revision is the row above.",
      "caveat": null,
      "note": null
    }
  ]
}