{
  "alt": "Dumbbell chart with one row per policy, each joining an across-issues AUC to a same-issue AUC. The model policies sit modestly above the coin-flip value on both. The rule policies fall to about the coin-flip value within an issue.",
  "caption": "The open dot compares patches across all issues, and the filled dot compares patches for the same issue. Both sit far left of the scores on the written patches. This label is a benchmark outcome and not a count of defects that would ship.",
  "data": [
    {
      "across": 0.604,
      "hi1": 0.642,
      "hi2": 0.646,
      "lo1": 0.559,
      "lo2": 0.506,
      "n1": 1941,
      "n2": 1941,
      "policy": "Jev 1.14.0, whole question",
      "within": 0.58
    },
    {
      "across": 0.618,
      "hi1": 0.665,
      "hi2": 0.642,
      "lo1": 0.574,
      "lo2": 0.49,
      "n1": 1941,
      "n2": 1941,
      "policy": "OpenAI Luna, whole question",
      "within": 0.557
    },
    {
      "across": 0.608,
      "hi1": 0.651,
      "hi2": 0.624,
      "lo1": 0.567,
      "lo2": 0.502,
      "n1": 1941,
      "n2": 1941,
      "policy": "Claude Haiku adapter, whole question",
      "within": 0.562
    },
    {
      "across": 0.572,
      "hi1": 0.614,
      "hi2": 0.678,
      "lo1": 0.53,
      "lo2": 0.524,
      "n1": 1941,
      "n2": 1941,
      "policy": "Jev 1.14.0, equal weight",
      "within": 0.595
    },
    {
      "across": 0.542,
      "hi1": 0.58,
      "hi2": 0.667,
      "lo1": 0.498,
      "lo2": 0.503,
      "n1": 1941,
      "n2": 1941,
      "policy": "Jev 1.14.0, fitted weights",
      "within": 0.578
    },
    {
      "across": 0.597,
      "hi1": 0.648,
      "hi2": 0.608,
      "lo1": 0.554,
      "lo2": 0.442,
      "n1": 1941,
      "n2": 1941,
      "policy": "Largest diff first",
      "within": 0.517
    },
    {
      "across": 0.543,
      "hi1": 0.594,
      "hi2": 0.601,
      "lo1": 0.494,
      "lo2": 0.444,
      "n1": 1941,
      "n2": 1941,
      "policy": "File-ownership rule",
      "within": 0.512
    },
    {
      "across": 0.575,
      "hi1": 0.623,
      "hi2": 0.608,
      "lo1": 0.531,
      "lo2": 0.438,
      "n1": 1941,
      "n2": 1941,
      "policy": "Keyword rule",
      "within": 0.515
    }
  ],
  "id": "same-issue-is-harder",
  "source": "SWE-rebench and Nebius OpenHands trajectories, issues created by 2025; 2,000 patches, weighted; engine calls October 2026",
  "spec": {
    "hi1": "hi1",
    "hi2": "hi2",
    "label1": "Across issues",
    "label2": "Same issue",
    "lo1": "lo1",
    "lo2": "lo2",
    "n1": "n1",
    "n2": "n2",
    "sort": "data",
    "x1": "across",
    "x2": "within",
    "xDomain": [
      0.4,
      0.9
    ],
    "xFormat": {
      "digits": 2
    },
    "xLabel": "AUC, unresolved ranked above resolved",
    "y": "policy"
  },
  "subtitle": "AUC for the benchmark's unresolved label, across all patches and within the same issue, with 95% intervals",
  "title": "On real agent patches, neither the models nor the rules clearly beat sorting by size",
  "type": "dumbbell"
}