{
  "auc_across_diff_jev_size": {
    "ci": [
      -0.03,
      0.05
    ],
    "ciLevel": 0.95,
    "display": "+0.01",
    "estimand": "jev minus largest diff first",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "difference in AUC",
    "value": 0.00630002,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_across_diff_luna_size": {
    "ci": [
      -0.01,
      0.06
    ],
    "ciLevel": 0.95,
    "display": "+0.02",
    "estimand": "luna minus largest diff first",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "difference in AUC",
    "value": 0.0206545,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_across_jev": {
    "ci": [
      0.56,
      0.64
    ],
    "ciLevel": 0.95,
    "display": "0.60",
    "estimand": "unresolved ranked above resolved, patches for different issues",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "area under the ROC curve",
    "value": 0.603585,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_across_luna": {
    "ci": [
      0.57,
      0.66
    ],
    "ciLevel": 0.95,
    "display": "0.62",
    "estimand": "unresolved ranked above resolved, patches for different issues",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "area under the ROC curve",
    "value": 0.617939,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_across_size": {
    "ci": [
      0.55,
      0.65
    ],
    "ciLevel": 0.95,
    "display": "0.60",
    "estimand": "unresolved ranked above resolved, patches for different issues",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "area under the ROC curve",
    "value": 0.597285,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_addr_jev": {
    "display": "0.88",
    "estimand": "the addresses-the-issue question alone",
    "n": 40,
    "nUnit": "patches",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "area under the ROC curve",
    "value": 0.8775,
    "weights": "unweighted; evenly split by construction"
  },
  "auc_con_adapter": {
    "ci": [
      0.64,
      0.89
    ],
    "ciLevel": 0.95,
    "display": "0.78",
    "estimand": "ranks defective patches above acceptable ones",
    "n": 40,
    "nUnit": "patches",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "area under the ROC curve",
    "value": 0.77625,
    "weights": "unweighted; evenly split by construction"
  },
  "auc_con_jev": {
    "ci": [
      0.69,
      0.89
    ],
    "ciLevel": 0.95,
    "display": "0.79",
    "estimand": "ranks defective patches above acceptable ones",
    "n": 40,
    "nUnit": "patches",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "area under the ROC curve",
    "value": 0.785,
    "weights": "unweighted; evenly split by construction"
  },
  "auc_con_luna": {
    "ci": [
      0.77,
      0.99
    ],
    "ciLevel": 0.95,
    "display": "0.89",
    "estimand": "ranks defective patches above acceptable ones",
    "n": 40,
    "nUnit": "patches",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "area under the ROC curve",
    "value": 0.8875,
    "weights": "unweighted; evenly split by construction"
  },
  "auc_con_size": {
    "ci": [
      0.33,
      0.55
    ],
    "ciLevel": 0.95,
    "display": "0.45",
    "estimand": "ranks defective patches above acceptable ones",
    "n": 40,
    "nUnit": "patches",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "area under the ROC curve",
    "value": 0.4475,
    "weights": "unweighted; evenly split by construction"
  },
  "auc_dawes_jev": {
    "display": "0.66",
    "estimand": "equal weight on six direction-fixed questions",
    "n": 40,
    "nUnit": "patches",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "area under the ROC curve",
    "value": 0.6625,
    "weights": "unweighted; evenly split by construction"
  },
  "auc_within_diff_jev_size": {
    "ci": [
      -0.04,
      0.16
    ],
    "ciLevel": 0.95,
    "display": "+0.06",
    "estimand": "jev minus largest diff first",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "difference in AUC",
    "value": 0.0620818,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_within_diff_luna_size": {
    "ci": [
      -0.05,
      0.13
    ],
    "ciLevel": 0.95,
    "display": "+0.04",
    "estimand": "luna minus largest diff first",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "difference in AUC",
    "value": 0.0396528,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_within_jev": {
    "ci": [
      0.51,
      0.65
    ],
    "ciLevel": 0.95,
    "display": "0.58",
    "estimand": "unresolved ranked above resolved, patches for the same issue",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "area under the ROC curve",
    "value": 0.579562,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_within_luna": {
    "ci": [
      0.49,
      0.64
    ],
    "ciLevel": 0.95,
    "display": "0.56",
    "estimand": "unresolved ranked above resolved, patches for the same issue",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "area under the ROC curve",
    "value": 0.557133,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "auc_within_size": {
    "ci": [
      0.44,
      0.61
    ],
    "ciLevel": 0.95,
    "display": "0.52",
    "estimand": "unresolved ranked above resolved, patches for the same issue",
    "n": 1941,
    "nUnit": "patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "area under the ROC curve",
    "value": 0.51748,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "detect_rate": {
    "display": "0.7",
    "estimand": "assumed reviewer detection rate",
    "note": "",
    "period": "assumed inputs, October 2026",
    "population": "a weekly review budget (assumed inputs)",
    "source": "constructed-patches",
    "unit": "share of defects a reviewer finds",
    "value": 0.7,
    "weights": "not applicable; assumed inputs"
  },
  "found10_jev": {
    "display": "9",
    "estimand": "defective patches among the ten ranked riskiest",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "defective patches of 20",
    "value": 9,
    "weights": "unweighted; evenly split by construction"
  },
  "found10_luna": {
    "display": "10",
    "estimand": "defective patches among the ten ranked riskiest",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "defective patches of 20",
    "value": 10,
    "weights": "unweighted; evenly split by construction"
  },
  "found10_size": {
    "display": "5",
    "estimand": "defective patches among the ten ranked riskiest",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "defective patches of 20",
    "value": 5,
    "weights": "unweighted; evenly split by construction"
  },
  "hour_cost": {
    "display": "$150",
    "estimand": "assumed cost",
    "note": "",
    "period": "assumed inputs, October 2026",
    "population": "a weekly review budget (assumed inputs)",
    "source": "constructed-patches",
    "unit": "US dollars per reviewer hour",
    "value": 150,
    "weights": "not applicable; assumed inputs"
  },
  "keyword_hits": {
    "display": "0",
    "estimand": "patches whose added lines match auth, security or api",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "patches of 40",
    "value": 0,
    "weights": "unweighted; evenly split by construction"
  },
  "miss_cost": {
    "display": "$2,000",
    "estimand": "assumed cost",
    "note": "",
    "period": "assumed inputs, October 2026",
    "population": "a weekly review budget (assumed inputs)",
    "source": "constructed-patches",
    "unit": "US dollars per defect that ships",
    "value": 2000,
    "weights": "not applicable; assumed inputs"
  },
  "n_con": {
    "display": "40",
    "estimand": "constructed patches",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "patches",
    "value": 40,
    "weights": "unweighted; evenly split by construction"
  },
  "n_empty": {
    "display": "59",
    "estimand": "runs that ended with no diff, excluded",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "agent runs",
    "value": 59,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "n_hist": {
    "display": "1,941",
    "estimand": "sampled agent patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "patches",
    "value": 1941,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "n_hist_issues": {
    "display": "259",
    "estimand": "issues in the sample",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "issues",
    "value": 259,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "n_issues": {
    "display": "20",
    "estimand": "issues, each with one acceptable and one defective patch",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "issues",
    "value": 20,
    "weights": "unweighted; evenly split by construction"
  },
  "n_mixed": {
    "display": "215",
    "estimand": "issues with both resolved and unresolved patches",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "issues",
    "value": 215,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "pairs_jev": {
    "display": "18.5",
    "estimand": "defective patch rated riskier than the acceptable patch for the same issue; a tie is split evenly",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "issues of 20",
    "value": 18.5,
    "weights": "unweighted; evenly split by construction"
  },
  "pairs_luna": {
    "display": "18",
    "estimand": "defective patch rated riskier than the acceptable patch for the same issue; a tie is split evenly",
    "note": "",
    "period": "patches written October 2026; engines called October 2026",
    "population": "40 patches written for this booklet",
    "source": "constructed-patches",
    "unit": "issues of 20",
    "value": 18,
    "weights": "unweighted; evenly split by construction"
  },
  "share_empty": {
    "display": "2.6%",
    "estimand": "runs that ended with no diff, weighted",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "percent of agent runs",
    "value": 2.58,
    "weights": "sample_weight restores the outcome mix of the patches"
  },
  "share_unresolved": {
    "display": "43.9%",
    "estimand": "benchmark outcome is unresolved",
    "note": "",
    "period": "issues created by March 2025; engines called October 2026",
    "population": "1,941 agent patches for 259 issues in 20 repositories",
    "source": "swe-rebench",
    "unit": "percent of patches",
    "value": 43.9426,
    "weights": "sample_weight restores the outcome mix of the patches"
  }
}