{
  "alt": "Dot chart with one row per question. For unhappy reviewers, both models are near the top of the scale. For money back within a product, the classifier sits a little ahead of Jev. For written patches Jev sits well above real patches, and whether an agent's work is done falls between them.",
  "caption": "Read down the rows. Each question has its own label, population and weighting, so these scores are not directly comparable. One reading is that they track how much of the answer is visible in the text, which these cases cannot test.",
  "data": [
    {
      "auc": 0.980376,
      "conf_high": 0.982,
      "conf_low": 0.978,
      "model": "Jev",
      "task": "Is this reviewer unhappy?"
    },
    {
      "auc": 0.968036,
      "conf_high": 0.972,
      "conf_low": 0.964,
      "model": "Trained classifier",
      "task": "Is this reviewer unhappy?"
    },
    {
      "auc": 0.741876,
      "conf_high": 0.77,
      "conf_low": 0.709,
      "model": "Jev",
      "task": "Will this complaint end in money back, within a product?"
    },
    {
      "auc": 0.768982,
      "conf_high": 0.802,
      "conf_low": 0.736,
      "model": "Trained classifier",
      "task": "Will this complaint end in money back, within a product?"
    },
    {
      "auc": 0.785,
      "conf_high": 0.89,
      "conf_low": 0.69,
      "model": "Jev",
      "task": "Would this written patch fail its hidden test?"
    },
    {
      "auc": 0.603585,
      "conf_high": 0.64,
      "conf_low": 0.56,
      "model": "Jev",
      "task": "Did this real agent patch fail its benchmark?"
    },
    {
      "auc": 0.713257,
      "conf_high": 0.83,
      "conf_low": 0.55,
      "model": "Jev",
      "task": "Is the agent's work really done?"
    }
  ],
  "id": "same-model-three-questions",
  "source": "Case chapters of this booklet: coffee reviews and CFPB complaints, weighted; SWE-rebench patches, weighted; constructed patches and agent tasks, unweighted; engine calls October 2026; costs assumed",
  "spec": {
    "colors": {
      "Jev": "cat1",
      "Trained classifier": "cat5"
    },
    "group": "model",
    "hi": "conf_high",
    "lo": "conf_low",
    "reference": 0.5,
    "referenceLabel": "Coin flip",
    "sort": "data",
    "x": "auc",
    "xDomain": [
      0.4,
      1
    ],
    "xFormat": {
      "digits": 2
    },
    "xLabel": "AUC",
    "y": "task"
  },
  "subtitle": "AUC by question, Jev and the trained classifier, with 95% intervals; no classifier was trained for the patch and agent cases",
  "title": "Jev's ranking score differs widely from one question to the next",
  "type": "dot"
}