{
  "alt": "Dot chart with one dot per model on an AUC scale, all well below the perfect score and above the coin-flip line, with wide overlapping intervals.",
  "caption": "The dashed line marks a coin flip, and the right edge of the scale is perfect. Each whisker is an interval from resampling tasks, and with this few tasks the intervals are wide.",
  "data": [
    {
      "auc": 0.713,
      "conf_high": 0.83,
      "conf_low": 0.551,
      "kind": "Decision engine",
      "model": "Jev 1.14.0",
      "n": 200
    },
    {
      "auc": 0.76,
      "conf_high": 0.862,
      "conf_low": 0.624,
      "kind": "Decision engine",
      "model": "OpenAI Luna",
      "n": 200
    },
    {
      "auc": 0.765,
      "conf_high": 0.873,
      "conf_low": 0.621,
      "kind": "General language model",
      "model": "Claude Haiku adapter",
      "n": 200
    }
  ],
  "id": "judging-done",
  "source": "Constructed agent tasks written for this booklet; one agent (GPT-6 Luna), 200 runs with injected faults, October 2026; unweighted",
  "spec": {
    "colors": {
      "Decision engine": "cat1",
      "General language model": "cat3"
    },
    "group": "kind",
    "hi": "conf_high",
    "lo": "conf_low",
    "n": "n",
    "reference": 0.5,
    "referenceLabel": "Coin flip",
    "sort": "desc",
    "x": "auc",
    "xDomain": [
      0.4,
      1
    ],
    "xFormat": {
      "digits": 2
    },
    "xLabel": "AUC, judging whether the work is done",
    "y": "model"
  },
  "subtitle": "AUC of each model's P(done) against the grader's verdict, with 95% intervals, finish attempts",
  "title": "Reading the trace tells the models something about whether the work is done, with wide intervals on this few tasks",
  "type": "dot"
}