{
  "alt": "Dot chart of AUC for the trained classifier and for Jev and Luna under four versions of the money-back question. The classifier is furthest right. For both engines, the version with a description of past outcomes is the highest of the four, the original question is the lowest, and ten labeled examples land in between.",
  "caption": "Each dot is one model with one version of the question. Moving right means the model ranks money-back complaints above the rest more often. The classifier learned from earlier outcomes; the engines saw only the question, plus ten labeled examples or a short description of what tended to get paid.",
  "data": [
    {
      "auc": 0.947,
      "conf_high": 0.959,
      "conf_low": 0.932,
      "group": "Classifier trained on past outcomes",
      "label": "Classifier",
      "n": 2000
    },
    {
      "auc": 0.881,
      "conf_high": 0.912,
      "conf_low": 0.847,
      "group": "Jev",
      "label": "Jev: original question",
      "n": 2000
    },
    {
      "auc": 0.899,
      "conf_high": 0.922,
      "conf_low": 0.873,
      "group": "Jev",
      "label": "Jev: reworded to predict payment",
      "n": 2000
    },
    {
      "auc": 0.903,
      "conf_high": 0.926,
      "conf_low": 0.878,
      "group": "Jev",
      "label": "Jev: reworded + ten labeled examples",
      "n": 2000
    },
    {
      "auc": 0.913,
      "conf_high": 0.931,
      "conf_low": 0.891,
      "group": "Jev",
      "label": "Jev: reworded + outcome criteria",
      "n": 2000
    },
    {
      "auc": 0.887,
      "conf_high": 0.92,
      "conf_low": 0.845,
      "group": "Luna",
      "label": "Luna: original question",
      "n": 2000
    },
    {
      "auc": 0.903,
      "conf_high": 0.929,
      "conf_low": 0.875,
      "group": "Luna",
      "label": "Luna: reworded to predict payment",
      "n": 2000
    },
    {
      "auc": 0.889,
      "conf_high": 0.911,
      "conf_low": 0.866,
      "group": "Luna",
      "label": "Luna: reworded + ten labeled examples",
      "n": 2000
    },
    {
      "auc": 0.915,
      "conf_high": 0.934,
      "conf_low": 0.891,
      "group": "Luna",
      "label": "Luna: reworded + outcome criteria",
      "n": 2000
    }
  ],
  "id": "money-back-prompts",
  "notes": [
    "Only about one complaint in forty ended in money back, so the intervals are wide."
  ],
  "source": "CFPB Consumer Complaint Database, 2025 complaints; 2,000 drawn at random from the sample, weighted; engine calls October 2026",
  "spec": {
    "colors": {
      "Classifier trained on past outcomes": "cat2",
      "Jev": "cat1",
      "Luna": "cat3"
    },
    "group": "group",
    "hi": "conf_high",
    "lo": "conf_low",
    "n": "n",
    "sort": "data",
    "x": "auc",
    "xDomain": [
      0.75,
      1
    ],
    "xFormat": {
      "digits": 3
    },
    "xLabel": "AUC, money back (higher is better)",
    "y": "label"
  },
  "subtitle": "AUC for complaints closed with monetary relief, by model and how the question was posed, with 95% intervals; 2,000 sampled 2025 complaints",
  "title": "Describing past outcomes in the question helps the engines a little, and examples do not close the gap to the classifier",
  "type": "dot"
}