{
  "alt": "Dot chart of agreement with the consumer's product for two decision engines, a general language model and one trained classifier. Jev and the classifier overlap, and Luna and the language model sit a little to the right. Their intervals overlap one another.",
  "caption": "Each dot is one model, and the whisker is its 95% interval. Agreement is not the same as being right, because some consumers pick the wrong product. Counted equally across products, Jev leads the classifier ({fact:agree_jev_equal_products} against {fact:agree_tfidf_equal_products}), because the classifier was fit to the actual mix.",
  "data": [
    {
      "agree": 81.9,
      "conf_high": 83.2,
      "conf_low": 80.5,
      "kind": "Decision engine",
      "model": "Jev 1.14.0",
      "n": 10000
    },
    {
      "agree": 84.2,
      "conf_high": 85.3,
      "conf_low": 82.9,
      "kind": "Decision engine",
      "model": "OpenAI Luna",
      "n": 10000
    },
    {
      "agree": 84.4,
      "conf_high": 85.5,
      "conf_low": 83.3,
      "kind": "General language model",
      "model": "Claude Haiku adapter",
      "n": 10000
    },
    {
      "agree": 82.4,
      "conf_high": 83.7,
      "conf_low": 81.2,
      "kind": "Trained classifier",
      "model": "TF-IDF + logistic",
      "n": 10000
    }
  ],
  "id": "agree-by-model",
  "notes": [
    "Jev minus the classifier: {fact:agree_diff_jev_tfidf} percentage points, from a bootstrap that resamples complaints within products."
  ],
  "source": "CFPB Consumer Complaint Database, 2025 complaints with narratives of at most 3,000 characters; 10,000 sampled, weighted; engine calls October 2026",
  "spec": {
    "colors": {
      "Decision engine": "cat1",
      "General language model": "cat3",
      "Trained classifier": "cat2"
    },
    "group": "kind",
    "hi": "conf_high",
    "lo": "conf_low",
    "n": "n",
    "sort": "desc",
    "x": "agree",
    "xDomain": [
      60,
      100
    ],
    "xFormat": {
      "digits": 1
    },
    "xLabel": "Matches the consumer's product (%)",
    "y": "model"
  },
  "subtitle": "Complaints whose predicted product matches the one the consumer selected, percent, by model, with 95% intervals",
  "title": "A classifier trained on earlier complaints agrees with consumers' product choices about as often as Jev",
  "type": "dot"
}