{
  "schema": 1,
  "evaluated": "2026-10-19",
  "models": {
    "0.8b": {
      "repo": "lothnic/themis-0.8b",
      "revision": "d135c1fa312ab2091ca302f4ff5389e1876b72f6",
      "step": 4153,
      "context": 2048
    },
    "2b": {
      "repo": "lothnic/themis-vl-2b",
      "revision": "86bdc06930bca9717caec4e6f4a296e32d913af8",
      "step": 3258,
      "context": 4096
    }
  },
  "decisionbench": {
    "name": "Atlan Decision Bench bench-v4",
    "rows": 1071,
    "revision": "353cc17388ca67913320a39dacd043524a35090a",
    "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
    "overall": {
      "n": 1071,
      "correct": 889,
      "accuracy": 0.8300653594771242,
      "wilson_95": [
        0.8064013619587728,
        0.851369978488043
      ],
      "brier": 0.23375115745798444,
      "nll": 0.40579411307024194,
      "ece_10bin": 0.024745223429801854
    },
    "raw": {
      "n": 1071,
      "correct": 889,
      "accuracy": 0.8300653594771242,
      "wilson_95": [
        0.8064013619587728,
        0.851369978488043
      ],
      "brier": 0.23890439169159144,
      "nll": 0.41896512174802786,
      "ece_10bin": 0.04766178305764628
    },
    "text_baseline": {
      "n": 949,
      "correct": 688,
      "accuracy": 0.7249736564805058,
      "wilson_95": [
        0.6957000006593718,
        0.7524333126501542
      ],
      "brier": 0.35628704320295485,
      "nll": 0.6233650821561088,
      "ece_10bin": 0.03043775777042981
    },
    "text_baseline_strict": {
      "n": 949,
      "correct": 670,
      "accuracy": 0.7060063224446786,
      "wilson_95": [
        0.6762363604064593,
        0.7341152215701795
      ],
      "brier": 0.34924103534634166,
      "nll": 0.6173254965496132,
      "ece_10bin": 0.03315468122692484
    },
    "strict_correct": 884,
    "modalities": {
      "image": {
        "n": 60,
        "correct": 57,
        "accuracy": 0.95,
        "wilson_95": [
          0.8629926537490016,
          0.9828508088208587
        ],
        "brier": 0.07700115709804695,
        "nll": 0.13402728607031422,
        "ece_10bin": 0.040320357680320745
      },
      "text": {
        "n": 949,
        "correct": 775,
        "accuracy": 0.8166491043203372,
        "wilson_95": [
          0.7907693261533318,
          0.8399755951431355
        ],
        "brier": 0.25086539355243354,
        "nll": 0.43638094380944414,
        "ece_10bin": 0.02478873692021103
      },
      "text+image": {
        "n": 62,
        "correct": 57,
        "accuracy": 0.9193548387096774,
        "wilson_95": [
          0.824712077925188,
          0.9650621681441662
        ],
        "brier": 0.12348647952192036,
        "nll": 0.20061842352980244,
        "ece_10bin": 0.05966712053745025
      }
    },
    "categories": {
      "AI agents & evals": {
        "n": 154,
        "correct": 116,
        "accuracy": 0.7532467532467533,
        "wilson_95": [
          0.6795430976326048,
          0.8146232058007107
        ],
        "brier": 0.3215660881870158,
        "nll": 0.479531306189952,
        "ece_10bin": 0.06925412006192393
      },
      "Customer support": {
        "n": 67,
        "correct": 62,
        "accuracy": 0.9253731343283582,
        "wilson_95": [
          0.8369075523798613,
          0.967704427022072
        ],
        "brier": 0.12795552009982336,
        "nll": 0.26981688129353126,
        "ece_10bin": 0.09484563211896527
      },
      "Data & analytics": {
        "n": 126,
        "correct": 110,
        "accuracy": 0.873015873015873,
        "wilson_95": [
          0.8036548163021325,
          0.9203042230196258
        ],
        "brier": 0.18026740760486976,
        "nll": 0.33489470684764516,
        "ece_10bin": 0.042787277509295736
      },
      "Design": {
        "n": 30,
        "correct": 27,
        "accuracy": 0.9,
        "wilson_95": [
          0.7437855868473339,
          0.9654007400402839
        ],
        "brier": 0.15366041733122054,
        "nll": 0.25909538513696023,
        "ece_10bin": 0.08948132991790768
      },
      "Documents & meetings": {
        "n": 87,
        "correct": 58,
        "accuracy": 0.6666666666666666,
        "wilson_95": [
          0.5624216326943015,
          0.7568153688556492
        ],
        "brier": 0.4490501424809068,
        "nll": 0.7725151891744416,
        "ece_10bin": 0.10849197979631095
      },
      "Engineering": {
        "n": 150,
        "correct": 119,
        "accuracy": 0.7933333333333333,
        "wilson_95": [
          0.7216049669151186,
          0.8504119647860597
        ],
        "brier": 0.2762172991560779,
        "nll": 0.4845433130226949,
        "ece_10bin": 0.07724843978881832
      },
      "Finance": {
        "n": 121,
        "correct": 114,
        "accuracy": 0.9421487603305785,
        "wilson_95": [
          0.885388492644203,
          0.9716976068610899
        ],
        "brier": 0.075883986935171,
        "nll": 0.13649248499955696,
        "ece_10bin": 0.033926915285015936
      },
      "Legal": {
        "n": 126,
        "correct": 113,
        "accuracy": 0.8968253968253969,
        "wilson_95": [
          0.8314608773520775,
          0.9387083134157506
        ],
        "brier": 0.15196735954503052,
        "nll": 0.31802730891486425,
        "ece_10bin": 0.05335422380576061
      },
      "Product management": {
        "n": 60,
        "correct": 36,
        "accuracy": 0.6,
        "wilson_95": [
          0.47365824394206846,
          0.714306969962345
        ],
        "brier": 0.47137022159039454,
        "nll": 0.741193193074665,
        "ece_10bin": 0.12691946725050607
      },
      "Sales & commerce": {
        "n": 90,
        "correct": 78,
        "accuracy": 0.8666666666666667,
        "wilson_95": [
          0.7812588689264784,
          0.9220539475642889
        ],
        "brier": 0.185875609321068,
        "nll": 0.34876967277495685,
        "ece_10bin": 0.10566489464706845
      },
      "Trust & safety": {
        "n": 60,
        "correct": 56,
        "accuracy": 0.9333333333333333,
        "wilson_95": [
          0.8407442010623228,
          0.973771725856802
        ],
        "brier": 0.18481602010771012,
        "nll": 0.3395356021228635,
        "ece_10bin": 0.15566204786300658
      }
    },
    "tasks": {
      "AGT-1": {
        "n": 36,
        "correct": 35,
        "accuracy": 0.9722222222222222,
        "wilson_95": [
          0.8582996626496918,
          0.9950797197044302
        ],
        "brier": 0.030582898455168046,
        "nll": 0.06655684332752172,
        "ece_10bin": 0.027173144949807094,
        "name": "Which tool to call",
        "category": "AI agents & evals"
      },
      "AGT-2": {
        "n": 30,
        "correct": 23,
        "accuracy": 0.7666666666666667,
        "wilson_95": [
          0.5907132766967574,
          0.8820776078949878
        ],
        "brier": 0.28787776107788116,
        "nll": 0.43420013172936184,
        "ece_10bin": 0.12924168705940248,
        "name": "Did the injection succeed?",
        "category": "AI agents & evals"
      },
      "AGT-3": {
        "n": 30,
        "correct": 16,
        "accuracy": 0.5333333333333333,
        "wilson_95": [
          0.3614201328152505,
          0.6976787277587178
        ],
        "brier": 0.5182158363605527,
        "nll": 0.7539552459843567,
        "ece_10bin": 0.24240172902743018,
        "name": "Did the agent complete the task?",
        "category": "AI agents & evals"
      },
      "AGT-4": {
        "n": 30,
        "correct": 19,
        "accuracy": 0.6333333333333333,
        "wilson_95": [
          0.45513246433604937,
          0.7812629779598231
        ],
        "brier": 0.534163449840086,
        "nll": 0.7650495967034506,
        "ece_10bin": 0.24355113108952844,
        "name": "Is the answer backed by sources?",
        "category": "AI agents & evals"
      },
      "AGT-5": {
        "n": 28,
        "correct": 23,
        "accuracy": 0.8214285714285714,
        "wilson_95": [
          0.644082084514089,
          0.9212161417056989
        ],
        "brier": 0.29330292207352826,
        "nll": 0.45912948431952677,
        "ece_10bin": 0.12212274117129189,
        "name": "Which reply is better?",
        "category": "AI agents & evals"
      },
      "COM-1": {
        "n": 30,
        "correct": 18,
        "accuracy": 0.6,
        "wilson_95": [
          0.42320052498273836,
          0.7540960567391659
        ],
        "brier": 0.5369374114167635,
        "nll": 0.9452828457740636,
        "ece_10bin": 0.25685740212599434,
        "name": "Does this product match the query?",
        "category": "Sales & commerce"
      },
      "COM-2": {
        "n": 30,
        "correct": 30,
        "accuracy": 1.0,
        "wilson_95": [
          0.8864829086095221,
          1.0
        ],
        "brier": 0.0024303444868659504,
        "nll": 0.015614372377922625,
        "ece_10bin": 0.01448677976926165,
        "name": "What type of product is this?",
        "category": "Sales & commerce"
      },
      "COM-3": {
        "n": 30,
        "correct": 30,
        "accuracy": 1.0,
        "wilson_95": [
          0.8864829086095221,
          1.0
        ],
        "brier": 0.018259072059574465,
        "nll": 0.08541180017288426,
        "ece_10bin": 0.07913413047790524,
        "name": "Which industry is this company in?",
        "category": "Sales & commerce"
      },
      "DAT-1": {
        "n": 30,
        "correct": 30,
        "accuracy": 1.0,
        "wilson_95": [
          0.8864829086095221,
          1.0
        ],
        "brier": 0.0020509427148589342,
        "nll": 0.015067299309859799,
        "ece_10bin": 0.014395274718602498,
        "name": "Which SQL answers the question?",
        "category": "Data & analytics"
      },
      "DAT-2": {
        "n": 30,
        "correct": 23,
        "accuracy": 0.7666666666666667,
        "wilson_95": [
          0.5907132766967574,
          0.8820776078949878
        ],
        "brier": 0.3288479447176421,
        "nll": 0.6338153923990979,
        "ece_10bin": 0.12054505447546641,
        "name": "Which cell answers the question?",
        "category": "Data & analytics"
      },
      "DAT-3": {
        "n": 32,
        "correct": 30,
        "accuracy": 0.9375,
        "wilson_95": [
          0.7985253275199642,
          0.9826897968048427
        ],
        "brier": 0.09484936935292382,
        "nll": 0.1403999878767729,
        "ece_10bin": 0.05867942608892919,
        "name": "Does the chart support the claim?",
        "category": "Data & analytics"
      },
      "DAT-4": {
        "n": 34,
        "correct": 27,
        "accuracy": 0.7941176470588235,
        "wilson_95": [
          0.6320128389178601,
          0.8965060397764868
        ],
        "brier": 0.28681020341014696,
        "nll": 0.5363956676317008,
        "ece_10bin": 0.09929345022229585,
        "name": "What does this column hold?",
        "category": "Data & analytics"
      },
      "DOC-1": {
        "n": 30,
        "correct": 27,
        "accuracy": 0.9,
        "wilson_95": [
          0.7437855868473339,
          0.9654007400402839
        ],
        "brier": 0.15403273036885004,
        "nll": 0.2648514215597006,
        "ece_10bin": 0.06072066128253935,
        "name": "Where is this page from?",
        "category": "Documents & meetings"
      },
      "DOC-2": {
        "n": 30,
        "correct": 17,
        "accuracy": 0.5666666666666667,
        "wilson_95": [
          0.3919700954540873,
          0.7262276256938489
        ],
        "brier": 0.6223007371921281,
        "nll": 1.0129624287755297,
        "ece_10bin": 0.25450345476468406,
        "name": "Decision, action item, or neither?",
        "category": "Documents & meetings"
      },
      "DOC-3": {
        "n": 27,
        "correct": 14,
        "accuracy": 0.5185185185185185,
        "wilson_95": [
          0.3398532458621043,
          0.6925704935029092
        ],
        "brier": 0.584346606259613,
        "nll": 1.0694224425229446,
        "ece_10bin": 0.1376357001286966,
        "name": "Which summary is this meeting's?",
        "category": "Documents & meetings"
      },
      "DSN-1": {
        "n": 30,
        "correct": 27,
        "accuracy": 0.9,
        "wilson_95": [
          0.7437855868473339,
          0.9654007400402839
        ],
        "brier": 0.15366041733122054,
        "nll": 0.25909538513696023,
        "ece_10bin": 0.08948132991790768,
        "name": "What does this icon mean?",
        "category": "Design"
      },
      "ENG-1": {
        "n": 30,
        "correct": 27,
        "accuracy": 0.9,
        "wilson_95": [
          0.7437855868473339,
          0.9654007400402839
        ],
        "brier": 0.11132018659144961,
        "nll": 0.20477249019318336,
        "ece_10bin": 0.04837771554787957,
        "name": "Which file must change?",
        "category": "Engineering"
      },
      "ENG-2": {
        "n": 30,
        "correct": 21,
        "accuracy": 0.7,
        "wilson_95": [
          0.5212387898833981,
          0.8333543735604105
        ],
        "brier": 0.4363074322725319,
        "nll": 0.8313417805543007,
        "ece_10bin": 0.13177023231983184,
        "name": "What kind of change is this?",
        "category": "Engineering"
      },
      "ENG-3": {
        "n": 30,
        "correct": 27,
        "accuracy": 0.9,
        "wilson_95": [
          0.7437855868473339,
          0.9654007400402839
        ],
        "brier": 0.13934396279250957,
        "nll": 0.2782563397648229,
        "ece_10bin": 0.09578559895356492,
        "name": "Which weakness class is this?",
        "category": "Engineering"
      },
      "ENG-4": {
        "n": 30,
        "correct": 14,
        "accuracy": 0.4666666666666667,
        "wilson_95": [
          0.30232127224128236,
          0.6385798671847496
        ],
        "brier": 0.6935649909073598,
        "nll": 1.1006871812977823,
        "ece_10bin": 0.29937508205572766,
        "name": "Live secret or placeholder?",
        "category": "Engineering"
      },
      "ENG-5": {
        "n": 30,
        "correct": 30,
        "accuracy": 1.0,
        "wilson_95": [
          0.8864829086095221,
          1.0
        ],
        "brier": 0.0005499232165388203,
        "nll": 0.00765877330338525,
        "ece_10bin": 0.007508711020151826,
        "name": "Which message describes this diff?",
        "category": "Engineering"
      },
      "FIN-1": {
        "n": 30,
        "correct": 23,
        "accuracy": 0.7666666666666667,
        "wilson_95": [
          0.5907132766967574,
          0.8820776078949878
        ],
        "brier": 0.28353288337405524,
        "nll": 0.47289079112931975,
        "ece_10bin": 0.12330033679803212,
        "name": "Which part of the 10-K?",
        "category": "Finance"
      },
      "FIN-2": {
        "n": 30,
        "correct": 30,
        "accuracy": 1.0,
        "wilson_95": [
          0.8864829086095221,
          1.0
        ],
        "brier": 0.022112077835639385,
        "nll": 0.06341257084092414,
        "ece_10bin": 0.054986874262491846,
        "name": "What does this 8-K report?",
        "category": "Finance"
      },
      "FIN-3": {
        "n": 30,
        "correct": 30,
        "accuracy": 1.0,
        "wilson_95": [
          0.8864829086095221,
          1.0
        ],
        "brier": 0.00034189686487335783,
        "nll": 0.008959187003668193,
        "ece_10bin": 0.008840614557266258,
        "name": "Which figure is the total?",
        "category": "Finance"
      },
      "FIN-4": {
        "n": 31,
        "correct": 31,
        "accuracy": 1.0,
        "wilson_95": [
          0.889740999265246,
          0.9999999999999999
        ],
        "brier": 7.602183608546927e-05,
        "nll": 0.005087555346097793,
        "ece_10bin": 0.00506060738717351,
        "name": "What does this number report?",
        "category": "Finance"
      },
      "LEG-1": {
        "n": 33,
        "correct": 27,
        "accuracy": 0.8181818181818182,
        "wilson_95": [
          0.656114329507048,
          0.9138934877430174
        ],
        "brier": 0.2652116652478933,
        "nll": 0.5520196778849797,
        "ece_10bin": 0.08071879907087848,
        "name": "Does the NDA say this?",
        "category": "Legal"
      },
      "LEG-2": {
        "n": 30,
        "correct": 30,
        "accuracy": 1.0,
        "wilson_95": [
          0.8864829086095221,
          1.0
        ],
        "brier": 0.0050507226576597805,
        "nll": 0.03374539651113273,
        "ece_10bin": 0.03135175108909608,
        "name": "What kind of clause is this?",
        "category": "Legal"
      },
      "LEG-3": {
        "n": 33,
        "correct": 32,
        "accuracy": 0.9696969696969697,
        "wilson_95": [
          0.8468093832683193,
          0.9946307279103482
        ],
        "brier": 0.07690360038807299,
        "nll": 0.2568270228551126,
        "ece_10bin": 0.12009629245960353,
        "name": "Is this term unfair, and how?",
        "category": "Legal"
      },
      "LEG-4": {
        "n": 30,
        "correct": 24,
        "accuracy": 0.8,
        "wilson_95": [
          0.6269395261897256,
          0.9049502189759876
        ],
        "brier": 0.25688539523190546,
        "nll": 0.41223793011719534,
        "ece_10bin": 0.12718977530797326,
        "name": "Does the contract cover this?",
        "category": "Legal"
      },
      "PRD-1": {
        "n": 30,
        "correct": 13,
        "accuracy": 0.43333333333333335,
        "wilson_95": [
          0.2737723743061511,
          0.6080299045459127
        ],
        "brier": 0.58530396065978,
        "nll": 0.7964713375792108,
        "ece_10bin": 0.24999946554501848,
        "name": "Which headline won?",
        "category": "Product management"
      },
      "PRD-2": {
        "n": 30,
        "correct": 23,
        "accuracy": 0.7666666666666667,
        "wilson_95": [
          0.5907132766967574,
          0.8820776078949878
        ],
        "brier": 0.3574364825210092,
        "nll": 0.6859150485701193,
        "ece_10bin": 0.10831168492635095,
        "name": "Major, minor or patch?",
        "category": "Product management"
      },
      "SAF-1": {
        "n": 30,
        "correct": 27,
        "accuracy": 0.9,
        "wilson_95": [
          0.7437855868473339,
          0.9654007400402839
        ],
        "brier": 0.238962201442865,
        "nll": 0.415863060877413,
        "ece_10bin": 0.14678021669387817,
        "name": "Attack on the assistant",
        "category": "Trust & safety"
      },
      "SAF-2": {
        "n": 30,
        "correct": 29,
        "accuracy": 0.9666666666666667,
        "wilson_95": [
          0.8332924860304112,
          0.9940915620051425
        ],
        "brier": 0.13066983877255522,
        "nll": 0.263208143368314,
        "ece_10bin": 0.1850230137507121,
        "name": "Toxic comment or not",
        "category": "Trust & safety"
      },
      "SUP-1": {
        "n": 35,
        "correct": 33,
        "accuracy": 0.9428571428571428,
        "wilson_95": [
          0.8139261926062996,
          0.984187170385956
        ],
        "brier": 0.0961051187719877,
        "nll": 0.20748777988993264,
        "ece_10bin": 0.09738531708717343,
        "name": "Which team owns this complaint?",
        "category": "Customer support"
      },
      "SUP-2": {
        "n": 32,
        "correct": 29,
        "accuracy": 0.90625,
        "wilson_95": [
          0.7578150066422116,
          0.9675990373736805
        ],
        "brier": 0.16279189655214363,
        "nll": 0.33798933595371733,
        "ece_10bin": 0.09585115592926738,
        "name": "What is the complaint about?",
        "category": "Customer support"
      }
    },
    "notes": [
      "Our run, not an independently accepted leaderboard entry.",
      "9 requests exceeded 4,096 tokens and used an 8,192-token rendering limit; no evidence was truncated.",
      "Strict 4K: 884/1,071 = 82.54%. The 0.8B comparison allowed 8K; strict 2K gives 70.60% on text rows.",
      "Public source datasets may overlap pretraining or mixture inputs. This is not a contamination-free estimate."
    ]
  },
  "typed": {
    "cases": 400,
    "decisions": 2000,
    "revision": "7189f46b02248bd2569df7325c9df8e3b24e90a7",
    "models": {
      "0.8b": {
        "raw": {
          "count": 2000,
          "soft_ce": 0.8982348703026427,
          "kl": 0.13087631467831382,
          "brier": 0.03733508375955404,
          "hard_ece": 0.02813636220158644,
          "argmax_agreement": 0.7055,
          "gold_label_accuracy": 0.7045,
          "gold_label_correct": 1409,
          "full_vector_brier": 0.07467016751910809,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.9718636377984136,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 24,
              "confidence": 0.31520772849577633,
              "reference_agreement": 0.3355554526389121
            },
            {
              "count": 171,
              "confidence": 0.3746159687360105,
              "reference_agreement": 0.38106237192013953
            },
            {
              "count": 368,
              "confidence": 0.43486926682677546,
              "reference_agreement": 0.41427729467341934
            },
            {
              "count": 330,
              "confidence": 0.5005758970577506,
              "reference_agreement": 0.4898888885960213
            },
            {
              "count": 259,
              "confidence": 0.5661637493278858,
              "reference_agreement": 0.5401029315058599
            },
            {
              "count": 208,
              "confidence": 0.6334207148716826,
              "reference_agreement": 0.5813301291346679
            },
            {
              "count": 176,
              "confidence": 0.6976026942523854,
              "reference_agreement": 0.6650378634451258
            },
            {
              "count": 117,
              "confidence": 0.765488790351476,
              "reference_agreement": 0.6715926028234322
            },
            {
              "count": 95,
              "confidence": 0.8268994900267613,
              "reference_agreement": 0.7595845223159148
            },
            {
              "count": 90,
              "confidence": 0.8962419341009388,
              "reference_agreement": 0.9119985310000607
            },
            {
              "count": 162,
              "confidence": 0.9675863551773324,
              "reference_agreement": 0.9661111104013173
            }
          ]
        },
        "calibrated": {
          "count": 2000,
          "soft_ce": 0.8996057207086053,
          "kl": 0.13224716508427645,
          "brier": 0.0371886634007063,
          "hard_ece": 0.04532655734322799,
          "argmax_agreement": 0.7055,
          "gold_label_accuracy": 0.7045,
          "gold_label_correct": 1409,
          "full_vector_brier": 0.0743773268014126,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.954673442656772,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.26431766017833624,
              "reference_agreement": 0.6433336433336433
            },
            {
              "count": 40,
              "confidence": 0.3177876741133899,
              "reference_agreement": 0.3488332323333685
            },
            {
              "count": 214,
              "confidence": 0.3738574970766125,
              "reference_agreement": 0.39616281869943265
            },
            {
              "count": 379,
              "confidence": 0.4335478747419467,
              "reference_agreement": 0.421979419893986
            },
            {
              "count": 296,
              "confidence": 0.5011488512826869,
              "reference_agreement": 0.5069932456870009
            },
            {
              "count": 266,
              "confidence": 0.5645507022164814,
              "reference_agreement": 0.5461152577318915
            },
            {
              "count": 223,
              "confidence": 0.633536195077047,
              "reference_agreement": 0.5981748920314267
            },
            {
              "count": 156,
              "confidence": 0.6977244202652548,
              "reference_agreement": 0.6617542448932456
            },
            {
              "count": 130,
              "confidence": 0.7643242189776432,
              "reference_agreement": 0.6995359706949515
            },
            {
              "count": 92,
              "confidence": 0.8331337292292074,
              "reference_agreement": 0.8546781770653425
            },
            {
              "count": 68,
              "confidence": 0.9005759800850643,
              "reference_agreement": 0.9235195591667292
            },
            {
              "count": 135,
              "confidence": 0.9612556801725451,
              "reference_agreement": 0.9694123619383392
            }
          ]
        },
        "categories": {
          "agent_trace_observability": {
            "count": 500,
            "soft_ce": 0.97776378147771,
            "kl": 0.09929350853248653,
            "brier": 0.029319985421551766,
            "hard_ece": null,
            "argmax_agreement": 0.666,
            "gold_label_accuracy": 0.67,
            "gold_label_correct": 335,
            "full_vector_brier": 0.05863997084310353,
            "hard_reliability": [],
            "soft_reliability": [
              {
                "count": 1,
                "confidence": 0.3271961322725437,
                "reference_agreement": 0.333333
              },
              {
                "count": 35,
                "confidence": 0.3798205422614706,
                "reference_agreement": 0.40077146666672475
              },
              {
                "count": 132,
                "confidence": 0.43641602107268646,
                "reference_agreement": 0.4047222168687381
              },
              {
                "count": 117,
                "confidence": 0.5006302500044424,
                "reference_agreement": 0.47911679880348107
              },
              {
                "count": 76,
                "confidence": 0.5690619634091041,
                "reference_agreement": 0.5512718920614565
              },
              {
                "count": 45,
                "confidence": 0.6338679776529896,
                "reference_agreement": 0.6111852223703935
              },
              {
                "count": 25,
                "confidence": 0.6938113959359317,
                "reference_agreement": 0.6235998789334686
              },
              {
                "count": 14,
                "confidence": 0.7711383973508009,
                "reference_agreement": 0.6864285047620653
              },
              {
                "count": 32,
                "confidence": 0.823789261655095,
                "reference_agreement": 0.7322917248959386
              },
              {
                "count": 23,
                "confidence": 0.8940903067075735,
                "reference_agreement": 0.8727536086956522
              }
            ]
          },
          "customer_service": {
            "count": 500,
            "soft_ce": 0.9356140291801783,
            "kl": 0.19845939117069544,
            "brier": 0.054488558914342955,
            "hard_ece": 0.023186746014468573,
            "argmax_agreement": 0.694,
            "gold_label_accuracy": 0.694,
            "gold_label_correct": 347,
            "full_vector_brier": 0.10897711782868591,
            "hard_reliability": [
              {
                "count": 1,
                "confidence": 0.9768132539855314,
                "reference_agreement": 1.0
              }
            ],
            "soft_reliability": [
              {
                "count": 10,
                "confidence": 0.31477128374516633,
                "reference_agreement": 0.4056665940000813
              },
              {
                "count": 39,
                "confidence": 0.3723859508926986,
                "reference_agreement": 0.45632472794882123
              },
              {
                "count": 73,
                "confidence": 0.4380748853548461,
                "reference_agreement": 0.3939339658448251
              },
              {
                "count": 81,
                "confidence": 0.5011240542197034,
                "reference_agreement": 0.5185185261729083
              },
              {
                "count": 80,
                "confidence": 0.5649588987676203,
                "reference_agreement": 0.5116249845417362
              },
              {
                "count": 69,
                "confidence": 0.6279948891492104,
                "reference_agreement": 0.5734783022222739
              },
              {
                "count": 46,
                "confidence": 0.6947774859534929,
                "reference_agreement": 0.7216593691232294
              },
              {
                "count": 33,
                "confidence": 0.7624282639003352,
                "reference_agreement": 0.6597979875758164
              },
              {
                "count": 22,
                "confidence": 0.8310417874707298,
                "reference_agreement": 0.8232055630304752
              },
              {
                "count": 21,
                "confidence": 0.8956622165551549,
                "reference_agreement": 0.913333472222457
              },
              {
                "count": 26,
                "confidence": 0.9583538062949253,
                "reference_agreement": 0.955179537115826
              }
            ]
          },
          "invoice_processing": {
            "count": 500,
            "soft_ce": 0.6654727615892404,
            "kl": 0.09052528592434582,
            "brier": 0.02417431701033401,
            "hard_ece": 0.033085978388704196,
            "argmax_agreement": 0.84,
            "gold_label_accuracy": 0.834,
            "gold_label_correct": 417,
            "full_vector_brier": 0.04834863402066802,
            "hard_reliability": [
              {
                "count": 1,
                "confidence": 0.9669140216112958,
                "reference_agreement": 1.0
              }
            ],
            "soft_reliability": [
              {
                "count": 2,
                "confidence": 0.3278949544267834,
                "reference_agreement": 0.5716665
              },
              {
                "count": 37,
                "confidence": 0.36518717835723197,
                "reference_agreement": 0.3640449954955256
              },
              {
                "count": 44,
                "confidence": 0.43272176571282106,
                "reference_agreement": 0.4239439440863979
              },
              {
                "count": 47,
                "confidence": 0.4990750370635605,
                "reference_agreement": 0.4812765950355347
              },
              {
                "count": 40,
                "confidence": 0.5636676327793726,
                "reference_agreement": 0.5589167148334007
              },
              {
                "count": 44,
                "confidence": 0.6359292707198466,
                "reference_agreement": 0.618181796969778
              },
              {
                "count": 53,
                "confidence": 0.6993117889365471,
                "reference_agreement": 0.6881194986793073
              },
              {
                "count": 34,
                "confidence": 0.7613192737634586,
                "reference_agreement": 0.7343039374609032
              },
              {
                "count": 21,
                "confidence": 0.8299096395763272,
                "reference_agreement": 0.8130160017462573
              },
              {
                "count": 43,
                "confidence": 0.8979274834748605,
                "reference_agreement": 0.9520123691472993
              },
              {
                "count": 135,
                "confidence": 0.9695963375504574,
                "reference_agreement": 0.9691999920000142
              }
            ]
          },
          "security_incidents": {
            "count": 500,
            "soft_ce": 1.0140889089634417,
            "kl": 0.13522707308572743,
            "brier": 0.041357473691987466,
            "hard_ece": null,
            "argmax_agreement": 0.622,
            "gold_label_accuracy": 0.62,
            "gold_label_correct": 310,
            "full_vector_brier": 0.08271494738397493,
            "hard_reliability": [],
            "soft_reliability": [
              {
                "count": 11,
                "confidence": 0.31220787321098703,
                "reference_agreement": 0.2290908112120979
              },
              {
                "count": 60,
                "confidence": 0.37884389984455846,
                "reference_agreement": 0.33113891736116685
              },
              {
                "count": 119,
                "confidence": 0.4319811015831166,
                "reference_agreement": 0.43378150224093404
              },
              {
                "count": 85,
                "confidence": 0.5008086075853485,
                "reference_agreement": 0.48219608458828284
              },
              {
                "count": 63,
                "confidence": 0.5657822959404127,
                "reference_agreement": 0.550846477883684
              },
              {
                "count": 50,
                "confidence": 0.6382982887191337,
                "reference_agreement": 0.5328665986667214
              },
              {
                "count": 52,
                "confidence": 0.7001826946639205,
                "reference_agreement": 0.6113462035256458
              },
              {
                "count": 36,
                "confidence": 0.7700350803206342,
                "reference_agreement": 0.6174074444444444
              },
              {
                "count": 20,
                "confidence": 0.8241586712060176,
                "reference_agreement": 0.6771668
              },
              {
                "count": 3,
                "confidence": 0.8926362259110124,
                "reference_agreement": 0.63
              },
              {
                "count": 1,
                "confidence": 0.9362850057480346,
                "reference_agreement": 0.833333
              }
            ]
          }
        }
      },
      "2b": {
        "raw": {
          "count": 2000,
          "soft_ce": 0.8905161947908924,
          "kl": 0.1231576391665636,
          "brier": 0.03404154459336115,
          "hard_ece": 0.011185660540766307,
          "argmax_agreement": 0.7285,
          "gold_label_accuracy": 0.729,
          "gold_label_correct": 1458,
          "full_vector_brier": 0.0680830891867223,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.9888143394592337,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 13,
              "confidence": 0.3088639829497208,
              "reference_agreement": 0.4135898133333467
            },
            {
              "count": 134,
              "confidence": 0.3762417147843914,
              "reference_agreement": 0.39605719307219406
            },
            {
              "count": 300,
              "confidence": 0.4327188173876857,
              "reference_agreement": 0.4052444353656112
            },
            {
              "count": 345,
              "confidence": 0.5002324190281352,
              "reference_agreement": 0.4679947137778451
            },
            {
              "count": 249,
              "confidence": 0.5642030266585134,
              "reference_agreement": 0.5232792480849401
            },
            {
              "count": 225,
              "confidence": 0.6320250685648934,
              "reference_agreement": 0.5861629515852452
            },
            {
              "count": 199,
              "confidence": 0.6975184724935769,
              "reference_agreement": 0.6625309976885212
            },
            {
              "count": 138,
              "confidence": 0.7628361718778747,
              "reference_agreement": 0.6679952073672175
            },
            {
              "count": 100,
              "confidence": 0.8324806507886848,
              "reference_agreement": 0.7645266405667441
            },
            {
              "count": 84,
              "confidence": 0.9007336421886782,
              "reference_agreement": 0.8620991815476933
            },
            {
              "count": 213,
              "confidence": 0.979045139070839,
              "reference_agreement": 0.9552772714476669
            }
          ]
        },
        "calibrated": {
          "count": 2000,
          "soft_ce": 0.8847112215901843,
          "kl": 0.11735266596585546,
          "brier": 0.032662104520750475,
          "hard_ece": 0.03043870521542713,
          "argmax_agreement": 0.7285,
          "gold_label_accuracy": 0.729,
          "gold_label_correct": 1458,
          "full_vector_brier": 0.06532420904150095,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.9695612947845729,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 34,
              "confidence": 0.31414948972313944,
              "reference_agreement": 0.3915686254902435
            },
            {
              "count": 235,
              "confidence": 0.3763834264654181,
              "reference_agreement": 0.4107432504596411
            },
            {
              "count": 339,
              "confidence": 0.4350258557326406,
              "reference_agreement": 0.41825617827931044
            },
            {
              "count": 301,
              "confidence": 0.49879789075850456,
              "reference_agreement": 0.49976698815001347
            },
            {
              "count": 291,
              "confidence": 0.5658351793600359,
              "reference_agreement": 0.552885434570514
            },
            {
              "count": 237,
              "confidence": 0.6329843950656429,
              "reference_agreement": 0.642983131504989
            },
            {
              "count": 154,
              "confidence": 0.6981279578061982,
              "reference_agreement": 0.6780952719481249
            },
            {
              "count": 90,
              "confidence": 0.7653030888453866,
              "reference_agreement": 0.7386222120370977
            },
            {
              "count": 85,
              "confidence": 0.8317234638858025,
              "reference_agreement": 0.8519176247451653
            },
            {
              "count": 70,
              "confidence": 0.8960409155426925,
              "reference_agreement": 0.8906246001906306
            },
            {
              "count": 164,
              "confidence": 0.9699625521385267,
              "reference_agreement": 0.969758151026486
            }
          ]
        },
        "categories": {
          "agent_trace_observability": {
            "count": 500,
            "soft_ce": 0.974477817894443,
            "kl": 0.09600754494921938,
            "brier": 0.026986082938624777,
            "hard_ece": null,
            "argmax_agreement": 0.714,
            "gold_label_accuracy": 0.72,
            "gold_label_correct": 360,
            "full_vector_brier": 0.05397216587724955,
            "hard_reliability": [],
            "soft_reliability": [
              {
                "count": 37,
                "confidence": 0.3864067506953036,
                "reference_agreement": 0.39423422315320117
              },
              {
                "count": 75,
                "confidence": 0.43601945891983956,
                "reference_agreement": 0.39324889977783856
              },
              {
                "count": 124,
                "confidence": 0.5000985711197174,
                "reference_agreement": 0.48120426973125646
              },
              {
                "count": 76,
                "confidence": 0.5609762026944506,
                "reference_agreement": 0.5082412082018144
              },
              {
                "count": 64,
                "confidence": 0.6334990515507852,
                "reference_agreement": 0.6079686979167307
              },
              {
                "count": 38,
                "confidence": 0.6935412247701108,
                "reference_agreement": 0.6965789711404086
              },
              {
                "count": 35,
                "confidence": 0.7621672025278979,
                "reference_agreement": 0.6395238594286613
              },
              {
                "count": 30,
                "confidence": 0.8402241397042596,
                "reference_agreement": 0.8121111438889721
              },
              {
                "count": 21,
                "confidence": 0.903559172955109,
                "reference_agreement": 0.8344443573016649
              }
            ]
          },
          "customer_service": {
            "count": 500,
            "soft_ce": 0.8991096640077225,
            "kl": 0.16195502599823958,
            "brier": 0.04393123899656973,
            "hard_ece": 0.015161256578287818,
            "argmax_agreement": 0.708,
            "gold_label_accuracy": 0.712,
            "gold_label_correct": 356,
            "full_vector_brier": 0.08786247799313945,
            "hard_reliability": [
              {
                "count": 1,
                "confidence": 0.9848387434217122,
                "reference_agreement": 1.0
              }
            ],
            "soft_reliability": [
              {
                "count": 18,
                "confidence": 0.36205531573991584,
                "reference_agreement": 0.3816665590741483
              },
              {
                "count": 56,
                "confidence": 0.43945292772936134,
                "reference_agreement": 0.36279757130956736
              },
              {
                "count": 96,
                "confidence": 0.5018575257753607,
                "reference_agreement": 0.439102544965351
              },
              {
                "count": 75,
                "confidence": 0.565374144364121,
                "reference_agreement": 0.5291555608889729
              },
              {
                "count": 65,
                "confidence": 0.6331380635754497,
                "reference_agreement": 0.5745128662564378
              },
              {
                "count": 57,
                "confidence": 0.7025688001135957,
                "reference_agreement": 0.6523976915790378
              },
              {
                "count": 36,
                "confidence": 0.7616488467069966,
                "reference_agreement": 0.7475926394445652
              },
              {
                "count": 30,
                "confidence": 0.8274620872544971,
                "reference_agreement": 0.7837665297667034
              },
              {
                "count": 18,
                "confidence": 0.9005460601309712,
                "reference_agreement": 0.8996295020371518
              },
              {
                "count": 49,
                "confidence": 0.9631678977836736,
                "reference_agreement": 0.933099202007166
              }
            ]
          },
          "invoice_processing": {
            "count": 500,
            "soft_ce": 0.6740672163969859,
            "kl": 0.0991197407320913,
            "brier": 0.026686130174018666,
            "hard_ece": 0.007210064503244795,
            "argmax_agreement": 0.822,
            "gold_label_accuracy": 0.816,
            "gold_label_correct": 408,
            "full_vector_brier": 0.05337226034803733,
            "hard_reliability": [
              {
                "count": 1,
                "confidence": 0.9927899354967552,
                "reference_agreement": 1.0
              }
            ],
            "soft_reliability": [
              {
                "count": 2,
                "confidence": 0.31011293202790324,
                "reference_agreement": 0.5800000000000001
              },
              {
                "count": 20,
                "confidence": 0.3785522543445289,
                "reference_agreement": 0.39916663983331147
              },
              {
                "count": 67,
                "confidence": 0.43175210506110073,
                "reference_agreement": 0.410741255815974
              },
              {
                "count": 62,
                "confidence": 0.49531540835569626,
                "reference_agreement": 0.5438978752689022
              },
              {
                "count": 39,
                "confidence": 0.5608276035545503,
                "reference_agreement": 0.5637658267470493
              },
              {
                "count": 26,
                "confidence": 0.6360426792409636,
                "reference_agreement": 0.6294871435898423
              },
              {
                "count": 52,
                "confidence": 0.6957905578891667,
                "reference_agreement": 0.6729551100641851
              },
              {
                "count": 28,
                "confidence": 0.7608212688374972,
                "reference_agreement": 0.680595268452447
              },
              {
                "count": 15,
                "confidence": 0.8270372877314738,
                "reference_agreement": 0.7584221898002764
              },
              {
                "count": 28,
                "confidence": 0.9031127850082596,
                "reference_agreement": 0.9259405253572337
              },
              {
                "count": 161,
                "confidence": 0.9844285748099726,
                "reference_agreement": 0.9665767945341734
              }
            ]
          },
          "security_incidents": {
            "count": 500,
            "soft_ce": 1.0144100808644185,
            "kl": 0.13554824498670415,
            "brier": 0.038562726264231434,
            "hard_ece": null,
            "argmax_agreement": 0.67,
            "gold_label_accuracy": 0.668,
            "gold_label_correct": 334,
            "full_vector_brier": 0.07712545252846287,
            "hard_reliability": [],
            "soft_reliability": [
              {
                "count": 11,
                "confidence": 0.3086369012991421,
                "reference_agreement": 0.38333341575759156
              },
              {
                "count": 59,
                "confidence": 0.37341185144361266,
                "reference_agreement": 0.4005367246610959
              },
              {
                "count": 102,
                "confidence": 0.427229713778233,
                "reference_agreement": 0.43375818620921025
              },
              {
                "count": 63,
                "confidence": 0.5028584754503794,
                "reference_agreement": 0.411322765449764
              },
              {
                "count": 59,
                "confidence": 0.5691021163941534,
                "reference_agreement": 0.5084180786441408
              },
              {
                "count": 70,
                "confidence": 0.6281516762168783,
                "reference_agreement": 0.5609523628572147
              },
              {
                "count": 52,
                "confidence": 0.6966169013124227,
                "reference_agreement": 0.6383333364103352
              },
              {
                "count": 39,
                "confidence": 0.7659791185068839,
                "reference_agreement": 0.6110256666666667
              },
              {
                "count": 25,
                "confidence": 0.832476758165347,
                "reference_agreement": 0.68800004
              },
              {
                "count": 17,
                "confidence": 0.8935233087766431,
                "reference_agreement": 0.7513725882352942
              },
              {
                "count": 3,
                "confidence": 0.9494623620943644,
                "reference_agreement": 0.7111113333333333
              }
            ]
          }
        }
      },
      "probe": {
        "raw": {
          "count": 2000,
          "soft_ce": 0.8649300758461522,
          "kl": 0.09757152022182336,
          "brier": 0.027229588161560105,
          "hard_ece": 0.019624246396639622,
          "argmax_agreement": 0.7715,
          "gold_label_accuracy": 0.7755,
          "gold_label_correct": 1551,
          "full_vector_brier": 0.05445917632312021,
          "hard_reliability": [
            {
              "count": 2,
              "confidence": 0.9803757536033604,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 13,
              "confidence": 0.3127499630264448,
              "reference_agreement": 0.31333328461543847
            },
            {
              "count": 157,
              "confidence": 0.3744425717484434,
              "reference_agreement": 0.3834288762633071
            },
            {
              "count": 333,
              "confidence": 0.43493294774184765,
              "reference_agreement": 0.4414964809860609
            },
            {
              "count": 320,
              "confidence": 0.5009866781404044,
              "reference_agreement": 0.5131652101140275
            },
            {
              "count": 300,
              "confidence": 0.5649235944826095,
              "reference_agreement": 0.5680027983667341
            },
            {
              "count": 256,
              "confidence": 0.6316940823487122,
              "reference_agreement": 0.6060663548841791
            },
            {
              "count": 150,
              "confidence": 0.6953321117227326,
              "reference_agreement": 0.6761333542667338
            },
            {
              "count": 106,
              "confidence": 0.7639093823852469,
              "reference_agreement": 0.7366667328303035
            },
            {
              "count": 111,
              "confidence": 0.8325650671691083,
              "reference_agreement": 0.8248619123724371
            },
            {
              "count": 63,
              "confidence": 0.8983742523423454,
              "reference_agreement": 0.8693596968730851
            },
            {
              "count": 191,
              "confidence": 0.9699667909472699,
              "reference_agreement": 0.9642097165358748
            }
          ]
        },
        "calibrated": {
          "count": 2000,
          "soft_ce": 0.8850909091691178,
          "kl": 0.11773235354478902,
          "brier": 0.030601322688672217,
          "hard_ece": 0.08700802456622558,
          "argmax_agreement": 0.7715,
          "gold_label_accuracy": 0.7755,
          "gold_label_correct": 1551,
          "full_vector_brier": 0.06120264537734443,
          "hard_reliability": [
            {
              "count": 1,
              "confidence": 0.8629581297954875,
              "reference_agreement": 1.0
            },
            {
              "count": 1,
              "confidence": 0.9630258210720614,
              "reference_agreement": 1.0
            }
          ],
          "soft_reliability": [
            {
              "count": 1,
              "confidence": 0.2645863327471326,
              "reference_agreement": 0.2666667333332667
            },
            {
              "count": 67,
              "confidence": 0.31716427158631993,
              "reference_agreement": 0.349875621442791
            },
            {
              "count": 360,
              "confidence": 0.36751161971689855,
              "reference_agreement": 0.43418053666211054
            },
            {
              "count": 379,
              "confidence": 0.4331792205734263,
              "reference_agreement": 0.4932366997005272
            },
            {
              "count": 297,
              "confidence": 0.5005267046059604,
              "reference_agreement": 0.5653883302020873
            },
            {
              "count": 307,
              "confidence": 0.5660711391021781,
              "reference_agreement": 0.6197382883616286
            },
            {
              "count": 151,
              "confidence": 0.6317209880014436,
              "reference_agreement": 0.6816335782340621
            },
            {
              "count": 114,
              "confidence": 0.7003484367913859,
              "reference_agreement": 0.7568129062281621
            },
            {
              "count": 115,
              "confidence": 0.7700242192396172,
              "reference_agreement": 0.8478509065478818
            },
            {
              "count": 52,
              "confidence": 0.8247261297745008,
              "reference_agreement": 0.907055111474529
            },
            {
              "count": 56,
              "confidence": 0.9074347548715843,
              "reference_agreement": 0.9628810512203716
            },
            {
              "count": 101,
              "confidence": 0.9525550691253913,
              "reference_agreement": 0.9716072178217822
            }
          ]
        },
        "categories": {
          "agent_trace_observability": {
            "count": 500,
            "soft_ce": 0.9647398273718357,
            "kl": 0.08626955442661217,
            "brier": 0.024895023440682007,
            "hard_ece": null,
            "argmax_agreement": 0.744,
            "gold_label_accuracy": 0.756,
            "gold_label_correct": 378,
            "full_vector_brier": 0.049790046881364014,
            "hard_reliability": [],
            "soft_reliability": [
              {
                "count": 2,
                "confidence": 0.32688700513155255,
                "reference_agreement": 0.39166650000000003
              },
              {
                "count": 55,
                "confidence": 0.3758267244645977,
                "reference_agreement": 0.4091515044848785
              },
              {
                "count": 111,
                "confidence": 0.43061364082616854,
                "reference_agreement": 0.439579575525598
              },
              {
                "count": 93,
                "confidence": 0.501333097556737,
                "reference_agreement": 0.48465232688178134
              },
              {
                "count": 96,
                "confidence": 0.5669055504229205,
                "reference_agreement": 0.5817013908681125
              },
              {
                "count": 62,
                "confidence": 0.6296184334075055,
                "reference_agreement": 0.6035483822043773
              },
              {
                "count": 20,
                "confidence": 0.6922020486426954,
                "reference_agreement": 0.6636665595001039
              },
              {
                "count": 15,
                "confidence": 0.7697122289822431,
                "reference_agreement": 0.7875555780001677
              },
              {
                "count": 32,
                "confidence": 0.835110513279655,
                "reference_agreement": 0.825312586250055
              },
              {
                "count": 14,
                "confidence": 0.8842376667580375,
                "reference_agreement": 0.8028569121429451
              }
            ]
          },
          "customer_service": {
            "count": 500,
            "soft_ce": 0.8555058069044625,
            "kl": 0.11835116889497967,
            "brier": 0.03136507331686432,
            "hard_ece": 0.024870610760817313,
            "argmax_agreement": 0.786,
            "gold_label_accuracy": 0.79,
            "gold_label_correct": 395,
            "full_vector_brier": 0.06273014663372864,
            "hard_reliability": [
              {
                "count": 1,
                "confidence": 0.9751293892391827,
                "reference_agreement": 1.0
              }
            ],
            "soft_reliability": [
              {
                "count": 5,
                "confidence": 0.3161479906623053,
                "reference_agreement": 0.33999994000014
              },
              {
                "count": 17,
                "confidence": 0.3762959661567679,
                "reference_agreement": 0.31019611764709376
              },
              {
                "count": 61,
                "confidence": 0.43738446834384653,
                "reference_agreement": 0.4459015715301556
              },
              {
                "count": 86,
                "confidence": 0.49882211569554796,
                "reference_agreement": 0.5238371818217792
              },
              {
                "count": 81,
                "confidence": 0.5635199244406663,
                "reference_agreement": 0.5552737504527587
              },
              {
                "count": 62,
                "confidence": 0.6310117585739786,
                "reference_agreement": 0.5970429596774731
              },
              {
                "count": 47,
                "confidence": 0.6948558234546154,
                "reference_agreement": 0.6841135739007279
              },
              {
                "count": 40,
                "confidence": 0.7629425713006321,
                "reference_agreement": 0.7810834319168298
              },
              {
                "count": 34,
                "confidence": 0.8289102595584726,
                "reference_agreement": 0.8093137257844144
              },
              {
                "count": 28,
                "confidence": 0.9087207513543797,
                "reference_agreement": 0.8935593955834087
              },
              {
                "count": 39,
                "confidence": 0.9585044783102166,
                "reference_agreement": 0.9474835368807735
              }
            ]
          },
          "invoice_processing": {
            "count": 500,
            "soft_ce": 0.659989316345021,
            "kl": 0.0850418406801265,
            "brier": 0.023802400826455464,
            "hard_ece": 0.01437788203246182,
            "argmax_agreement": 0.852,
            "gold_label_accuracy": 0.854,
            "gold_label_correct": 427,
            "full_vector_brier": 0.04760480165291093,
            "hard_reliability": [
              {
                "count": 1,
                "confidence": 0.9856221179675382,
                "reference_agreement": 1.0
              }
            ],
            "soft_reliability": [
              {
                "count": 2,
                "confidence": 0.30070721485913454,
                "reference_agreement": 0.12000000000000001
              },
              {
                "count": 42,
                "confidence": 0.37320195967767156,
                "reference_agreement": 0.33964284865080885
              },
              {
                "count": 42,
                "confidence": 0.4397646834028689,
                "reference_agreement": 0.4361904914286279
              },
              {
                "count": 59,
                "confidence": 0.5028580640840516,
                "reference_agreement": 0.5594949624260568
              },
              {
                "count": 47,
                "confidence": 0.5603372734664648,
                "reference_agreement": 0.5872411664539838
              },
              {
                "count": 70,
                "confidence": 0.6344607099008017,
                "reference_agreement": 0.6737094276714951
              },
              {
                "count": 33,
                "confidence": 0.693902187889614,
                "reference_agreement": 0.6925252785859709
              },
              {
                "count": 15,
                "confidence": 0.7726704353767797,
                "reference_agreement": 0.7391112000001823
              },
              {
                "count": 22,
                "confidence": 0.8372016661533274,
                "reference_agreement": 0.9169545834849395
              },
              {
                "count": 17,
                "confidence": 0.89763014763758,
                "reference_agreement": 0.9180392386275111
              },
              {
                "count": 151,
                "confidence": 0.9731341688338102,
                "reference_agreement": 0.9701028537748471
              }
            ]
          },
          "security_incidents": {
            "count": 500,
            "soft_ce": 0.9794853527632895,
            "kl": 0.10062351688557511,
            "brier": 0.028855855062238627,
            "hard_ece": null,
            "argmax_agreement": 0.704,
            "gold_label_accuracy": 0.702,
            "gold_label_correct": 351,
            "full_vector_brier": 0.057711710124477254,
            "hard_reliability": [],
            "soft_reliability": [
              {
                "count": 4,
                "confidence": 0.3074552815127205,
                "reference_agreement": 0.3375
              },
              {
                "count": 43,
                "confidence": 0.3731511671587323,
                "reference_agreement": 0.4222480740310775
              },
              {
                "count": 119,
                "confidence": 0.43599990079357503,
                "reference_agreement": 0.4428991494258409
              },
              {
                "count": 82,
                "confidence": 0.5015174536753257,
                "reference_agreement": 0.5009756148374728
              },
              {
                "count": 76,
                "confidence": 0.5667523654153249,
                "reference_agreement": 0.5523683865351414
              },
              {
                "count": 62,
                "confidence": 0.6313284433122933,
                "reference_agreement": 0.5412365115592014
              },
              {
                "count": 50,
                "confidence": 0.6979755976566357,
                "reference_agreement": 0.662799995666735
              },
              {
                "count": 36,
                "confidence": 0.7589153254284873,
                "reference_agreement": 0.6650926314814927
              },
              {
                "count": 23,
                "confidence": 0.8299913282809037,
                "reference_agreement": 0.7591305217391304
              },
              {
                "count": 4,
                "confidence": 0.8785892537984352,
                "reference_agreement": 0.7258335
              },
              {
                "count": 1,
                "confidence": 0.93872292292476,
                "reference_agreement": 0.726667
              }
            ]
          }
        }
      }
    },
    "notes": [
      "400 cases, five questions each. Accuracy uses stored gold labels, not a rounded distribution's first argmax.",
      "Fitted evaluation: the 2B mixture contains 554 official training cases.",
      "No official test state matched mixture training, validation or calibration by normalized exact-state checks.",
      "172 official test cases also appear as partial-question records in the mixture test split; those evaluations are not independent.",
      "Paraphrase overlap and base-model pretraining exposure were not assessed."
    ]
  },
  "sysone": {
    "decisions": 1240,
    "revision": "93afcfe1bd450dc052b21768e2b95b20316ebb68",
    "models": {
      "0.8b": {
        "raw": 0.7685483870967742,
        "calibrated": 0.7685483870967742,
        "suites": {
          "triage": {
            "accuracy": 0.9270833333333334,
            "n": 192
          },
          "guardrails": {
            "accuracy": 0.8125,
            "n": 96
          },
          "moderation": {
            "accuracy": 0.9166666666666666,
            "n": 144
          },
          "agnews": {
            "accuracy": 0.8125,
            "n": 160
          },
          "emotion": {
            "accuracy": 0.7239583333333334,
            "n": 192
          },
          "banking77_12": {
            "accuracy": 0.8020833333333334,
            "n": 96
          },
          "mnli": {
            "accuracy": 0.5416666666666666,
            "n": 120
          },
          "sst5": {
            "accuracy": 0.3333333333333333,
            "n": 120
          },
          "multilingual_intent": {
            "accuracy": 0.95,
            "n": 120
          }
        }
      },
      "2b": {
        "raw": 0.8193548387096774,
        "calibrated": 0.8169354838709677,
        "suites": {
          "triage": {
            "accuracy": 0.953125,
            "n": 192
          },
          "guardrails": {
            "accuracy": 0.8125,
            "n": 96
          },
          "moderation": {
            "accuracy": 0.9027777777777778,
            "n": 144
          },
          "agnews": {
            "accuracy": 0.85,
            "n": 160
          },
          "emotion": {
            "accuracy": 0.8072916666666666,
            "n": 192
          },
          "banking77_12": {
            "accuracy": 0.8645833333333334,
            "n": 96
          },
          "mnli": {
            "accuracy": 0.7333333333333333,
            "n": 120
          },
          "sst5": {
            "accuracy": 0.375,
            "n": 120
          },
          "multilingual_intent": {
            "accuracy": 0.9833333333333333,
            "n": 120
          }
        }
      },
      "probe": {
        "raw": 0.8088709677419355,
        "calibrated": 0.8080645161290323,
        "suites": {
          "triage": {
            "accuracy": 0.953125,
            "n": 192
          },
          "guardrails": {
            "accuracy": 0.8333333333333334,
            "n": 96
          },
          "moderation": {
            "accuracy": 0.9097222222222222,
            "n": 144
          },
          "agnews": {
            "accuracy": 0.8375,
            "n": 160
          },
          "emotion": {
            "accuracy": 0.765625,
            "n": 192
          },
          "banking77_12": {
            "accuracy": 0.8541666666666666,
            "n": 96
          },
          "mnli": {
            "accuracy": 0.7,
            "n": 120
          },
          "sst5": {
            "accuracy": 0.38333333333333336,
            "n": 120
          },
          "multilingual_intent": {
            "accuracy": 0.9666666666666667,
            "n": 120
          }
        }
      }
    },
    "notes": [
      "SysOne Bench v2.1.0: 1,240 evaluation decisions; separate calibration cases excluded.",
      "Headline is raw accuracy. 2B default-calibrated output scores 81.69%; expected score rounding can change an answer.",
      "No exact evaluation-state matches found. One generic phrase also occurs in training in a different task.",
      "Our local measurements, not an official submission. Source-label and single-review limitations remain."
    ]
  },
  "mixture": {
    "raw": {
      "count": 5177,
      "soft_ce": 0.5378481567236334,
      "kl": 0.3128500667943611,
      "brier": 0.07795102462368915,
      "hard_ece": 0.03592472800759368,
      "argmax_agreement": 0.8392891636082673,
      "hard_reliability": [
        {
          "count": 1,
          "confidence": 0.12810141456117147,
          "reference_agreement": 0.0
        },
        {
          "count": 1,
          "confidence": 0.17808204802119829,
          "reference_agreement": 1.0
        },
        {
          "count": 14,
          "confidence": 0.24257392114260015,
          "reference_agreement": 0.14285714285714285
        },
        {
          "count": 36,
          "confidence": 0.2981857696374177,
          "reference_agreement": 0.3611111111111111
        },
        {
          "count": 41,
          "confidence": 0.3667942153417707,
          "reference_agreement": 0.3170731707317073
        },
        {
          "count": 48,
          "confidence": 0.4332494215910779,
          "reference_agreement": 0.3958333333333333
        },
        {
          "count": 72,
          "confidence": 0.5019543202367454,
          "reference_agreement": 0.4166666666666667
        },
        {
          "count": 119,
          "confidence": 0.5629103692065585,
          "reference_agreement": 0.5462184873949579
        },
        {
          "count": 104,
          "confidence": 0.6332285741759146,
          "reference_agreement": 0.5865384615384616
        },
        {
          "count": 105,
          "confidence": 0.6987865669759383,
          "reference_agreement": 0.6095238095238096
        },
        {
          "count": 136,
          "confidence": 0.767071222198325,
          "reference_agreement": 0.7647058823529411
        },
        {
          "count": 162,
          "confidence": 0.8382690977224995,
          "reference_agreement": 0.654320987654321
        },
        {
          "count": 239,
          "confidence": 0.9019639675591342,
          "reference_agreement": 0.7824267782426778
        },
        {
          "count": 2540,
          "confidence": 0.9923164132970297,
          "reference_agreement": 0.9763779527559056
        }
      ],
      "soft_reliability": [
        {
          "count": 1,
          "confidence": 0.12810141456117147,
          "reference_agreement": 0.0
        },
        {
          "count": 3,
          "confidence": 0.18793035987717818,
          "reference_agreement": 0.585952380952381
        },
        {
          "count": 31,
          "confidence": 0.24339919176557098,
          "reference_agreement": 0.1732258064516129
        },
        {
          "count": 92,
          "confidence": 0.3015779123151043,
          "reference_agreement": 0.3153051772435928
        },
        {
          "count": 109,
          "confidence": 0.37125895464207054,
          "reference_agreement": 0.3584004971161288
        },
        {
          "count": 178,
          "confidence": 0.4356524134426011,
          "reference_agreement": 0.40527245817509266
        },
        {
          "count": 257,
          "confidence": 0.5009535780119421,
          "reference_agreement": 0.47064608134187214
        },
        {
          "count": 303,
          "confidence": 0.5644296377273155,
          "reference_agreement": 0.541720527163873
        },
        {
          "count": 294,
          "confidence": 0.6355385674144598,
          "reference_agreement": 0.6042432957696756
        },
        {
          "count": 277,
          "confidence": 0.6999761682566418,
          "reference_agreement": 0.6524029210972869
        },
        {
          "count": 307,
          "confidence": 0.7658967738044244,
          "reference_agreement": 0.7420202291653142
        },
        {
          "count": 298,
          "confidence": 0.8359378521417429,
          "reference_agreement": 0.712327916554792
        },
        {
          "count": 357,
          "confidence": 0.9012147149990467,
          "reference_agreement": 0.8085002060109497
        },
        {
          "count": 2670,
          "confidence": 0.9912683101168284,
          "reference_agreement": 0.9745381151013129
        }
      ]
    },
    "calibrated": {
      "count": 5177,
      "soft_ce": 0.5273850754441781,
      "kl": 0.30238698551490584,
      "brier": 0.07667068372978918,
      "hard_ece": 0.022683582047145094,
      "argmax_agreement": 0.8392891636082673,
      "hard_reliability": [
        {
          "count": 1,
          "confidence": 0.0902039656414318,
          "reference_agreement": 0.0
        },
        {
          "count": 7,
          "confidence": 0.18178814458207726,
          "reference_agreement": 0.14285714285714285
        },
        {
          "count": 26,
          "confidence": 0.2463164869035683,
          "reference_agreement": 0.34615384615384615
        },
        {
          "count": 45,
          "confidence": 0.3018375825956067,
          "reference_agreement": 0.28888888888888886
        },
        {
          "count": 45,
          "confidence": 0.3693818253306442,
          "reference_agreement": 0.4888888888888889
        },
        {
          "count": 61,
          "confidence": 0.4342818859901273,
          "reference_agreement": 0.39344262295081966
        },
        {
          "count": 108,
          "confidence": 0.506915649882339,
          "reference_agreement": 0.48148148148148145
        },
        {
          "count": 120,
          "confidence": 0.5656389407554023,
          "reference_agreement": 0.5916666666666667
        },
        {
          "count": 128,
          "confidence": 0.6355079058052238,
          "reference_agreement": 0.609375
        },
        {
          "count": 121,
          "confidence": 0.6987122183405449,
          "reference_agreement": 0.768595041322314
        },
        {
          "count": 164,
          "confidence": 0.7678756385989138,
          "reference_agreement": 0.6524390243902439
        },
        {
          "count": 197,
          "confidence": 0.8352630985018289,
          "reference_agreement": 0.7614213197969543
        },
        {
          "count": 288,
          "confidence": 0.9043781621738223,
          "reference_agreement": 0.875
        },
        {
          "count": 2307,
          "confidence": 0.9900514116794004,
          "reference_agreement": 0.9852622453402687
        }
      ],
      "soft_reliability": [
        {
          "count": 1,
          "confidence": 0.0902039656414318,
          "reference_agreement": 0.0
        },
        {
          "count": 11,
          "confidence": 0.1764049356043685,
          "reference_agreement": 0.16025974025974027
        },
        {
          "count": 62,
          "confidence": 0.24563573044070408,
          "reference_agreement": 0.2852109181141439
        },
        {
          "count": 109,
          "confidence": 0.30274815792412224,
          "reference_agreement": 0.3152670061997441
        },
        {
          "count": 145,
          "confidence": 0.37130052424867427,
          "reference_agreement": 0.4404963276342849
        },
        {
          "count": 257,
          "confidence": 0.4347165893758491,
          "reference_agreement": 0.4412616319029475
        },
        {
          "count": 309,
          "confidence": 0.5047072100582236,
          "reference_agreement": 0.5101343600735807
        },
        {
          "count": 332,
          "confidence": 0.5671596851896168,
          "reference_agreement": 0.5964502930649225
        },
        {
          "count": 318,
          "confidence": 0.6339718748644634,
          "reference_agreement": 0.6425300739404627
        },
        {
          "count": 289,
          "confidence": 0.6986792109051184,
          "reference_agreement": 0.749686749335722
        },
        {
          "count": 293,
          "confidence": 0.767310440474794,
          "reference_agreement": 0.7051816244953425
        },
        {
          "count": 304,
          "confidence": 0.8347280482715814,
          "reference_agreement": 0.7916243648298814
        },
        {
          "count": 365,
          "confidence": 0.9033124397139142,
          "reference_agreement": 0.8794087464700682
        },
        {
          "count": 2382,
          "confidence": 0.989372309638942,
          "reference_agreement": 0.9846961996864917
        }
      ]
    },
    "text_baseline": {
      "count": 3693,
      "soft_ce": 0.7041168734088186,
      "kl": 0.3887052537056387,
      "brier": 0.09690042536340486,
      "hard_ece": 0.022770969448581948,
      "argmax_agreement": 0.7733549959382616,
      "hard_reliability": [
        {
          "count": 3,
          "confidence": 0.11589315469434829,
          "reference_agreement": 0.3333333333333333
        },
        {
          "count": 17,
          "confidence": 0.17569618187345318,
          "reference_agreement": 0.17647058823529413
        },
        {
          "count": 26,
          "confidence": 0.23574174336686576,
          "reference_agreement": 0.23076923076923078
        },
        {
          "count": 43,
          "confidence": 0.30201610783522903,
          "reference_agreement": 0.23255813953488372
        },
        {
          "count": 51,
          "confidence": 0.3728587574318181,
          "reference_agreement": 0.35294117647058826
        },
        {
          "count": 55,
          "confidence": 0.43468960041354515,
          "reference_agreement": 0.36363636363636365
        },
        {
          "count": 78,
          "confidence": 0.5068967125125762,
          "reference_agreement": 0.47435897435897434
        },
        {
          "count": 116,
          "confidence": 0.5692509240339776,
          "reference_agreement": 0.5689655172413793
        },
        {
          "count": 92,
          "confidence": 0.6274691893699693,
          "reference_agreement": 0.5760869565217391
        },
        {
          "count": 113,
          "confidence": 0.6976255782232683,
          "reference_agreement": 0.584070796460177
        },
        {
          "count": 125,
          "confidence": 0.7655885050647548,
          "reference_agreement": 0.72
        },
        {
          "count": 153,
          "confidence": 0.8348620973385892,
          "reference_agreement": 0.7777777777777778
        },
        {
          "count": 193,
          "confidence": 0.9005488041523733,
          "reference_agreement": 0.8860103626943006
        },
        {
          "count": 1069,
          "confidence": 0.9873898699630395,
          "reference_agreement": 0.9850327408793265
        }
      ],
      "soft_reliability": [
        {
          "count": 3,
          "confidence": 0.11589315469434829,
          "reference_agreement": 0.3333333333333333
        },
        {
          "count": 23,
          "confidence": 0.1712254393277906,
          "reference_agreement": 0.22601507061461293
        },
        {
          "count": 65,
          "confidence": 0.241401549839449,
          "reference_agreement": 0.27101087801087803
        },
        {
          "count": 136,
          "confidence": 0.3047942264115045,
          "reference_agreement": 0.30327285409683113
        },
        {
          "count": 180,
          "confidence": 0.3704360020830201,
          "reference_agreement": 0.3727716890584122
        },
        {
          "count": 239,
          "confidence": 0.4341484282824121,
          "reference_agreement": 0.4200372081322403
        },
        {
          "count": 268,
          "confidence": 0.5028656676116453,
          "reference_agreement": 0.5155282699000787
        },
        {
          "count": 335,
          "confidence": 0.5660902452493901,
          "reference_agreement": 0.600639823533436
        },
        {
          "count": 269,
          "confidence": 0.631949793983103,
          "reference_agreement": 0.6359665011640041
        },
        {
          "count": 261,
          "confidence": 0.6984239785753936,
          "reference_agreement": 0.6471423835974844
        },
        {
          "count": 249,
          "confidence": 0.7657109724234291,
          "reference_agreement": 0.7478915785725628
        },
        {
          "count": 253,
          "confidence": 0.8344639775274048,
          "reference_agreement": 0.8022606606356278
        },
        {
          "count": 261,
          "confidence": 0.8995167793570855,
          "reference_agreement": 0.8815173885724905
        },
        {
          "count": 1151,
          "confidence": 0.9856462873311037,
          "reference_agreement": 0.9822842533704442
        }
      ]
    },
    "probe": {
      "count": 5177,
      "soft_ce": 0.543304013622336,
      "kl": 0.3183059236930638,
      "brier": 0.07889779767016061,
      "hard_ece": 0.03444247489252796,
      "argmax_agreement": 0.8406412980490632,
      "hard_reliability": [
        {
          "count": 1,
          "confidence": 0.0477185636723529,
          "reference_agreement": 1.0
        },
        {
          "count": 4,
          "confidence": 0.10406288943267586,
          "reference_agreement": 0.25
        },
        {
          "count": 5,
          "confidence": 0.1556279029025097,
          "reference_agreement": 0.0
        },
        {
          "count": 13,
          "confidence": 0.2430852660395774,
          "reference_agreement": 0.07692307692307693
        },
        {
          "count": 14,
          "confidence": 0.306646971695625,
          "reference_agreement": 0.2857142857142857
        },
        {
          "count": 35,
          "confidence": 0.3637846193556214,
          "reference_agreement": 0.34285714285714286
        },
        {
          "count": 57,
          "confidence": 0.4336975008206835,
          "reference_agreement": 0.45614035087719296
        },
        {
          "count": 81,
          "confidence": 0.49849495356422063,
          "reference_agreement": 0.4567901234567901
        },
        {
          "count": 91,
          "confidence": 0.5676232336987276,
          "reference_agreement": 0.4945054945054945
        },
        {
          "count": 102,
          "confidence": 0.6351665776487163,
          "reference_agreement": 0.6176470588235294
        },
        {
          "count": 97,
          "confidence": 0.7019808772191405,
          "reference_agreement": 0.6701030927835051
        },
        {
          "count": 105,
          "confidence": 0.7636261487164356,
          "reference_agreement": 0.6285714285714286
        },
        {
          "count": 191,
          "confidence": 0.8320508600860989,
          "reference_agreement": 0.6858638743455497
        },
        {
          "count": 336,
          "confidence": 0.9045248525591127,
          "reference_agreement": 0.8095238095238095
        },
        {
          "count": 2486,
          "confidence": 0.9878922295687415,
          "reference_agreement": 0.9762670957361222
        }
      ],
      "soft_reliability": [
        {
          "count": 1,
          "confidence": 0.0477185636723529,
          "reference_agreement": 1.0
        },
        {
          "count": 6,
          "confidence": 0.1077881939981522,
          "reference_agreement": 0.16749999999999998
        },
        {
          "count": 8,
          "confidence": 0.15318790941130353,
          "reference_agreement": 0.10598214285714286
        },
        {
          "count": 67,
          "confidence": 0.24205331108775274,
          "reference_agreement": 0.23008231067932558
        },
        {
          "count": 103,
          "confidence": 0.3037312059490086,
          "reference_agreement": 0.35951691556861026
        },
        {
          "count": 217,
          "confidence": 0.36553920775732335,
          "reference_agreement": 0.41878631854128884
        },
        {
          "count": 269,
          "confidence": 0.4337044346532178,
          "reference_agreement": 0.5020223418117048
        },
        {
          "count": 331,
          "confidence": 0.49968041214346126,
          "reference_agreement": 0.5716128350107388
        },
        {
          "count": 309,
          "confidence": 0.5650443437867192,
          "reference_agreement": 0.6157936701296838
        },
        {
          "count": 258,
          "confidence": 0.6326105548378836,
          "reference_agreement": 0.6945477350971593
        },
        {
          "count": 222,
          "confidence": 0.7013481193949082,
          "reference_agreement": 0.725677340398377
        },
        {
          "count": 213,
          "confidence": 0.7651030918065335,
          "reference_agreement": 0.7269792061866328
        },
        {
          "count": 247,
          "confidence": 0.8319646744206926,
          "reference_agreement": 0.7299155500140936
        },
        {
          "count": 399,
          "confidence": 0.9031562802158783,
          "reference_agreement": 0.8284408554681669
        },
        {
          "count": 2527,
          "confidence": 0.9873996385186923,
          "reference_agreement": 0.9757507282166693
        }
      ]
    },
    "notes": [
      "2B mixture v1: 5,177 held-out decisions. Agreement with target-distribution argmax is not independent human correctness.",
      "0.8B mixture v0 has 3,693 different held-out decisions; its score is not a controlled size comparison.",
      "Calibration uses only the calibration split. ECE here is measured on one-hot targets; definitions differ across benchmarks."
    ]
  },
  "performance": {
    "text": {
      "median_evaluation_call_ms": 38.19018650006001,
      "peak_allocated_gib": 1.5406303405761719,
      "scope": "Local RTX4060 Laptop batch-one case, includes tokenization, native forward/readout, answer conversion and validation; not comparable to hosted latency or official operator hardware"
    },
    "vision": {
      "median_evaluation_call_ms": 45.64361150005425,
      "peak_allocated_gib": 4.281294822692871,
      "scope": "Local RTX4060 Laptop batch-one case, includes tokenization, native forward/readout, answer conversion and validation; not comparable to hosted latency or official operator hardware"
    },
    "typed_vision": {
      "median_preencoded_inference_ms": 139.48801850028758,
      "peak_allocated_gib": 4.326559543609619,
      "latency_scope": "Local batch-one per case, all five questions; pretokenized, excludes model load/network; forward includes device-to-host logits"
    }
  }
}