{ "base_model": { "repo": "ibm-granite/granite-4.0-350m", "revision": "bd8a1497065c0d6ba1ef19af6b0d2b14bacf71c2" }, "control": { "adapters_sha256": "c28c7d8e659fb1e8b58d3e377f0bfaf316ac7b6fb993c986361b39ff2a573dfa", "domains": { "public_nli": { "accuracy": 0.2760416666666667, "always_none_accuracy": 0.25, "correct": 106, "correct_none": 32, "count": 384, "first_option_accuracy": 0.3151041666666667, "nll": 1.5817945361999355, "none_cases": 96, "none_precision": 0.32, "none_recall": 0.3333333333333333, "predicted_none": 100, "probabilities_calibrated": false, "supported_accuracy": 0.2569444444444444, "supported_cases": 288, "uniform_choice_expected_accuracy": 0.3018663194444444 }, "structured": { "accuracy": 0.5442708333333334, "always_none_accuracy": 0.25, "correct": 209, "correct_none": 89, "count": 384, "first_option_accuracy": 0.22916666666666666, "nll": 1.1534728874348312, "none_cases": 96, "none_precision": 0.3920704845814978, "none_recall": 0.9270833333333334, "predicted_none": 227, "probabilities_calibrated": false, "supported_accuracy": 0.4166666666666667, "supported_cases": 288, "uniform_choice_expected_accuracy": 0.22820560515873015 } }, "families": { "active_budget": { "accuracy": 0.34375, "always_none_accuracy": 0.25, "correct": 11, "correct_none": 8, "count": 32, "first_option_accuracy": 0.25, "nll": 1.887038529287684, "none_cases": 8, "none_precision": 0.4, "none_recall": 1.0, "predicted_none": 20, "probabilities_calibrated": false, "supported_accuracy": 0.125, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.22604166666666667 }, "budget": { "accuracy": 0.25, "always_none_accuracy": 0.25, "correct": 8, "correct_none": 8, "count": 32, "first_option_accuracy": 0.1875, "nll": 2.0620913420888045, "none_cases": 8, "none_precision": 0.34782608695652173, "none_recall": 1.0, "predicted_none": 23, "probabilities_calibrated": false, "supported_accuracy": 0.0, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23191964285714287 }, "capabilities": { "accuracy": 0.25, "always_none_accuracy": 0.25, "correct": 8, "correct_none": 8, "count": 32, "first_option_accuracy": 0.34375, "nll": 1.970350448545683, "none_cases": 8, "none_precision": 0.25, "none_recall": 1.0, "predicted_none": 32, "probabilities_calibrated": false, "supported_accuracy": 0.0, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2363095238095238 }, "capability_priority": { "accuracy": 0.28125, "always_none_accuracy": 0.25, "correct": 9, "correct_none": 8, "count": 32, "first_option_accuracy": 0.21875, "nll": 1.7792178794110025, "none_cases": 8, "none_precision": 0.27586206896551724, "none_recall": 1.0, "predicted_none": 29, "probabilities_calibrated": false, "supported_accuracy": 0.041666666666666664, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.22879464285714285 }, "category": { "accuracy": 0.875, "always_none_accuracy": 0.25, "correct": 28, "correct_none": 8, "count": 32, "first_option_accuracy": 0.125, "nll": 0.3475019025975081, "none_cases": 8, "none_precision": 0.6666666666666666, "none_recall": 1.0, "predicted_none": 12, "probabilities_calibrated": false, "supported_accuracy": 0.8333333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.21220238095238095 }, "category_deadline": { "accuracy": 0.59375, "always_none_accuracy": 0.25, "correct": 19, "correct_none": 8, "count": 32, "first_option_accuracy": 0.25, "nll": 0.9800761008438408, "none_cases": 8, "none_precision": 0.4, "none_recall": 1.0, "predicted_none": 20, "probabilities_calibrated": false, "supported_accuracy": 0.4583333333333333, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.22633928571428572 }, "deadline": { "accuracy": 0.875, "always_none_accuracy": 0.25, "correct": 28, "correct_none": 8, "count": 32, "first_option_accuracy": 0.1875, "nll": 0.7100813722029362, "none_cases": 8, "none_precision": 0.6666666666666666, "none_recall": 1.0, "predicted_none": 12, "probabilities_calibrated": false, "supported_accuracy": 0.8333333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2125 }, "evidence": { "accuracy": 0.6875, "always_none_accuracy": 0.25, "correct": 22, "correct_none": 7, "count": 32, "first_option_accuracy": 0.3125, "nll": 0.6439506544988722, "none_cases": 8, "none_precision": 0.4375, "none_recall": 0.875, "predicted_none": 16, "probabilities_calibrated": false, "supported_accuracy": 0.625, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23623511904761904 }, "language": { "accuracy": 0.6875, "always_none_accuracy": 0.25, "correct": 22, "correct_none": 8, "count": 32, "first_option_accuracy": 0.25, "nll": 0.5433717814656607, "none_cases": 8, "none_precision": 0.4444444444444444, "none_recall": 1.0, "predicted_none": 18, "probabilities_calibrated": false, "supported_accuracy": 0.5833333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23616071428571428 }, "language_evidence": { "accuracy": 0.59375, "always_none_accuracy": 0.25, "correct": 19, "correct_none": 2, "count": 32, "first_option_accuracy": 0.21875, "nll": 1.205724265738985, "none_cases": 8, "none_precision": 0.5, "none_recall": 0.25, "predicted_none": 4, "probabilities_calibrated": false, "supported_accuracy": 0.7083333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23087797619047618 }, "lookup": { "accuracy": 0.84375, "always_none_accuracy": 0.25, "correct": 27, "correct_none": 8, "count": 32, "first_option_accuracy": 0.1875, "nll": 0.43465208126857946, "none_cases": 8, "none_precision": 0.6153846153846154, "none_recall": 1.0, "predicted_none": 13, "probabilities_calibrated": false, "supported_accuracy": 0.7916666666666666, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2200892857142857 }, "priority": { "accuracy": 0.25, "always_none_accuracy": 0.25, "correct": 8, "correct_none": 8, "count": 32, "first_option_accuracy": 0.21875, "nll": 1.2776182912684264, "none_cases": 8, "none_precision": 0.2857142857142857, "none_recall": 1.0, "predicted_none": 28, "probabilities_calibrated": false, "supported_accuracy": 0.0, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2409970238095238 }, "public_nli": { "accuracy": 0.2760416666666667, "always_none_accuracy": 0.25, "correct": 106, "correct_none": 32, "count": 384, "first_option_accuracy": 0.3151041666666667, "nll": 1.5817945361999355, "none_cases": 96, "none_precision": 0.32, "none_recall": 0.3333333333333333, "predicted_none": 100, "probabilities_calibrated": false, "supported_accuracy": 0.2569444444444444, "supported_cases": 288, "uniform_choice_expected_accuracy": 0.3018663194444444 } }, "metrics": { "accuracy": 0.41015625, "always_none_accuracy": 0.25, "correct": 315, "correct_none": 121, "count": 768, "first_option_accuracy": 0.2721354166666667, "nll": 1.3676337118173836, "none_cases": 192, "none_precision": 0.37003058103975534, "none_recall": 0.6302083333333334, "predicted_none": 327, "probabilities_calibrated": false, "supported_accuracy": 0.3368055555555556, "supported_cases": 576, "uniform_choice_expected_accuracy": 0.26503596230158727 }, "robustness": { "rename": { "agreement_count": 32, "agreement_rate": 1.0, "complete": true, "distribution_agreement_count": 32, "distribution_agreement_rate": 1.0, "expected_count": 32, "rows": [ { "choice_original_id": "19d1cbc43aa6fc88", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:03548e607f76440158e61651", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-12e2ada3fb2f6228", "distribution": [ { "id": "opaque-bc7db2510f59d92c", "logit": 0.2736167907714844, "probability": 0.15414613485336304 }, { "id": "opaque-12e2ada3fb2f6228", "logit": 1.376638650894165, "probability": 0.46448203921318054 }, { "id": "opaque-0401520b989e4ae3", "logit": 0.6827830672264099, "probability": 0.23207655549049377 }, { "id": null, "logit": 0.24164168536663055, "probability": 0.14929525554180145 } ], "elapsed_seconds": 0.043427337979665026, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 110, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-0401520b989e4ae3" }, { "choice_original_id": "c0ecb839ba1e7566", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:65510055ee8da319", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-312cca3e7e9a712b", "distribution": [ { "id": "opaque-312cca3e7e9a712b", "logit": 0.6204967498779297, "probability": 0.5613521933555603 }, { "id": "opaque-8a5e8b808a3271bf", "logit": -3.4770052433013916, "probability": 0.009326377883553505 }, { "id": null, "logit": 0.35235413908958435, "probability": 0.42932143807411194 } ], "elapsed_seconds": 0.04321074500330724, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 107, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-312cca3e7e9a712b" }, { "choice_original_id": null, "distribution_exact": true, "family": "capability_priority", "id": "canonical-confirmation:direct:9e75a5299593dd59", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-591589b68baebe0b", "logit": 0.24900807440280914, "probability": 0.2675934433937073 }, { "id": "opaque-0118b16068da7b37", "logit": -1.6718116998672485, "probability": 0.03919891640543938 }, { "id": "opaque-8b79fde3a0e1041f", "logit": -1.5443333387374878, "probability": 0.04452841356396675 }, { "id": "opaque-b38eafbb4b7148aa", "logit": -1.0005723237991333, "probability": 0.07669898867607117 }, { "id": null, "logit": 1.0086435079574585, "probability": 0.5719801783561707 } ], "elapsed_seconds": 0.05760294297942892, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-0118b16068da7b37" }, { "choice_original_id": "bd557427fdc08978", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:2815c175bf14eed80b84559b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-d409f4f7b4301854", "distribution": [ { "id": "opaque-d409f4f7b4301854", "logit": 2.5377461910247803, "probability": 0.7979376912117004 }, { "id": "opaque-0edc2369a225d228", "logit": 0.7606664896011353, "probability": 0.1349562704563141 }, { "id": null, "logit": 0.061989326030015945, "probability": 0.06710601598024368 } ], "elapsed_seconds": 0.04357933398569003, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 109, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-d409f4f7b4301854" }, { "choice_original_id": "fe8d0c5b94a69c0e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:c80d3546c430419f2cf9b772", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-baeddc43d62a5584", "distribution": [ { "id": "opaque-baeddc43d62a5584", "logit": 0.7290850281715393, "probability": 0.5200138688087463 }, { "id": "opaque-901035a3aa7ad27b", "logit": -0.6967300772666931, "probability": 0.1249658390879631 }, { "id": null, "logit": 0.34740450978279114, "probability": 0.35502031445503235 } ], "elapsed_seconds": 0.03690821799682453, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "94e4c20de1984ebd", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1b279ebfa2f80fd339e274a9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-e2f9d4122fe1169f", "distribution": [ { "id": "opaque-e2f9d4122fe1169f", "logit": 0.6887701153755188, "probability": 0.5049523711204529 }, { "id": "opaque-eab7750e7bc7e6d1", "logit": 0.1226280927658081, "probability": 0.28666722774505615 }, { "id": "opaque-7b56d045ac525f45", "logit": -0.7176096439361572, "probability": 0.12372785806655884 }, { "id": null, "logit": -1.0971393585205078, "probability": 0.08465250581502914 } ], "elapsed_seconds": 0.04768142799730413, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 145, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-eab7750e7bc7e6d1" }, { "choice_original_id": null, "distribution_exact": true, "family": "priority", "id": "canonical-confirmation:direct:03316fb9e789b103", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-b0b2e6dfa3cc5b35", "logit": -2.199821949005127, "probability": 0.041109420359134674 }, { "id": "opaque-1b2ffeace2cfd5e6", "logit": -2.0946810245513916, "probability": 0.04566710442304611 }, { "id": null, "logit": 0.9009214043617249, "probability": 0.9132235050201416 } ], "elapsed_seconds": 0.0370456239907071, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "8a108f66b2d6f2a8", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:dcb34c581cba491997e1effd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-0bdb96ba82e959a6", "distribution": [ { "id": "opaque-0bdb96ba82e959a6", "logit": -0.260655015707016, "probability": 0.49206316471099854 }, { "id": "opaque-e5b8b40969f05e2c", "logit": -0.6589393019676208, "probability": 0.33040618896484375 }, { "id": "opaque-7f1d1077ef912a8d", "logit": -2.3992602825164795, "probability": 0.05797440931200981 }, { "id": null, "logit": -1.675474762916565, "probability": 0.11955629289150238 } ], "elapsed_seconds": 0.04359779300284572, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 124, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-0bdb96ba82e959a6" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:e623e0ccf963a9f3", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-fa4e88dfff02e984", "logit": -1.9700998067855835, "probability": 0.032328445464372635 }, { "id": "opaque-a10c6645f8d4cc7c", "logit": -2.2015905380249023, "probability": 0.02564777061343193 }, { "id": "opaque-dd45d30fd7c2f12f", "logit": -2.5901806354522705, "probability": 0.01738949865102768 }, { "id": "opaque-91c9f1e920776777", "logit": -2.6375539302825928, "probability": 0.016584910452365875 }, { "id": "opaque-53e80da74632d0e9", "logit": -0.9347931146621704, "probability": 0.09103590250015259 }, { "id": "opaque-5ed0964c7c04217e", "logit": -1.5617583990097046, "probability": 0.04863234609365463 }, { "id": null, "logit": 1.1982386112213135, "probability": 0.7683810591697693 } ], "elapsed_seconds": 0.06447403199854307, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 200, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "5b795f5258fbf396", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:7dc843eca0200afe90992317", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-68a9282e226bbbb1", "distribution": [ { "id": "opaque-78e4219772dc6fd9", "logit": -0.38993674516677856, "probability": 0.029356293380260468 }, { "id": "opaque-68a9282e226bbbb1", "logit": 2.8106319904327393, "probability": 0.7205938696861267 }, { "id": null, "logit": 1.7522165775299072, "probability": 0.2500498294830322 } ], "elapsed_seconds": 0.04334643000038341, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 99, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-78e4219772dc6fd9" }, { "choice_original_id": null, "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:ed140718172a188d", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-c6fac6893f59a196", "logit": -1.4101358652114868, "probability": 0.029979661107063293 }, { "id": "opaque-6f5ddeba8caeff07", "logit": -0.5350826382637024, "probability": 0.07192131131887436 }, { "id": "opaque-deedc594dcf1e513", "logit": 0.09746363759040833, "probability": 0.13538450002670288 }, { "id": "opaque-bfa435be381fade2", "logit": 0.45415711402893066, "probability": 0.1934100091457367 }, { "id": "opaque-761f6d89b3718efa", "logit": 0.5474462509155273, "probability": 0.2123214602470398 }, { "id": null, "logit": 1.067033052444458, "probability": 0.3569830358028412 } ], "elapsed_seconds": 0.04783360299188644, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 147, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-bfa435be381fade2" }, { "choice_original_id": "0ba2cad0d646a4ed", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:225545e13b196dc9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-71c4f97f3359e493", "distribution": [ { "id": "opaque-b6999fd7a9434040", "logit": -1.9775400161743164, "probability": 0.02824152633547783 }, { "id": "opaque-48ee4bd29b089264", "logit": -0.3190062344074249, "probability": 0.14831332862377167 }, { "id": "opaque-71c4f97f3359e493", "logit": 0.8250051736831665, "probability": 0.46560540795326233 }, { "id": null, "logit": 0.5617518424987793, "probability": 0.3578397333621979 } ], "elapsed_seconds": 0.047943840007064864, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-71c4f97f3359e493" }, { "choice_original_id": "41458bcc2739bf4c", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:cbbfa8df77bd4dfce682b0bb", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-d90f3865e3bf1cab", "distribution": [ { "id": "opaque-d90f3865e3bf1cab", "logit": 2.4997496604919434, "probability": 0.6957250833511353 }, { "id": "opaque-cb28a50cfc974c06", "logit": 0.6876723766326904, "probability": 0.1136220246553421 }, { "id": "opaque-7e49720fef4af5a9", "logit": -0.05988604947924614, "probability": 0.05380244925618172 }, { "id": null, "logit": 0.8736838102340698, "probability": 0.13685044646263123 } ], "elapsed_seconds": 0.043642331002047285, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-cb28a50cfc974c06" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:74fc8ce91192c581766967f0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-40c85e474d616664", "logit": -1.525586485862732, "probability": 0.04006703570485115 }, { "id": "opaque-4880362ed73fd795", "logit": -0.010158630087971687, "probability": 0.18235981464385986 }, { "id": "opaque-48713e52aedd84e9", "logit": 0.21509204804897308, "probability": 0.2284305989742279 }, { "id": null, "logit": 1.0922176837921143, "probability": 0.5491425395011902 } ], "elapsed_seconds": 0.04320797399850562, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 128, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-48713e52aedd84e9" }, { "choice_original_id": "39389fe9ba71711c", "distribution_exact": true, "family": "category", "id": "canonical-confirmation:direct:18052382cdc27ffa", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-82950d984f7f1a48", "distribution": [ { "id": "opaque-1b954a60876cac8c", "logit": -1.8476002216339111, "probability": 0.02667872980237007 }, { "id": "opaque-2c9af7535da1fd9e", "logit": -1.2209552526474, "probability": 0.04992448166012764 }, { "id": "opaque-82950d984f7f1a48", "logit": 1.3295307159423828, "probability": 0.63969886302948 }, { "id": "opaque-9c25d2421375c022", "logit": -1.1315284967422485, "probability": 0.05459478870034218 }, { "id": null, "logit": 0.3027056157588959, "probability": 0.2291031777858734 } ], "elapsed_seconds": 0.04380694599240087, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 126, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-82950d984f7f1a48" }, { "choice_original_id": "e220c9996bcc3aa9", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:cd7288ebebd6ebd0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-200eb6ef1f673492", "distribution": [ { "id": "opaque-4bd49ae09586d288", "logit": -0.9795262813568115, "probability": 0.03249669075012207 }, { "id": "opaque-b40e0e7d42ce3985", "logit": -2.1815545558929443, "probability": 0.009767980314791203 }, { "id": "opaque-200eb6ef1f673492", "logit": 2.1983251571655273, "probability": 0.7797627449035645 }, { "id": null, "logit": 0.7209650278091431, "probability": 0.17797259986400604 } ], "elapsed_seconds": 0.04773450599168427, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-200eb6ef1f673492" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1036efba79c0542aaeef3c97", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-568cd16b291b5816", "logit": 1.0969780683517456, "probability": 0.08074428141117096 }, { "id": "opaque-bc46a5e962ce8d1d", "logit": 2.808837890625, "probability": 0.44726306200027466 }, { "id": null, "logit": 2.862654447555542, "probability": 0.4719926714897156 } ], "elapsed_seconds": 0.04299776101834141, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 97, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-568cd16b291b5816" }, { "choice_original_id": "bb24192e2625edf1", "distribution_exact": true, "family": "evidence", "id": "canonical-confirmation:direct:089b2000bb7a8d02", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-059e9ce177f12279", "distribution": [ { "id": "opaque-059e9ce177f12279", "logit": 1.3803554773330688, "probability": 0.580704391002655 }, { "id": "opaque-26dcf8c14de27430", "logit": -0.8865097165107727, "probability": 0.06018219515681267 }, { "id": null, "logit": 0.8997520208358765, "probability": 0.3591134548187256 } ], "elapsed_seconds": 0.036420562013518065, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 88, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-059e9ce177f12279" }, { "choice_original_id": "e9bd53c9f5d94f15", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:02719b1656991c1243d0cdb5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-db222ea100d3e7b3", "distribution": [ { "id": "opaque-e58b44b85027071d", "logit": -1.125968098640442, "probability": 0.07446815818548203 }, { "id": "opaque-dd16462f056edd3f", "logit": 0.13984692096710205, "probability": 0.26406270265579224 }, { "id": "opaque-db222ea100d3e7b3", "logit": 0.5170528292655945, "probability": 0.38505738973617554 }, { "id": null, "logit": 0.18555207550525665, "probability": 0.276411771774292 } ], "elapsed_seconds": 0.043315521004842594, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-dd16462f056edd3f" }, { "choice_original_id": "f005e77e2222a8de", "distribution_exact": true, "family": "language_evidence", "id": "canonical-confirmation:direct:dac514809d6319dc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-a228de578e869ce4", "distribution": [ { "id": "opaque-466b29b2a1aa4a1d", "logit": -2.0697524547576904, "probability": 0.014280811883509159 }, { "id": "opaque-46b5b26e21d86f46", "logit": -0.5216320157051086, "probability": 0.06715727597475052 }, { "id": "opaque-a228de578e869ce4", "logit": 1.6429344415664673, "probability": 0.5849953293800354 }, { "id": "opaque-f8bc0ce4e2d0a295", "logit": -1.5107536315917969, "probability": 0.024976013228297234 }, { "id": "opaque-ee3cf91d6a58e07c", "logit": 0.5947917699813843, "probability": 0.20509250462055206 }, { "id": null, "logit": -0.08911596238613129, "probability": 0.10349811613559723 } ], "elapsed_seconds": 0.057641891995444894, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-a228de578e869ce4" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:8a2c005ecf0095bd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-4ddd09ee01a12d17", "logit": -0.8736057877540588, "probability": 0.1380721926689148 }, { "id": "opaque-b2cbff940f0b10cc", "logit": -2.250525951385498, "probability": 0.034843143075704575 }, { "id": "opaque-bbdca15a4c1ee7ba", "logit": -2.6774675846099854, "probability": 0.02273521013557911 }, { "id": "opaque-176918f028afffc1", "logit": -3.2116141319274902, "probability": 0.013326682150363922 }, { "id": null, "logit": 0.8719443678855896, "probability": 0.7910228371620178 } ], "elapsed_seconds": 0.04913923199637793, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 155, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "capabilities", "id": "canonical-confirmation:direct:17f6ca9ce2782904", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-e4fb1eb496fbfc70", "logit": 0.24436567723751068, "probability": 0.27215641736984253 }, { "id": "opaque-7ff3201335aec9d6", "logit": -1.8830775022506714, "probability": 0.032425135374069214 }, { "id": "opaque-71ec2a50dfc414be", "logit": -3.5141544342041016, "probability": 0.0063462029211223125 }, { "id": null, "logit": 1.1733348369598389, "probability": 0.6890722513198853 } ], "elapsed_seconds": 0.04802849600673653, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 129, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-e4fb1eb496fbfc70" }, { "choice_original_id": "d711e3472cc27285", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:9ede679f928684b3dafb6313", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-0ffcaca59073e77d", "distribution": [ { "id": "opaque-0ffcaca59073e77d", "logit": 2.1605730056762695, "probability": 0.49968263506889343 }, { "id": "opaque-9c15e3e0e96dcdae", "logit": 0.6952272653579712, "probability": 0.11542574316263199 }, { "id": null, "logit": 1.8995616436004639, "probability": 0.3848915994167328 } ], "elapsed_seconds": 0.037063152994960546, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 90, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-9c15e3e0e96dcdae" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:3301caf361ab71a8", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-d7c6e35c384e1aa4", "logit": -2.5476932525634766, "probability": 0.042109936475753784 }, { "id": "opaque-e71a5d9703cc68e1", "logit": -3.2507712841033936, "probability": 0.020846908912062645 }, { "id": "opaque-0326b3888078ce6d", "logit": -3.119550943374634, "probability": 0.0237700417637825 }, { "id": "opaque-209fdc568fa474aa", "logit": -0.7297782301902771, "probability": 0.2593552768230438 }, { "id": null, "logit": 0.1950046569108963, "probability": 0.6539177894592285 } ], "elapsed_seconds": 0.048137503006728366, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 156, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "c35770eec265bb9e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:eefdcea7cdfa3c6365491577", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-4f51f76028ab9213", "distribution": [ { "id": "opaque-4f51f76028ab9213", "logit": 2.584247350692749, "probability": 0.7695059180259705 }, { "id": "opaque-52ea141023981f0d", "logit": 1.197462797164917, "probability": 0.19228219985961914 }, { "id": null, "logit": -0.4183555543422699, "probability": 0.03821185603737831 } ], "elapsed_seconds": 0.04358351297560148, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-52ea141023981f0d" }, { "choice_original_id": "4a51c4ddc32d5eef", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:a7d25ab08a792cf8f3c50b3b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-bbb5d408c4aa2861", "distribution": [ { "id": "opaque-bbb5d408c4aa2861", "logit": 2.2102553844451904, "probability": 0.8631445169448853 }, { "id": "opaque-8aff54da3a90c870", "logit": -0.4100944995880127, "probability": 0.06281740963459015 }, { "id": null, "logit": -0.24574731290340424, "probability": 0.074038065969944 } ], "elapsed_seconds": 0.04761323001002893, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 131, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:8f4e7e3049890865642f13fc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-88469ccdea00d36d", "logit": 1.8875857591629028, "probability": 0.3019196093082428 }, { "id": "opaque-216404ed15307e9f", "logit": -0.0917062908411026, "probability": 0.04171533137559891 }, { "id": null, "logit": 2.66414213180542, "probability": 0.6563650369644165 } ], "elapsed_seconds": 0.036942876991815865, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "budget", "id": "canonical-confirmation:direct:63849bf99697604b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-2c723b933134ab59", "logit": -1.7078907489776611, "probability": 0.08150847256183624 }, { "id": "opaque-c94fd650b55fe9df", "logit": -1.3181856870651245, "probability": 0.1203509271144867 }, { "id": null, "logit": 0.5736871361732483, "probability": 0.7981406450271606 } ], "elapsed_seconds": 0.03672028399887495, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-2c723b933134ab59" }, { "choice_original_id": "9d4bc8c6f1ff3ffb", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:07ae8d2974d5d511eea3f9b5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-3b32cbf759f4a179", "distribution": [ { "id": "opaque-15761e529b538ddb", "logit": 1.3850442171096802, "probability": 0.41871902346611023 }, { "id": "opaque-3b32cbf759f4a179", "logit": 1.485336422920227, "probability": 0.4628913104534149 }, { "id": null, "logit": 0.12182560563087463, "probability": 0.11838966608047485 } ], "elapsed_seconds": 0.04308834800031036, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 106, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-15761e529b538ddb" }, { "choice_original_id": null, "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:37b8cda8ac2c8294", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-f6f7521f46fb9a9e", "logit": -0.5374056696891785, "probability": 0.1723310500383377 }, { "id": "opaque-3db3c9e49993d651", "logit": -0.932090699672699, "probability": 0.11613256484270096 }, { "id": null, "logit": 0.8806034922599792, "probability": 0.7115363478660583 } ], "elapsed_seconds": 0.036555838014464825, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "c2f741799f839a2c", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:db97eb1a3f6e85d65712bc56", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-77f7d707ec80e453", "distribution": [ { "id": "opaque-77f7d707ec80e453", "logit": 0.9072933197021484, "probability": 0.38301798701286316 }, { "id": "opaque-9cde2e8e22863a2a", "logit": 0.4461953938007355, "probability": 0.24152766168117523 }, { "id": null, "logit": 0.8873482346534729, "probability": 0.3754543364048004 } ], "elapsed_seconds": 0.04322978400159627, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "99632bb0863c7081", "distribution_exact": true, "family": "lookup", "id": "canonical-confirmation:direct:fa8c40fd9550a870", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-fe22cd863ec7b9a6", "distribution": [ { "id": "opaque-de7840d60997c933", "logit": -1.7539016008377075, "probability": 0.02461240068078041 }, { "id": "opaque-74d80858f70d7e32", "logit": -2.058035373687744, "probability": 0.01815810054540634 }, { "id": "opaque-059367613e9a098f", "logit": -2.1184473037719727, "probability": 0.017093611881136894 }, { "id": "opaque-fe22cd863ec7b9a6", "logit": 1.6068394184112549, "probability": 0.7090964317321777 }, { "id": "opaque-0fb9efcc88be25a6", "logit": -1.026995062828064, "probability": 0.050914980471134186 }, { "id": "opaque-346165b257a85c7d", "logit": -2.1581156253814697, "probability": 0.016428809612989426 }, { "id": null, "logit": 0.14085684716701508, "probability": 0.16369566321372986 } ], "elapsed_seconds": 0.057520666014170274, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 165, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-fe22cd863ec7b9a6" } ], "sample_count": 32 }, "reverse": { "agreement_count": 32, "agreement_rate": 1.0, "complete": true, "distribution_agreement_count": 32, "distribution_agreement_rate": 1.0, "expected_count": 32, "rows": [ { "choice_original_id": "19d1cbc43aa6fc88", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:03548e607f76440158e61651", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "19d1cbc43aa6fc88", "distribution": [ { "id": "b75ffb314e5d5487", "logit": 0.6827830672264099, "probability": 0.23207655549049377 }, { "id": "19d1cbc43aa6fc88", "logit": 1.376638650894165, "probability": 0.46448203921318054 }, { "id": "efb6a822718dc8c7", "logit": 0.2736167907714844, "probability": 0.15414613485336304 }, { "id": null, "logit": 0.24164168536663055, "probability": 0.14929525554180145 } ], "elapsed_seconds": 0.04199989201151766, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 110, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "b75ffb314e5d5487" }, { "choice_original_id": "c0ecb839ba1e7566", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:65510055ee8da319", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "c0ecb839ba1e7566", "distribution": [ { "id": "6f5277e14a0f433b", "logit": -3.4770052433013916, "probability": 0.009326377883553505 }, { "id": "c0ecb839ba1e7566", "logit": 0.6204967498779297, "probability": 0.5613521933555603 }, { "id": null, "logit": 0.35235413908958435, "probability": 0.42932143807411194 } ], "elapsed_seconds": 0.04253286600578576, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 107, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "c0ecb839ba1e7566" }, { "choice_original_id": null, "distribution_exact": true, "family": "capability_priority", "id": "canonical-confirmation:direct:9e75a5299593dd59", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "7112a6332052b9ff", "logit": -1.0005723237991333, "probability": 0.07669898867607117 }, { "id": "070ba78983441ec4", "logit": -1.5443333387374878, "probability": 0.04452841356396675 }, { "id": "4ce69769781021fc", "logit": -1.6718116998672485, "probability": 0.03919891640543938 }, { "id": "1a9d14479ab835cd", "logit": 0.24900807440280914, "probability": 0.2675934433937073 }, { "id": null, "logit": 1.0086435079574585, "probability": 0.5719801783561707 } ], "elapsed_seconds": 0.05758130399044603, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "4ce69769781021fc" }, { "choice_original_id": "bd557427fdc08978", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:2815c175bf14eed80b84559b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "bd557427fdc08978", "distribution": [ { "id": "25f8970e22f3c4f1", "logit": 0.7606664896011353, "probability": 0.1349562704563141 }, { "id": "bd557427fdc08978", "logit": 2.5377461910247803, "probability": 0.7979376912117004 }, { "id": null, "logit": 0.061989326030015945, "probability": 0.06710601598024368 } ], "elapsed_seconds": 0.04365983098978177, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 109, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "bd557427fdc08978" }, { "choice_original_id": "fe8d0c5b94a69c0e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:c80d3546c430419f2cf9b772", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "fe8d0c5b94a69c0e", "distribution": [ { "id": "d1909d28a04b6855", "logit": -0.6967300772666931, "probability": 0.1249658390879631 }, { "id": "fe8d0c5b94a69c0e", "logit": 0.7290850281715393, "probability": 0.5200138688087463 }, { "id": null, "logit": 0.34740450978279114, "probability": 0.35502031445503235 } ], "elapsed_seconds": 0.036703113990370184, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "94e4c20de1984ebd", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1b279ebfa2f80fd339e274a9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "94e4c20de1984ebd", "distribution": [ { "id": "6f3e6021727694dc", "logit": -0.7176096439361572, "probability": 0.12372785806655884 }, { "id": "275b2c19296206a4", "logit": 0.1226280927658081, "probability": 0.28666722774505615 }, { "id": "94e4c20de1984ebd", "logit": 0.6887701153755188, "probability": 0.5049523711204529 }, { "id": null, "logit": -1.0971393585205078, "probability": 0.08465250581502914 } ], "elapsed_seconds": 0.04795373900560662, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 145, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "275b2c19296206a4" }, { "choice_original_id": null, "distribution_exact": true, "family": "priority", "id": "canonical-confirmation:direct:03316fb9e789b103", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "37667e37345c013e", "logit": -2.0946810245513916, "probability": 0.04566710442304611 }, { "id": "95d6e9a6489b7f20", "logit": -2.199821949005127, "probability": 0.041109420359134674 }, { "id": null, "logit": 0.9009214043617249, "probability": 0.9132235050201416 } ], "elapsed_seconds": 0.037025623983936384, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "8a108f66b2d6f2a8", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:dcb34c581cba491997e1effd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "8a108f66b2d6f2a8", "distribution": [ { "id": "8826fa788322d575", "logit": -2.3992602825164795, "probability": 0.05797440931200981 }, { "id": "9b64120262f697b7", "logit": -0.6589393019676208, "probability": 0.33040618896484375 }, { "id": "8a108f66b2d6f2a8", "logit": -0.260655015707016, "probability": 0.49206316471099854 }, { "id": null, "logit": -1.675474762916565, "probability": 0.11955629289150238 } ], "elapsed_seconds": 0.04328894198988564, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 124, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "8a108f66b2d6f2a8" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:e623e0ccf963a9f3", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "7582cba3a4f1c900", "logit": -1.5617583990097046, "probability": 0.04863234609365463 }, { "id": "e39430e598907458", "logit": -0.9347931146621704, "probability": 0.09103590250015259 }, { "id": "d096db4a72da038a", "logit": -2.6375539302825928, "probability": 0.016584910452365875 }, { "id": "63bb7cf0107eed47", "logit": -2.5901806354522705, "probability": 0.01738949865102768 }, { "id": "334a8592ce63e429", "logit": -2.2015905380249023, "probability": 0.02564777061343193 }, { "id": "359c953097ca7fef", "logit": -1.9700998067855835, "probability": 0.032328445464372635 }, { "id": null, "logit": 1.1982386112213135, "probability": 0.7683810591697693 } ], "elapsed_seconds": 0.06474683398846537, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 200, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "5b795f5258fbf396", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:7dc843eca0200afe90992317", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "5b795f5258fbf396", "distribution": [ { "id": "5b795f5258fbf396", "logit": 2.8106319904327393, "probability": 0.7205938696861267 }, { "id": "96ba6910d8949928", "logit": -0.38993674516677856, "probability": 0.029356293380260468 }, { "id": null, "logit": 1.7522165775299072, "probability": 0.2500498294830322 } ], "elapsed_seconds": 0.04359462298452854, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 99, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "96ba6910d8949928" }, { "choice_original_id": null, "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:ed140718172a188d", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "095c0c9eee4d98a9", "logit": 0.5474462509155273, "probability": 0.2123214602470398 }, { "id": "9807e64c4702e584", "logit": 0.45415711402893066, "probability": 0.1934100091457367 }, { "id": "770646ae20f726b6", "logit": 0.09746363759040833, "probability": 0.13538450002670288 }, { "id": "9a7fc9871af27760", "logit": -0.5350826382637024, "probability": 0.07192131131887436 }, { "id": "5d1acb45a8c8b604", "logit": -1.4101358652114868, "probability": 0.029979661107063293 }, { "id": null, "logit": 1.067033052444458, "probability": 0.3569830358028412 } ], "elapsed_seconds": 0.04840419499669224, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 147, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "9807e64c4702e584" }, { "choice_original_id": "0ba2cad0d646a4ed", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:225545e13b196dc9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "0ba2cad0d646a4ed", "distribution": [ { "id": "0ba2cad0d646a4ed", "logit": 0.8250051736831665, "probability": 0.46560540795326233 }, { "id": "c79332bdc25aa21d", "logit": -0.3190062344074249, "probability": 0.14831332862377167 }, { "id": "d570197d7fce642c", "logit": -1.9775400161743164, "probability": 0.02824152633547783 }, { "id": null, "logit": 0.5617518424987793, "probability": 0.3578397333621979 } ], "elapsed_seconds": 0.04809181499877013, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "0ba2cad0d646a4ed" }, { "choice_original_id": "41458bcc2739bf4c", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:cbbfa8df77bd4dfce682b0bb", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "41458bcc2739bf4c", "distribution": [ { "id": "7a199bacc2fcdb63", "logit": -0.05988604947924614, "probability": 0.05380244925618172 }, { "id": "4475e3e948fcb9a4", "logit": 0.6876723766326904, "probability": 0.1136220246553421 }, { "id": "41458bcc2739bf4c", "logit": 2.4997496604919434, "probability": 0.6957250833511353 }, { "id": null, "logit": 0.8736838102340698, "probability": 0.13685044646263123 } ], "elapsed_seconds": 0.04361610198975541, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "4475e3e948fcb9a4" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:74fc8ce91192c581766967f0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "b44f372f2b75ca2e", "logit": 0.21509204804897308, "probability": 0.2284305989742279 }, { "id": "7e8f5ea9a3659a4e", "logit": -0.010158630087971687, "probability": 0.18235981464385986 }, { "id": "ea0a5389c32ad8cd", "logit": -1.525586485862732, "probability": 0.04006703570485115 }, { "id": null, "logit": 1.0922176837921143, "probability": 0.5491425395011902 } ], "elapsed_seconds": 0.04312946801655926, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 128, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "b44f372f2b75ca2e" }, { "choice_original_id": "39389fe9ba71711c", "distribution_exact": true, "family": "category", "id": "canonical-confirmation:direct:18052382cdc27ffa", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "39389fe9ba71711c", "distribution": [ { "id": "38b4ce10d19bb6b3", "logit": -1.1315284967422485, "probability": 0.05459478870034218 }, { "id": "39389fe9ba71711c", "logit": 1.3295307159423828, "probability": 0.63969886302948 }, { "id": "4c1149263772b1a0", "logit": -1.2209552526474, "probability": 0.04992448166012764 }, { "id": "f017bbe9d3a47fd9", "logit": -1.8476002216339111, "probability": 0.02667872980237007 }, { "id": null, "logit": 0.3027056157588959, "probability": 0.2291031777858734 } ], "elapsed_seconds": 0.044030279997969046, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 126, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "39389fe9ba71711c" }, { "choice_original_id": "e220c9996bcc3aa9", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:cd7288ebebd6ebd0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "e220c9996bcc3aa9", "distribution": [ { "id": "e220c9996bcc3aa9", "logit": 2.1983251571655273, "probability": 0.7797627449035645 }, { "id": "106f6f3f671d090a", "logit": -2.1815545558929443, "probability": 0.009767980314791203 }, { "id": "85f53f020ea99380", "logit": -0.9795262813568115, "probability": 0.03249669075012207 }, { "id": null, "logit": 0.7209650278091431, "probability": 0.17797259986400604 } ], "elapsed_seconds": 0.04786079199402593, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "e220c9996bcc3aa9" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1036efba79c0542aaeef3c97", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "04ce2823d078e021", "logit": 2.808837890625, "probability": 0.44726306200027466 }, { "id": "2ea4e55981623071", "logit": 1.0969780683517456, "probability": 0.08074428141117096 }, { "id": null, "logit": 2.862654447555542, "probability": 0.4719926714897156 } ], "elapsed_seconds": 0.04313502597506158, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 97, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "2ea4e55981623071" }, { "choice_original_id": "bb24192e2625edf1", "distribution_exact": true, "family": "evidence", "id": "canonical-confirmation:direct:089b2000bb7a8d02", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "bb24192e2625edf1", "distribution": [ { "id": "c1a807174cbb3ecb", "logit": -0.8865097165107727, "probability": 0.06018219515681267 }, { "id": "bb24192e2625edf1", "logit": 1.3803554773330688, "probability": 0.580704391002655 }, { "id": null, "logit": 0.8997520208358765, "probability": 0.3591134548187256 } ], "elapsed_seconds": 0.03656408900860697, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 88, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "bb24192e2625edf1" }, { "choice_original_id": "e9bd53c9f5d94f15", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:02719b1656991c1243d0cdb5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "e9bd53c9f5d94f15", "distribution": [ { "id": "e9bd53c9f5d94f15", "logit": 0.5170528292655945, "probability": 0.38505738973617554 }, { "id": "fb9e7dcfe00f7f16", "logit": 0.13984692096710205, "probability": 0.26406270265579224 }, { "id": "b506f558303241ec", "logit": -1.125968098640442, "probability": 0.07446815818548203 }, { "id": null, "logit": 0.18555207550525665, "probability": 0.276411771774292 } ], "elapsed_seconds": 0.043292062007822096, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "fb9e7dcfe00f7f16" }, { "choice_original_id": "f005e77e2222a8de", "distribution_exact": true, "family": "language_evidence", "id": "canonical-confirmation:direct:dac514809d6319dc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "f005e77e2222a8de", "distribution": [ { "id": "c3bd6af6729ded16", "logit": 0.5947917699813843, "probability": 0.20509250462055206 }, { "id": "072d339d9818dba9", "logit": -1.5107536315917969, "probability": 0.024976013228297234 }, { "id": "f005e77e2222a8de", "logit": 1.6429344415664673, "probability": 0.5849953293800354 }, { "id": "68a9f9da2e63732d", "logit": -0.5216320157051086, "probability": 0.06715727597475052 }, { "id": "c56dc92b7e227c45", "logit": -2.0697524547576904, "probability": 0.014280811883509159 }, { "id": null, "logit": -0.08911596238613129, "probability": 0.10349811613559723 } ], "elapsed_seconds": 0.05763363200821914, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "f005e77e2222a8de" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:8a2c005ecf0095bd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "28d3798e2ff35c4c", "logit": -3.2116141319274902, "probability": 0.013326682150363922 }, { "id": "8c62e5a48f920b9c", "logit": -2.6774675846099854, "probability": 0.02273521013557911 }, { "id": "2f25ba50cfedc1d0", "logit": -2.250525951385498, "probability": 0.034843143075704575 }, { "id": "8e246d8efbfecde7", "logit": -0.8736057877540588, "probability": 0.1380721926689148 }, { "id": null, "logit": 0.8719443678855896, "probability": 0.7910228371620178 } ], "elapsed_seconds": 0.04891250899527222, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 155, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "capabilities", "id": "canonical-confirmation:direct:17f6ca9ce2782904", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "29766f5af26f4ce7", "logit": -3.5141544342041016, "probability": 0.0063462029211223125 }, { "id": "6cbc20b346ef289d", "logit": -1.8830775022506714, "probability": 0.032425135374069214 }, { "id": "7007f74c373dab78", "logit": 0.24436567723751068, "probability": 0.27215641736984253 }, { "id": null, "logit": 1.1733348369598389, "probability": 0.6890722513198853 } ], "elapsed_seconds": 0.04793321999022737, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 129, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "7007f74c373dab78" }, { "choice_original_id": "d711e3472cc27285", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:9ede679f928684b3dafb6313", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "d711e3472cc27285", "distribution": [ { "id": "ad5e5ffec891418d", "logit": 0.6952272653579712, "probability": 0.11542574316263199 }, { "id": "d711e3472cc27285", "logit": 2.1605730056762695, "probability": 0.49968263506889343 }, { "id": null, "logit": 1.8995616436004639, "probability": 0.3848915994167328 } ], "elapsed_seconds": 0.03713792099733837, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 90, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "ad5e5ffec891418d" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:3301caf361ab71a8", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "e483be92d7bd7cfb", "logit": -0.7297782301902771, "probability": 0.2593552768230438 }, { "id": "8518116bda018422", "logit": -3.119550943374634, "probability": 0.0237700417637825 }, { "id": "5a382b14a36236e5", "logit": -3.2507712841033936, "probability": 0.020846908912062645 }, { "id": "1d39ccd03a91cfac", "logit": -2.5476932525634766, "probability": 0.042109936475753784 }, { "id": null, "logit": 0.1950046569108963, "probability": 0.6539177894592285 } ], "elapsed_seconds": 0.04807750598411076, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 156, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "c35770eec265bb9e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:eefdcea7cdfa3c6365491577", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "c35770eec265bb9e", "distribution": [ { "id": "2b58ef5bf973a8d6", "logit": 1.197462797164917, "probability": 0.19228219985961914 }, { "id": "c35770eec265bb9e", "logit": 2.584247350692749, "probability": 0.7695059180259705 }, { "id": null, "logit": -0.4183555543422699, "probability": 0.03821185603737831 } ], "elapsed_seconds": 0.04360230200109072, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "2b58ef5bf973a8d6" }, { "choice_original_id": "4a51c4ddc32d5eef", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:a7d25ab08a792cf8f3c50b3b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "4a51c4ddc32d5eef", "distribution": [ { "id": "6b8e4c0ad12e0e1c", "logit": -0.4100944995880127, "probability": 0.06281740963459015 }, { "id": "4a51c4ddc32d5eef", "logit": 2.2102553844451904, "probability": 0.8631445169448853 }, { "id": null, "logit": -0.24574731290340424, "probability": 0.074038065969944 } ], "elapsed_seconds": 0.047578830999555066, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 131, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:8f4e7e3049890865642f13fc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "1e573c04b67ea3f2", "logit": -0.0917062908411026, "probability": 0.04171533137559891 }, { "id": "f3bff744ef6784ce", "logit": 1.8875857591629028, "probability": 0.3019196093082428 }, { "id": null, "logit": 2.66414213180542, "probability": 0.6563650369644165 } ], "elapsed_seconds": 0.036803070979658514, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "budget", "id": "canonical-confirmation:direct:63849bf99697604b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "43cdd07a93440816", "logit": -1.3181856870651245, "probability": 0.1203509271144867 }, { "id": "ad42d6a6c953882f", "logit": -1.7078907489776611, "probability": 0.08150847256183624 }, { "id": null, "logit": 0.5736871361732483, "probability": 0.7981406450271606 } ], "elapsed_seconds": 0.03643898302107118, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "ad42d6a6c953882f" }, { "choice_original_id": "9d4bc8c6f1ff3ffb", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:07ae8d2974d5d511eea3f9b5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "9d4bc8c6f1ff3ffb", "distribution": [ { "id": "9d4bc8c6f1ff3ffb", "logit": 1.485336422920227, "probability": 0.4628913104534149 }, { "id": "4c9fca103c5dea48", "logit": 1.3850442171096802, "probability": 0.41871902346611023 }, { "id": null, "logit": 0.12182560563087463, "probability": 0.11838966608047485 } ], "elapsed_seconds": 0.042817225999897346, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 106, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "4c9fca103c5dea48" }, { "choice_original_id": null, "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:37b8cda8ac2c8294", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "3b4a74c6a0f44fbd", "logit": -0.932090699672699, "probability": 0.11613256484270096 }, { "id": "5c24f96f4b3139c8", "logit": -0.5374056696891785, "probability": 0.1723310500383377 }, { "id": null, "logit": 0.8806034922599792, "probability": 0.7115363478660583 } ], "elapsed_seconds": 0.03659340800368227, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "c2f741799f839a2c", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:db97eb1a3f6e85d65712bc56", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "c2f741799f839a2c", "distribution": [ { "id": "e71dd00b24c510d4", "logit": 0.4461953938007355, "probability": 0.24152766168117523 }, { "id": "c2f741799f839a2c", "logit": 0.9072933197021484, "probability": 0.38301798701286316 }, { "id": null, "logit": 0.8873482346534729, "probability": 0.3754543364048004 } ], "elapsed_seconds": 0.043279192002955824, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "99632bb0863c7081", "distribution_exact": true, "family": "lookup", "id": "canonical-confirmation:direct:fa8c40fd9550a870", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "99632bb0863c7081", "distribution": [ { "id": "4c3e4e677d1ee034", "logit": -2.1581156253814697, "probability": 0.016428809612989426 }, { "id": "e4d7573daa91e94d", "logit": -1.026995062828064, "probability": 0.050914980471134186 }, { "id": "99632bb0863c7081", "logit": 1.6068394184112549, "probability": 0.7090964317321777 }, { "id": "55ba1c70034f7f2b", "logit": -2.1184473037719727, "probability": 0.017093611881136894 }, { "id": "a1896bf19fceb3da", "logit": -2.058035373687744, "probability": 0.01815810054540634 }, { "id": "7f1a1d767df0791d", "logit": -1.7539016008377075, "probability": 0.02461240068078041 }, { "id": null, "logit": 0.14085684716701508, "probability": 0.16369566321372986 } ], "elapsed_seconds": 0.057375249976757914, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 165, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "99632bb0863c7081" } ], "sample_count": 32 } } }, "evaluation_data_sha256": "3db223190445a89c379a5fe4b6e2f477bce19858da88f5ed6672c3f4e33b4014", "evaluation_scope": "768 locally fresh instances in trained task families; not unseen-family transfer, not an official SNLI benchmark", "experiment": "canonical-course", "full_context_execution_demonstrated": false, "probabilities_calibrated": false, "quality_gates": { "each_domain_accuracy": false, "none_precision": false, "none_recall": false, "overall_accuracy": false, "rename_exact_distribution_invariance": true, "reverse_exact_distribution_invariance": true, "strict_improvement_over_control": true, "supported_accuracy": false }, "quality_gates_passed": 3, "quality_gates_total": 8, "schema_version": 1, "status": "FRESH_IMPROVEMENT_BUT_GENERAL_QUALITY_GATES_FAILED", "timing_seconds": { "fresh_comparison_helper_seconds": 82.52273996901931, "fresh_comparison_service_seconds": 84.683, "training_helper_seconds": 1000.4108294059988, "training_service_seconds": 1007.386, "two_service_wall_seconds": 1092.069 }, "trained": { "adapters_sha256": "adffd25386d5615acac4c2158789ff4af93d21fa4f73a33f66629d4673911b82", "domains": { "public_nli": { "accuracy": 0.3932291666666667, "always_none_accuracy": 0.25, "correct": 151, "correct_none": 62, "count": 384, "first_option_accuracy": 0.3151041666666667, "nll": 1.1078937428953253, "none_cases": 96, "none_precision": 0.4189189189189189, "none_recall": 0.6458333333333334, "predicted_none": 148, "probabilities_calibrated": false, "supported_accuracy": 0.3090277777777778, "supported_cases": 288, "uniform_choice_expected_accuracy": 0.3018663194444444 }, "structured": { "accuracy": 0.7239583333333334, "always_none_accuracy": 0.25, "correct": 278, "correct_none": 65, "count": 384, "first_option_accuracy": 0.22916666666666666, "nll": 0.781663102036906, "none_cases": 96, "none_precision": 0.8666666666666667, "none_recall": 0.6770833333333334, "predicted_none": 75, "probabilities_calibrated": false, "supported_accuracy": 0.7395833333333334, "supported_cases": 288, "uniform_choice_expected_accuracy": 0.22820560515873015 } }, "families": { "active_budget": { "accuracy": 0.59375, "always_none_accuracy": 0.25, "correct": 19, "correct_none": 8, "count": 32, "first_option_accuracy": 0.25, "nll": 1.210970407335013, "none_cases": 8, "none_precision": 1.0, "none_recall": 1.0, "predicted_none": 8, "probabilities_calibrated": false, "supported_accuracy": 0.4583333333333333, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.22604166666666667 }, "budget": { "accuracy": 0.21875, "always_none_accuracy": 0.25, "correct": 7, "correct_none": 2, "count": 32, "first_option_accuracy": 0.1875, "nll": 1.8351480186458053, "none_cases": 8, "none_precision": 1.0, "none_recall": 0.25, "predicted_none": 2, "probabilities_calibrated": false, "supported_accuracy": 0.20833333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23191964285714287 }, "capabilities": { "accuracy": 0.59375, "always_none_accuracy": 0.25, "correct": 19, "correct_none": 3, "count": 32, "first_option_accuracy": 0.34375, "nll": 0.8728004785584496, "none_cases": 8, "none_precision": 0.3333333333333333, "none_recall": 0.375, "predicted_none": 9, "probabilities_calibrated": false, "supported_accuracy": 0.6666666666666666, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2363095238095238 }, "capability_priority": { "accuracy": 0.40625, "always_none_accuracy": 0.25, "correct": 13, "correct_none": 4, "count": 32, "first_option_accuracy": 0.21875, "nll": 1.3531768502760468, "none_cases": 8, "none_precision": 1.0, "none_recall": 0.5, "predicted_none": 4, "probabilities_calibrated": false, "supported_accuracy": 0.375, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.22879464285714285 }, "category": { "accuracy": 1.0, "always_none_accuracy": 0.25, "correct": 32, "correct_none": 8, "count": 32, "first_option_accuracy": 0.125, "nll": 0.1687908049418356, "none_cases": 8, "none_precision": 1.0, "none_recall": 1.0, "predicted_none": 8, "probabilities_calibrated": false, "supported_accuracy": 1.0, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.21220238095238095 }, "category_deadline": { "accuracy": 0.96875, "always_none_accuracy": 0.25, "correct": 31, "correct_none": 8, "count": 32, "first_option_accuracy": 0.25, "nll": 0.3920327269290444, "none_cases": 8, "none_precision": 1.0, "none_recall": 1.0, "predicted_none": 8, "probabilities_calibrated": false, "supported_accuracy": 0.9583333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.22633928571428572 }, "deadline": { "accuracy": 0.75, "always_none_accuracy": 0.25, "correct": 24, "correct_none": 0, "count": 32, "first_option_accuracy": 0.1875, "nll": 0.8097748523765361, "none_cases": 8, "none_precision": null, "none_recall": 0.0, "predicted_none": 0, "probabilities_calibrated": false, "supported_accuracy": 1.0, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2125 }, "evidence": { "accuracy": 0.8125, "always_none_accuracy": 0.25, "correct": 26, "correct_none": 7, "count": 32, "first_option_accuracy": 0.3125, "nll": 0.5390170451972376, "none_cases": 8, "none_precision": 0.875, "none_recall": 0.875, "predicted_none": 8, "probabilities_calibrated": false, "supported_accuracy": 0.7916666666666666, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23623511904761904 }, "language": { "accuracy": 0.96875, "always_none_accuracy": 0.25, "correct": 31, "correct_none": 8, "count": 32, "first_option_accuracy": 0.25, "nll": 0.17660762997674478, "none_cases": 8, "none_precision": 0.8888888888888888, "none_recall": 1.0, "predicted_none": 9, "probabilities_calibrated": false, "supported_accuracy": 0.9583333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23616071428571428 }, "language_evidence": { "accuracy": 0.65625, "always_none_accuracy": 0.25, "correct": 21, "correct_none": 3, "count": 32, "first_option_accuracy": 0.21875, "nll": 0.9488952381459622, "none_cases": 8, "none_precision": 0.6, "none_recall": 0.375, "predicted_none": 5, "probabilities_calibrated": false, "supported_accuracy": 0.75, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.23087797619047618 }, "lookup": { "accuracy": 0.96875, "always_none_accuracy": 0.25, "correct": 31, "correct_none": 7, "count": 32, "first_option_accuracy": 0.1875, "nll": 0.25926072011215384, "none_cases": 8, "none_precision": 1.0, "none_recall": 0.875, "predicted_none": 7, "probabilities_calibrated": false, "supported_accuracy": 1.0, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2200892857142857 }, "priority": { "accuracy": 0.75, "always_none_accuracy": 0.25, "correct": 24, "correct_none": 7, "count": 32, "first_option_accuracy": 0.21875, "nll": 0.8134824519480416, "none_cases": 8, "none_precision": 1.0, "none_recall": 0.875, "predicted_none": 7, "probabilities_calibrated": false, "supported_accuracy": 0.7083333333333334, "supported_cases": 24, "uniform_choice_expected_accuracy": 0.2409970238095238 }, "public_nli": { "accuracy": 0.3932291666666667, "always_none_accuracy": 0.25, "correct": 151, "correct_none": 62, "count": 384, "first_option_accuracy": 0.3151041666666667, "nll": 1.1078937428953253, "none_cases": 96, "none_precision": 0.4189189189189189, "none_recall": 0.6458333333333334, "predicted_none": 148, "probabilities_calibrated": false, "supported_accuracy": 0.3090277777777778, "supported_cases": 288, "uniform_choice_expected_accuracy": 0.3018663194444444 } }, "metrics": { "accuracy": 0.55859375, "always_none_accuracy": 0.25, "correct": 429, "correct_none": 127, "count": 768, "first_option_accuracy": 0.2721354166666667, "nll": 0.9447784224661161, "none_cases": 192, "none_precision": 0.5695067264573991, "none_recall": 0.6614583333333334, "predicted_none": 223, "probabilities_calibrated": false, "supported_accuracy": 0.5243055555555556, "supported_cases": 576, "uniform_choice_expected_accuracy": 0.26503596230158727 }, "robustness": { "rename": { "agreement_count": 32, "agreement_rate": 1.0, "complete": true, "distribution_agreement_count": 32, "distribution_agreement_rate": 1.0, "expected_count": 32, "rows": [ { "choice_original_id": "19d1cbc43aa6fc88", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:03548e607f76440158e61651", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-12e2ada3fb2f6228", "distribution": [ { "id": "opaque-bc7db2510f59d92c", "logit": 0.5482270121574402, "probability": 0.24394874274730682 }, { "id": "opaque-12e2ada3fb2f6228", "logit": 1.1469435691833496, "probability": 0.44393348693847656 }, { "id": "opaque-0401520b989e4ae3", "logit": 0.7749781012535095, "probability": 0.3060379922389984 }, { "id": null, "logit": -3.143768072128296, "probability": 0.006079748272895813 } ], "elapsed_seconds": 0.044820174982305616, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 110, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-0401520b989e4ae3" }, { "choice_original_id": "c0ecb839ba1e7566", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:65510055ee8da319", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-312cca3e7e9a712b", "distribution": [ { "id": "opaque-312cca3e7e9a712b", "logit": 1.134496808052063, "probability": 0.94630366563797 }, { "id": "opaque-8a5e8b808a3271bf", "logit": -3.987795352935791, "probability": 0.005642184987664223 }, { "id": null, "logit": -1.8457382917404175, "probability": 0.04805414378643036 } ], "elapsed_seconds": 0.04529006901429966, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 107, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-312cca3e7e9a712b" }, { "choice_original_id": "1a9d14479ab835cd", "distribution_exact": true, "family": "capability_priority", "id": "canonical-confirmation:direct:9e75a5299593dd59", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-591589b68baebe0b", "distribution": [ { "id": "opaque-591589b68baebe0b", "logit": -0.017771948128938675, "probability": 0.41383394598960876 }, { "id": "opaque-0118b16068da7b37", "logit": -1.146085500717163, "probability": 0.1339077651500702 }, { "id": "opaque-8b79fde3a0e1041f", "logit": -0.8601003289222717, "probability": 0.17824096977710724 }, { "id": "opaque-b38eafbb4b7148aa", "logit": -0.853622317314148, "probability": 0.17939937114715576 }, { "id": null, "logit": -1.4933903217315674, "probability": 0.09461788088083267 } ], "elapsed_seconds": 0.060451794997788966, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-0118b16068da7b37" }, { "choice_original_id": "25f8970e22f3c4f1", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:2815c175bf14eed80b84559b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-0edc2369a225d228", "distribution": [ { "id": "opaque-d409f4f7b4301854", "logit": -0.059720832854509354, "probability": 0.2551456689834595 }, { "id": "opaque-0edc2369a225d228", "logit": 0.3453811705112457, "probability": 0.3825795650482178 }, { "id": null, "logit": 0.2908475399017334, "probability": 0.36227479577064514 } ], "elapsed_seconds": 0.04445729500730522, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 109, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-d409f4f7b4301854" }, { "choice_original_id": "fe8d0c5b94a69c0e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:c80d3546c430419f2cf9b772", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-baeddc43d62a5584", "distribution": [ { "id": "opaque-baeddc43d62a5584", "logit": 1.2180156707763672, "probability": 0.451945424079895 }, { "id": "opaque-901035a3aa7ad27b", "logit": 0.19202689826488495, "probability": 0.16199615597724915 }, { "id": null, "logit": 1.0604428052902222, "probability": 0.38605836033821106 } ], "elapsed_seconds": 0.0376892130007036, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "94e4c20de1984ebd", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1b279ebfa2f80fd339e274a9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-e2f9d4122fe1169f", "distribution": [ { "id": "opaque-e2f9d4122fe1169f", "logit": 1.1097180843353271, "probability": 0.5585915446281433 }, { "id": "opaque-eab7750e7bc7e6d1", "logit": -0.028186241164803505, "probability": 0.17902298271656036 }, { "id": "opaque-7b56d045ac525f45", "logit": 0.2942873537540436, "probability": 0.24714809656143188 }, { "id": null, "logit": -2.4919488430023193, "probability": 0.015237378887832165 } ], "elapsed_seconds": 0.04988542900537141, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 145, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-eab7750e7bc7e6d1" }, { "choice_original_id": null, "distribution_exact": true, "family": "priority", "id": "canonical-confirmation:direct:03316fb9e789b103", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-b0b2e6dfa3cc5b35", "logit": -3.8692755699157715, "probability": 0.04983150586485863 }, { "id": "opaque-1b2ffeace2cfd5e6", "logit": -3.1114516258239746, "probability": 0.10632190853357315 }, { "id": null, "logit": -1.0399521589279175, "probability": 0.8438465595245361 } ], "elapsed_seconds": 0.038476118992548436, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "8a108f66b2d6f2a8", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:dcb34c581cba491997e1effd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-0bdb96ba82e959a6", "distribution": [ { "id": "opaque-0bdb96ba82e959a6", "logit": 1.7972785234451294, "probability": 0.5732753276824951 }, { "id": "opaque-e5b8b40969f05e2c", "logit": 0.27642491459846497, "probability": 0.1252751499414444 }, { "id": "opaque-7f1d1077ef912a8d", "logit": 1.1091783046722412, "probability": 0.28808796405792236 }, { "id": null, "logit": -1.9617094993591309, "probability": 0.013361510820686817 } ], "elapsed_seconds": 0.04485145301441662, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 124, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-0bdb96ba82e959a6" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:e623e0ccf963a9f3", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-fa4e88dfff02e984", "logit": -2.629603385925293, "probability": 0.05061854049563408 }, { "id": "opaque-a10c6645f8d4cc7c", "logit": -1.7517799139022827, "probability": 0.12177089601755142 }, { "id": "opaque-dd45d30fd7c2f12f", "logit": -2.0539231300354004, "probability": 0.09001696854829788 }, { "id": "opaque-91c9f1e920776777", "logit": -2.8269124031066895, "probability": 0.04155462607741356 }, { "id": "opaque-53e80da74632d0e9", "logit": -1.713044285774231, "probability": 0.12658031284809113 }, { "id": "opaque-5ed0964c7c04217e", "logit": -2.877108097076416, "probability": 0.039520252496004105 }, { "id": null, "logit": -0.2811605632305145, "probability": 0.5299383401870728 } ], "elapsed_seconds": 0.06677952100289986, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 200, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:7dc843eca0200afe90992317", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-78e4219772dc6fd9", "logit": 0.6711658835411072, "probability": 0.24482934176921844 }, { "id": "opaque-68a9282e226bbbb1", "logit": 0.9146880507469177, "probability": 0.3123372495174408 }, { "id": null, "logit": 1.2637982368469238, "probability": 0.44283345341682434 } ], "elapsed_seconds": 0.04419666400644928, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 99, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-78e4219772dc6fd9" }, { "choice_original_id": "9807e64c4702e584", "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:ed140718172a188d", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-bfa435be381fade2", "distribution": [ { "id": "opaque-c6fac6893f59a196", "logit": -0.8412978649139404, "probability": 0.04915045574307442 }, { "id": "opaque-6f5ddeba8caeff07", "logit": -0.09964147210121155, "probability": 0.1031869500875473 }, { "id": "opaque-deedc594dcf1e513", "logit": 0.9851800799369812, "probability": 0.30532106757164 }, { "id": "opaque-bfa435be381fade2", "logit": 0.997035562992096, "probability": 0.3089623749256134 }, { "id": "opaque-761f6d89b3718efa", "logit": 0.2598293125629425, "probability": 0.14782264828681946 }, { "id": null, "logit": -0.28700658679008484, "probability": 0.0855565294623375 } ], "elapsed_seconds": 0.04937776501174085, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 147, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-bfa435be381fade2" }, { "choice_original_id": "0ba2cad0d646a4ed", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:225545e13b196dc9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-71c4f97f3359e493", "distribution": [ { "id": "opaque-b6999fd7a9434040", "logit": -0.7837391495704651, "probability": 0.05343678966164589 }, { "id": "opaque-48ee4bd29b089264", "logit": -0.1706295609474182, "probability": 0.09865304082632065 }, { "id": "opaque-71c4f97f3359e493", "logit": 1.925339937210083, "probability": 0.8023769855499268 }, { "id": null, "logit": -0.9437956213951111, "probability": 0.045533254742622375 } ], "elapsed_seconds": 0.049202269001398236, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-71c4f97f3359e493" }, { "choice_original_id": "4475e3e948fcb9a4", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:cbbfa8df77bd4dfce682b0bb", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-cb28a50cfc974c06", "distribution": [ { "id": "opaque-d90f3865e3bf1cab", "logit": 0.7425800561904907, "probability": 0.3370814025402069 }, { "id": "opaque-cb28a50cfc974c06", "logit": 0.9781201481819153, "probability": 0.42660754919052124 }, { "id": "opaque-7e49720fef4af5a9", "logit": 0.3218015134334564, "probability": 0.22130587697029114 }, { "id": null, "logit": -2.3693556785583496, "probability": 0.0150050800293684 } ], "elapsed_seconds": 0.0454229160095565, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-cb28a50cfc974c06" }, { "choice_original_id": "7e8f5ea9a3659a4e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:74fc8ce91192c581766967f0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-4880362ed73fd795", "distribution": [ { "id": "opaque-40c85e474d616664", "logit": 0.9532279372215271, "probability": 0.31694546341896057 }, { "id": "opaque-4880362ed73fd795", "logit": 1.1828924417495728, "probability": 0.39877375960350037 }, { "id": "opaque-48713e52aedd84e9", "logit": 0.6873573064804077, "probability": 0.24295082688331604 }, { "id": null, "logit": -1.08391535282135, "probability": 0.04132990911602974 } ], "elapsed_seconds": 0.04576046598958783, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 128, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-48713e52aedd84e9" }, { "choice_original_id": "39389fe9ba71711c", "distribution_exact": true, "family": "category", "id": "canonical-confirmation:direct:18052382cdc27ffa", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-82950d984f7f1a48", "distribution": [ { "id": "opaque-1b954a60876cac8c", "logit": -1.704192042350769, "probability": 0.008141100406646729 }, { "id": "opaque-2c9af7535da1fd9e", "logit": -0.9241003394126892, "probability": 0.017761213704943657 }, { "id": "opaque-82950d984f7f1a48", "logit": 3.055756092071533, "probability": 0.9503911137580872 }, { "id": "opaque-9c25d2421375c022", "logit": -2.7516138553619385, "probability": 0.0028562324587255716 }, { "id": null, "logit": -0.763746976852417, "probability": 0.020850345492362976 } ], "elapsed_seconds": 0.04515130398795009, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 126, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-82950d984f7f1a48" }, { "choice_original_id": "e220c9996bcc3aa9", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:cd7288ebebd6ebd0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-200eb6ef1f673492", "distribution": [ { "id": "opaque-4bd49ae09586d288", "logit": -0.9881320595741272, "probability": 0.015324955806136131 }, { "id": "opaque-b40e0e7d42ce3985", "logit": -1.4890239238739014, "probability": 0.009286770597100258 }, { "id": "opaque-200eb6ef1f673492", "logit": 3.150034189224243, "probability": 0.9606872797012329 }, { "id": null, "logit": -1.0297013521194458, "probability": 0.014700966887176037 } ], "elapsed_seconds": 0.049447972007328644, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-200eb6ef1f673492" }, { "choice_original_id": "2ea4e55981623071", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1036efba79c0542aaeef3c97", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-568cd16b291b5816", "distribution": [ { "id": "opaque-568cd16b291b5816", "logit": 1.3696633577346802, "probability": 0.4656517505645752 }, { "id": "opaque-bc46a5e962ce8d1d", "logit": 0.966858983039856, "probability": 0.3112615644931793 }, { "id": null, "logit": 0.6337862014770508, "probability": 0.22308677434921265 } ], "elapsed_seconds": 0.04436979899764992, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 97, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-568cd16b291b5816" }, { "choice_original_id": "bb24192e2625edf1", "distribution_exact": true, "family": "evidence", "id": "canonical-confirmation:direct:089b2000bb7a8d02", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-059e9ce177f12279", "distribution": [ { "id": "opaque-059e9ce177f12279", "logit": 2.099900245666504, "probability": 0.9447422027587891 }, { "id": "opaque-26dcf8c14de27430", "logit": -3.228045701980591, "probability": 0.004585808143019676 }, { "id": null, "logit": -0.8256396651268005, "probability": 0.05067192763090134 } ], "elapsed_seconds": 0.03802426301990636, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 88, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-059e9ce177f12279" }, { "choice_original_id": "e9bd53c9f5d94f15", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:02719b1656991c1243d0cdb5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-db222ea100d3e7b3", "distribution": [ { "id": "opaque-e58b44b85027071d", "logit": 0.6245986819267273, "probability": 0.20551122725009918 }, { "id": "opaque-dd16462f056edd3f", "logit": 0.9769009947776794, "probability": 0.292306512594223 }, { "id": "opaque-db222ea100d3e7b3", "logit": 1.4829150438308716, "probability": 0.4848390817642212 }, { "id": null, "logit": -1.847699761390686, "probability": 0.017343228682875633 } ], "elapsed_seconds": 0.04399015998933464, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-dd16462f056edd3f" }, { "choice_original_id": "f005e77e2222a8de", "distribution_exact": true, "family": "language_evidence", "id": "canonical-confirmation:direct:dac514809d6319dc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-a228de578e869ce4", "distribution": [ { "id": "opaque-466b29b2a1aa4a1d", "logit": -2.24950909614563, "probability": 0.00719729857519269 }, { "id": "opaque-46b5b26e21d86f46", "logit": -0.37323611974716187, "probability": 0.04699205979704857 }, { "id": "opaque-a228de578e869ce4", "logit": 2.3792879581451416, "probability": 0.7369372844696045 }, { "id": "opaque-f8bc0ce4e2d0a295", "logit": -2.09604811668396, "probability": 0.008391059003770351 }, { "id": "opaque-ee3cf91d6a58e07c", "logit": 0.9112727642059326, "probability": 0.1697773039340973 }, { "id": null, "logit": -0.7987887859344482, "probability": 0.03070501610636711 } ], "elapsed_seconds": 0.05903977801790461, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-a228de578e869ce4" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:8a2c005ecf0095bd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-4ddd09ee01a12d17", "logit": -1.5642991065979004, "probability": 0.21718795597553253 }, { "id": "opaque-b2cbff940f0b10cc", "logit": -3.42099928855896, "probability": 0.033921971917152405 }, { "id": "opaque-bbdca15a4c1ee7ba", "logit": -3.377971887588501, "probability": 0.03541339933872223 }, { "id": "opaque-176918f028afffc1", "logit": -3.3638076782226562, "probability": 0.035918571054935455 }, { "id": null, "logit": -0.4265669286251068, "probability": 0.6775581240653992 } ], "elapsed_seconds": 0.05109637198620476, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 155, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "7007f74c373dab78", "distribution_exact": true, "family": "capabilities", "id": "canonical-confirmation:direct:17f6ca9ce2782904", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-e4fb1eb496fbfc70", "distribution": [ { "id": "opaque-e4fb1eb496fbfc70", "logit": 0.7908933758735657, "probability": 0.7937299013137817 }, { "id": "opaque-7ff3201335aec9d6", "logit": -2.7851645946502686, "probability": 0.022213168442249298 }, { "id": "opaque-71ec2a50dfc414be", "logit": -4.44999885559082, "probability": 0.004203219432383776 }, { "id": null, "logit": -0.6937057375907898, "probability": 0.17985376715660095 } ], "elapsed_seconds": 0.04996621600002982, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 129, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-e4fb1eb496fbfc70" }, { "choice_original_id": "d711e3472cc27285", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:9ede679f928684b3dafb6313", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-0ffcaca59073e77d", "distribution": [ { "id": "opaque-0ffcaca59073e77d", "logit": 1.6177133321762085, "probability": 0.47902461886405945 }, { "id": "opaque-9c15e3e0e96dcdae", "logit": 0.3463936448097229, "probability": 0.1343478560447693 }, { "id": null, "logit": 1.4034231901168823, "probability": 0.38662755489349365 } ], "elapsed_seconds": 0.038234806997934356, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 90, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-9c15e3e0e96dcdae" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:3301caf361ab71a8", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-d7c6e35c384e1aa4", "logit": -3.6370575428009033, "probability": 0.03312469646334648 }, { "id": "opaque-e71a5d9703cc68e1", "logit": -4.030068874359131, "probability": 0.02235987037420273 }, { "id": "opaque-0326b3888078ce6d", "logit": -3.077439069747925, "probability": 0.0579683743417263 }, { "id": "opaque-209fdc568fa474aa", "logit": -1.3930315971374512, "probability": 0.312406450510025 }, { "id": null, "logit": -0.7844625115394592, "probability": 0.5741405487060547 } ], "elapsed_seconds": 0.04922223900211975, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 156, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "2b58ef5bf973a8d6", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:eefdcea7cdfa3c6365491577", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-52ea141023981f0d", "distribution": [ { "id": "opaque-4f51f76028ab9213", "logit": 0.2898178696632385, "probability": 0.20393863320350647 }, { "id": "opaque-52ea141023981f0d", "logit": 1.1809521913528442, "probability": 0.49718064069747925 }, { "id": null, "logit": 0.6720435619354248, "probability": 0.29888078570365906 } ], "elapsed_seconds": 0.04479959502350539, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-52ea141023981f0d" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:a7d25ab08a792cf8f3c50b3b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-bbb5d408c4aa2861", "logit": -0.10576401650905609, "probability": 0.13574162125587463 }, { "id": "opaque-8aff54da3a90c870", "logit": 9.685754776000977e-08, "probability": 0.15088492631912231 }, { "id": null, "logit": 1.553487777709961, "probability": 0.7133734822273254 } ], "elapsed_seconds": 0.04889593899133615, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 131, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:8f4e7e3049890865642f13fc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-88469ccdea00d36d", "logit": 0.8433347344398499, "probability": 0.34871968626976013 }, { "id": "opaque-216404ed15307e9f", "logit": 0.5243914723396301, "probability": 0.25349023938179016 }, { "id": null, "logit": 0.9749905467033386, "probability": 0.39779001474380493 } ], "elapsed_seconds": 0.03782691899687052, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "43cdd07a93440816", "distribution_exact": true, "family": "budget", "id": "canonical-confirmation:direct:63849bf99697604b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-c94fd650b55fe9df", "distribution": [ { "id": "opaque-2c723b933134ab59", "logit": -2.1269733905792236, "probability": 0.18641752004623413 }, { "id": "opaque-c94fd650b55fe9df", "logit": -1.104489803314209, "probability": 0.5182576179504395 }, { "id": null, "logit": -1.666886329650879, "probability": 0.2953248620033264 } ], "elapsed_seconds": 0.037667324009817094, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-2c723b933134ab59" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:07ae8d2974d5d511eea3f9b5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "opaque-15761e529b538ddb", "logit": 0.49326327443122864, "probability": 0.23063309490680695 }, { "id": "opaque-3b32cbf759f4a179", "logit": 0.5970414876937866, "probability": 0.2558538317680359 }, { "id": null, "logit": 1.2937105894088745, "probability": 0.5135130286216736 } ], "elapsed_seconds": 0.0437299869954586, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 106, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-15761e529b538ddb" }, { "choice_original_id": "5c24f96f4b3139c8", "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:37b8cda8ac2c8294", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-f6f7521f46fb9a9e", "distribution": [ { "id": "opaque-f6f7521f46fb9a9e", "logit": 0.20911633968353271, "probability": 0.6829322576522827 }, { "id": "opaque-3db3c9e49993d651", "logit": -1.2452515363693237, "probability": 0.1594974398612976 }, { "id": null, "logit": -1.257407784461975, "probability": 0.1575702726840973 } ], "elapsed_seconds": 0.03777055101818405, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "e71dd00b24c510d4", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:db97eb1a3f6e85d65712bc56", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-9cde2e8e22863a2a", "distribution": [ { "id": "opaque-77f7d707ec80e453", "logit": 0.24724352359771729, "probability": 0.1661594659090042 }, { "id": "opaque-9cde2e8e22863a2a", "logit": 1.2616242170333862, "probability": 0.4582104980945587 }, { "id": null, "logit": 1.062900185585022, "probability": 0.3756300210952759 } ], "elapsed_seconds": 0.04470216800109483, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "99632bb0863c7081", "distribution_exact": true, "family": "lookup", "id": "canonical-confirmation:direct:fa8c40fd9550a870", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "opaque-fe22cd863ec7b9a6", "distribution": [ { "id": "opaque-de7840d60997c933", "logit": -2.6931681632995605, "probability": 0.004461521282792091 }, { "id": "opaque-74d80858f70d7e32", "logit": -3.0006179809570312, "probability": 0.003280642908066511 }, { "id": "opaque-059367613e9a098f", "logit": -2.1361916065216064, "probability": 0.0077870809473097324 }, { "id": "opaque-fe22cd863ec7b9a6", "logit": 2.6531569957733154, "probability": 0.9361864924430847 }, { "id": "opaque-0fb9efcc88be25a6", "logit": -0.7179722785949707, "probability": 0.03215877711772919 }, { "id": "opaque-346165b257a85c7d", "logit": -2.4057323932647705, "probability": 0.005947228521108627 }, { "id": null, "logit": -1.868399739265442, "probability": 0.010178297758102417 } ], "elapsed_seconds": 0.059502474003238603, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 165, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "opaque-fe22cd863ec7b9a6" } ], "sample_count": 32 }, "reverse": { "agreement_count": 32, "agreement_rate": 1.0, "complete": true, "distribution_agreement_count": 32, "distribution_agreement_rate": 1.0, "expected_count": 32, "rows": [ { "choice_original_id": "19d1cbc43aa6fc88", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:03548e607f76440158e61651", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "19d1cbc43aa6fc88", "distribution": [ { "id": "b75ffb314e5d5487", "logit": 0.7749781012535095, "probability": 0.3060379922389984 }, { "id": "19d1cbc43aa6fc88", "logit": 1.1469435691833496, "probability": 0.44393348693847656 }, { "id": "efb6a822718dc8c7", "logit": 0.5482270121574402, "probability": 0.24394874274730682 }, { "id": null, "logit": -3.143768072128296, "probability": 0.006079748272895813 } ], "elapsed_seconds": 0.044313449994660914, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 110, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "b75ffb314e5d5487" }, { "choice_original_id": "c0ecb839ba1e7566", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:65510055ee8da319", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "c0ecb839ba1e7566", "distribution": [ { "id": "6f5277e14a0f433b", "logit": -3.987795352935791, "probability": 0.005642184987664223 }, { "id": "c0ecb839ba1e7566", "logit": 1.134496808052063, "probability": 0.94630366563797 }, { "id": null, "logit": -1.8457382917404175, "probability": 0.04805414378643036 } ], "elapsed_seconds": 0.04526530098519288, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 107, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "c0ecb839ba1e7566" }, { "choice_original_id": "1a9d14479ab835cd", "distribution_exact": true, "family": "capability_priority", "id": "canonical-confirmation:direct:9e75a5299593dd59", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "1a9d14479ab835cd", "distribution": [ { "id": "7112a6332052b9ff", "logit": -0.853622317314148, "probability": 0.17939937114715576 }, { "id": "070ba78983441ec4", "logit": -0.8601003289222717, "probability": 0.17824096977710724 }, { "id": "4ce69769781021fc", "logit": -1.146085500717163, "probability": 0.1339077651500702 }, { "id": "1a9d14479ab835cd", "logit": -0.017771948128938675, "probability": 0.41383394598960876 }, { "id": null, "logit": -1.4933903217315674, "probability": 0.09461788088083267 } ], "elapsed_seconds": 0.05980014498345554, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "4ce69769781021fc" }, { "choice_original_id": "25f8970e22f3c4f1", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:2815c175bf14eed80b84559b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "25f8970e22f3c4f1", "distribution": [ { "id": "25f8970e22f3c4f1", "logit": 0.3453811705112457, "probability": 0.3825795650482178 }, { "id": "bd557427fdc08978", "logit": -0.059720832854509354, "probability": 0.2551456689834595 }, { "id": null, "logit": 0.2908475399017334, "probability": 0.36227479577064514 } ], "elapsed_seconds": 0.04456410199054517, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 109, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "bd557427fdc08978" }, { "choice_original_id": "fe8d0c5b94a69c0e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:c80d3546c430419f2cf9b772", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "fe8d0c5b94a69c0e", "distribution": [ { "id": "d1909d28a04b6855", "logit": 0.19202689826488495, "probability": 0.16199615597724915 }, { "id": "fe8d0c5b94a69c0e", "logit": 1.2180156707763672, "probability": 0.451945424079895 }, { "id": null, "logit": 1.0604428052902222, "probability": 0.38605836033821106 } ], "elapsed_seconds": 0.03730673500103876, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "94e4c20de1984ebd", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1b279ebfa2f80fd339e274a9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "94e4c20de1984ebd", "distribution": [ { "id": "6f3e6021727694dc", "logit": 0.2942873537540436, "probability": 0.24714809656143188 }, { "id": "275b2c19296206a4", "logit": -0.028186241164803505, "probability": 0.17902298271656036 }, { "id": "94e4c20de1984ebd", "logit": 1.1097180843353271, "probability": 0.5585915446281433 }, { "id": null, "logit": -2.4919488430023193, "probability": 0.015237378887832165 } ], "elapsed_seconds": 0.049634837021585554, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 145, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "275b2c19296206a4" }, { "choice_original_id": null, "distribution_exact": true, "family": "priority", "id": "canonical-confirmation:direct:03316fb9e789b103", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "37667e37345c013e", "logit": -3.1114516258239746, "probability": 0.10632190853357315 }, { "id": "95d6e9a6489b7f20", "logit": -3.8692755699157715, "probability": 0.04983150586485863 }, { "id": null, "logit": -1.0399521589279175, "probability": 0.8438465595245361 } ], "elapsed_seconds": 0.03774679100024514, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 96, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "8a108f66b2d6f2a8", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:dcb34c581cba491997e1effd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "8a108f66b2d6f2a8", "distribution": [ { "id": "8826fa788322d575", "logit": 1.1091783046722412, "probability": 0.28808796405792236 }, { "id": "9b64120262f697b7", "logit": 0.27642491459846497, "probability": 0.1252751499414444 }, { "id": "8a108f66b2d6f2a8", "logit": 1.7972785234451294, "probability": 0.5732753276824951 }, { "id": null, "logit": -1.9617094993591309, "probability": 0.013361510820686817 } ], "elapsed_seconds": 0.04505197799880989, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 124, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "8a108f66b2d6f2a8" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:e623e0ccf963a9f3", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "7582cba3a4f1c900", "logit": -2.877108097076416, "probability": 0.039520252496004105 }, { "id": "e39430e598907458", "logit": -1.713044285774231, "probability": 0.12658031284809113 }, { "id": "d096db4a72da038a", "logit": -2.8269124031066895, "probability": 0.04155462607741356 }, { "id": "63bb7cf0107eed47", "logit": -2.0539231300354004, "probability": 0.09001696854829788 }, { "id": "334a8592ce63e429", "logit": -1.7517799139022827, "probability": 0.12177089601755142 }, { "id": "359c953097ca7fef", "logit": -2.629603385925293, "probability": 0.05061854049563408 }, { "id": null, "logit": -0.2811605632305145, "probability": 0.5299383401870728 } ], "elapsed_seconds": 0.06661205599084496, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 200, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:7dc843eca0200afe90992317", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "5b795f5258fbf396", "logit": 0.9146880507469177, "probability": 0.3123372495174408 }, { "id": "96ba6910d8949928", "logit": 0.6711658835411072, "probability": 0.24482934176921844 }, { "id": null, "logit": 1.2637982368469238, "probability": 0.44283345341682434 } ], "elapsed_seconds": 0.04516380399581976, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 99, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "96ba6910d8949928" }, { "choice_original_id": "9807e64c4702e584", "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:ed140718172a188d", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "9807e64c4702e584", "distribution": [ { "id": "095c0c9eee4d98a9", "logit": 0.2598293125629425, "probability": 0.14782264828681946 }, { "id": "9807e64c4702e584", "logit": 0.997035562992096, "probability": 0.3089623749256134 }, { "id": "770646ae20f726b6", "logit": 0.9851800799369812, "probability": 0.30532106757164 }, { "id": "9a7fc9871af27760", "logit": -0.09964147210121155, "probability": 0.1031869500875473 }, { "id": "5d1acb45a8c8b604", "logit": -0.8412978649139404, "probability": 0.04915045574307442 }, { "id": null, "logit": -0.28700658679008484, "probability": 0.0855565294623375 } ], "elapsed_seconds": 0.04968692499096505, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 147, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "9807e64c4702e584" }, { "choice_original_id": "0ba2cad0d646a4ed", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:225545e13b196dc9", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "0ba2cad0d646a4ed", "distribution": [ { "id": "0ba2cad0d646a4ed", "logit": 1.925339937210083, "probability": 0.8023769855499268 }, { "id": "c79332bdc25aa21d", "logit": -0.1706295609474182, "probability": 0.09865304082632065 }, { "id": "d570197d7fce642c", "logit": -0.7837391495704651, "probability": 0.05343678966164589 }, { "id": null, "logit": -0.9437956213951111, "probability": 0.045533254742622375 } ], "elapsed_seconds": 0.05011655201087706, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "0ba2cad0d646a4ed" }, { "choice_original_id": "4475e3e948fcb9a4", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:cbbfa8df77bd4dfce682b0bb", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "4475e3e948fcb9a4", "distribution": [ { "id": "7a199bacc2fcdb63", "logit": 0.3218015134334564, "probability": 0.22130587697029114 }, { "id": "4475e3e948fcb9a4", "logit": 0.9781201481819153, "probability": 0.42660754919052124 }, { "id": "41458bcc2739bf4c", "logit": 0.7425800561904907, "probability": 0.3370814025402069 }, { "id": null, "logit": -2.3693556785583496, "probability": 0.0150050800293684 } ], "elapsed_seconds": 0.04468537899083458, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "4475e3e948fcb9a4" }, { "choice_original_id": "7e8f5ea9a3659a4e", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:74fc8ce91192c581766967f0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "7e8f5ea9a3659a4e", "distribution": [ { "id": "b44f372f2b75ca2e", "logit": 0.6873573064804077, "probability": 0.24295082688331604 }, { "id": "7e8f5ea9a3659a4e", "logit": 1.1828924417495728, "probability": 0.39877375960350037 }, { "id": "ea0a5389c32ad8cd", "logit": 0.9532279372215271, "probability": 0.31694546341896057 }, { "id": null, "logit": -1.08391535282135, "probability": 0.04132990911602974 } ], "elapsed_seconds": 0.04449508400284685, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 128, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "b44f372f2b75ca2e" }, { "choice_original_id": "39389fe9ba71711c", "distribution_exact": true, "family": "category", "id": "canonical-confirmation:direct:18052382cdc27ffa", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "39389fe9ba71711c", "distribution": [ { "id": "38b4ce10d19bb6b3", "logit": -2.7516138553619385, "probability": 0.0028562324587255716 }, { "id": "39389fe9ba71711c", "logit": 3.055756092071533, "probability": 0.9503911137580872 }, { "id": "4c1149263772b1a0", "logit": -0.9241003394126892, "probability": 0.017761213704943657 }, { "id": "f017bbe9d3a47fd9", "logit": -1.704192042350769, "probability": 0.008141100406646729 }, { "id": null, "logit": -0.763746976852417, "probability": 0.020850345492362976 } ], "elapsed_seconds": 0.04538868702366017, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 126, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "39389fe9ba71711c" }, { "choice_original_id": "e220c9996bcc3aa9", "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:cd7288ebebd6ebd0", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "e220c9996bcc3aa9", "distribution": [ { "id": "e220c9996bcc3aa9", "logit": 3.150034189224243, "probability": 0.9606872797012329 }, { "id": "106f6f3f671d090a", "logit": -1.4890239238739014, "probability": 0.009286770597100258 }, { "id": "85f53f020ea99380", "logit": -0.9881320595741272, "probability": 0.015324955806136131 }, { "id": null, "logit": -1.0297013521194458, "probability": 0.014700966887176037 } ], "elapsed_seconds": 0.049401944008423015, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 135, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "e220c9996bcc3aa9" }, { "choice_original_id": "2ea4e55981623071", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:1036efba79c0542aaeef3c97", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "2ea4e55981623071", "distribution": [ { "id": "04ce2823d078e021", "logit": 0.966858983039856, "probability": 0.3112615644931793 }, { "id": "2ea4e55981623071", "logit": 1.3696633577346802, "probability": 0.4656517505645752 }, { "id": null, "logit": 0.6337862014770508, "probability": 0.22308677434921265 } ], "elapsed_seconds": 0.0444925140182022, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 97, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "2ea4e55981623071" }, { "choice_original_id": "bb24192e2625edf1", "distribution_exact": true, "family": "evidence", "id": "canonical-confirmation:direct:089b2000bb7a8d02", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "bb24192e2625edf1", "distribution": [ { "id": "c1a807174cbb3ecb", "logit": -3.228045701980591, "probability": 0.004585808143019676 }, { "id": "bb24192e2625edf1", "logit": 2.099900245666504, "probability": 0.9447422027587891 }, { "id": null, "logit": -0.8256396651268005, "probability": 0.05067192763090134 } ], "elapsed_seconds": 0.03753475798293948, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 88, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "bb24192e2625edf1" }, { "choice_original_id": "e9bd53c9f5d94f15", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:02719b1656991c1243d0cdb5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "e9bd53c9f5d94f15", "distribution": [ { "id": "e9bd53c9f5d94f15", "logit": 1.4829150438308716, "probability": 0.4848390817642212 }, { "id": "fb9e7dcfe00f7f16", "logit": 0.9769009947776794, "probability": 0.292306512594223 }, { "id": "b506f558303241ec", "logit": 0.6245986819267273, "probability": 0.20551122725009918 }, { "id": null, "logit": -1.847699761390686, "probability": 0.017343228682875633 } ], "elapsed_seconds": 0.04448285498074256, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "fb9e7dcfe00f7f16" }, { "choice_original_id": "f005e77e2222a8de", "distribution_exact": true, "family": "language_evidence", "id": "canonical-confirmation:direct:dac514809d6319dc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "f005e77e2222a8de", "distribution": [ { "id": "c3bd6af6729ded16", "logit": 0.9112727642059326, "probability": 0.1697773039340973 }, { "id": "072d339d9818dba9", "logit": -2.09604811668396, "probability": 0.008391059003770351 }, { "id": "f005e77e2222a8de", "logit": 2.3792879581451416, "probability": 0.7369372844696045 }, { "id": "68a9f9da2e63732d", "logit": -0.37323611974716187, "probability": 0.04699205979704857 }, { "id": "c56dc92b7e227c45", "logit": -2.24950909614563, "probability": 0.00719729857519269 }, { "id": null, "logit": -0.7987887859344482, "probability": 0.03070501610636711 } ], "elapsed_seconds": 0.05976810600259341, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 170, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "f005e77e2222a8de" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:8a2c005ecf0095bd", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "28d3798e2ff35c4c", "logit": -3.3638076782226562, "probability": 0.035918571054935455 }, { "id": "8c62e5a48f920b9c", "logit": -3.377971887588501, "probability": 0.03541339933872223 }, { "id": "2f25ba50cfedc1d0", "logit": -3.42099928855896, "probability": 0.033921971917152405 }, { "id": "8e246d8efbfecde7", "logit": -1.5642991065979004, "probability": 0.21718795597553253 }, { "id": null, "logit": -0.4265669286251068, "probability": 0.6775581240653992 } ], "elapsed_seconds": 0.05100433499319479, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 155, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "7007f74c373dab78", "distribution_exact": true, "family": "capabilities", "id": "canonical-confirmation:direct:17f6ca9ce2782904", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "7007f74c373dab78", "distribution": [ { "id": "29766f5af26f4ce7", "logit": -4.44999885559082, "probability": 0.004203219432383776 }, { "id": "6cbc20b346ef289d", "logit": -2.7851645946502686, "probability": 0.022213168442249298 }, { "id": "7007f74c373dab78", "logit": 0.7908933758735657, "probability": 0.7937299013137817 }, { "id": null, "logit": -0.6937057375907898, "probability": 0.17985376715660095 } ], "elapsed_seconds": 0.04976528199040331, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 129, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "7007f74c373dab78" }, { "choice_original_id": "d711e3472cc27285", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:9ede679f928684b3dafb6313", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "d711e3472cc27285", "distribution": [ { "id": "ad5e5ffec891418d", "logit": 0.3463936448097229, "probability": 0.1343478560447693 }, { "id": "d711e3472cc27285", "logit": 1.6177133321762085, "probability": 0.47902461886405945 }, { "id": null, "logit": 1.4034231901168823, "probability": 0.38662755489349365 } ], "elapsed_seconds": 0.03827302498393692, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 90, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "ad5e5ffec891418d" }, { "choice_original_id": null, "distribution_exact": true, "family": "category_deadline", "id": "canonical-confirmation:direct:3301caf361ab71a8", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "e483be92d7bd7cfb", "logit": -1.3930315971374512, "probability": 0.312406450510025 }, { "id": "8518116bda018422", "logit": -3.077439069747925, "probability": 0.0579683743417263 }, { "id": "5a382b14a36236e5", "logit": -4.030068874359131, "probability": 0.02235987037420273 }, { "id": "1d39ccd03a91cfac", "logit": -3.6370575428009033, "probability": 0.03312469646334648 }, { "id": null, "logit": -0.7844625115394592, "probability": 0.5741405487060547 } ], "elapsed_seconds": 0.049890519003383815, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 156, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "2b58ef5bf973a8d6", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:eefdcea7cdfa3c6365491577", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "2b58ef5bf973a8d6", "distribution": [ { "id": "2b58ef5bf973a8d6", "logit": 1.1809521913528442, "probability": 0.49718064069747925 }, { "id": "c35770eec265bb9e", "logit": 0.2898178696632385, "probability": 0.20393863320350647 }, { "id": null, "logit": 0.6720435619354248, "probability": 0.29888078570365906 } ], "elapsed_seconds": 0.04511833601281978, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 111, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "2b58ef5bf973a8d6" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:a7d25ab08a792cf8f3c50b3b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "6b8e4c0ad12e0e1c", "logit": 9.685754776000977e-08, "probability": 0.15088492631912231 }, { "id": "4a51c4ddc32d5eef", "logit": -0.10576401650905609, "probability": 0.13574162125587463 }, { "id": null, "logit": 1.553487777709961, "probability": 0.7133734822273254 } ], "elapsed_seconds": 0.04847174198948778, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 131, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:8f4e7e3049890865642f13fc", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "1e573c04b67ea3f2", "logit": 0.5243914723396301, "probability": 0.25349023938179016 }, { "id": "f3bff744ef6784ce", "logit": 0.8433347344398499, "probability": 0.34871968626976013 }, { "id": null, "logit": 0.9749905467033386, "probability": 0.39779001474380493 } ], "elapsed_seconds": 0.0380933310079854, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "43cdd07a93440816", "distribution_exact": true, "family": "budget", "id": "canonical-confirmation:direct:63849bf99697604b", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "43cdd07a93440816", "distribution": [ { "id": "43cdd07a93440816", "logit": -1.104489803314209, "probability": 0.5182576179504395 }, { "id": "ad42d6a6c953882f", "logit": -2.1269733905792236, "probability": 0.18641752004623413 }, { "id": null, "logit": -1.666886329650879, "probability": 0.2953248620033264 } ], "elapsed_seconds": 0.03768659400520846, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "ad42d6a6c953882f" }, { "choice_original_id": null, "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:07ae8d2974d5d511eea3f9b5", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": null, "distribution": [ { "id": "9d4bc8c6f1ff3ffb", "logit": 0.5970414876937866, "probability": 0.2558538317680359 }, { "id": "4c9fca103c5dea48", "logit": 0.49326327443122864, "probability": 0.23063309490680695 }, { "id": null, "logit": 1.2937105894088745, "probability": 0.5135130286216736 } ], "elapsed_seconds": 0.04456162202404812, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": true, "token_count": 106, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "4c9fca103c5dea48" }, { "choice_original_id": "5c24f96f4b3139c8", "distribution_exact": true, "family": "deadline", "id": "canonical-confirmation:direct:37b8cda8ac2c8294", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "5c24f96f4b3139c8", "distribution": [ { "id": "3b4a74c6a0f44fbd", "logit": -1.2452515363693237, "probability": 0.1594974398612976 }, { "id": "5c24f96f4b3139c8", "logit": 0.20911633968353271, "probability": 0.6829322576522827 }, { "id": null, "logit": -1.257407784461975, "probability": 0.1575702726840973 } ], "elapsed_seconds": 0.03765370399923995, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 93, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "e71dd00b24c510d4", "distribution_exact": true, "family": "public_nli", "id": "canonical-confirmation:db97eb1a3f6e85d65712bc56", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "e71dd00b24c510d4", "distribution": [ { "id": "e71dd00b24c510d4", "logit": 1.2616242170333862, "probability": 0.4582104980945587 }, { "id": "c2f741799f839a2c", "logit": 0.24724352359771729, "probability": 0.1661594659090042 }, { "id": null, "logit": 1.062900185585022, "probability": 0.3756300210952759 } ], "elapsed_seconds": 0.04461302098934539, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 117, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": null }, { "choice_original_id": "99632bb0863c7081", "distribution_exact": true, "family": "lookup", "id": "canonical-confirmation:direct:fa8c40fd9550a870", "prediction": { "argmax_tie_break": "canonical_description_order_none_last", "calibration": "raw_uncalibrated", "choice": "99632bb0863c7081", "distribution": [ { "id": "4c3e4e677d1ee034", "logit": -2.4057323932647705, "probability": 0.005947228521108627 }, { "id": "e4d7573daa91e94d", "logit": -0.7179722785949707, "probability": 0.03215877711772919 }, { "id": "99632bb0863c7081", "logit": 2.6531569957733154, "probability": 0.9361864924430847 }, { "id": "55ba1c70034f7f2b", "logit": -2.1361916065216064, "probability": 0.0077870809473097324 }, { "id": "a1896bf19fceb3da", "logit": -3.0006179809570312, "probability": 0.003280642908066511 }, { "id": "7f1a1d767df0791d", "logit": -2.6931681632995605, "probability": 0.004461521282792091 }, { "id": null, "logit": -1.868399739265442, "probability": 0.010178297758102417 } ], "elapsed_seconds": 0.06015262400615029, "encoding_version": "description_only_canonical_v2", "representation_shift_diagnostic": false, "semantic_abstention": false, "token_count": 165, "uncertainty": null, "uncertainty_status": "unavailable" }, "target": "99632bb0863c7081" } ], "sample_count": 32 } } } }