{"meta": {"model": "jev-1.13.0", "workers": 4, "server_ms_header": "x-envoy-upstream-service-time"}, "phases": {"cap": {"started": 1789639332.5043662, "api_calls": 7, "trials": [{"index": 0, "template": "cap", "n": 255, "position": 0, "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 142.0, "wall_ms": 949.0, "in_tok": 2472, "out_tok": 2460, "request_id": "req_01a0aed115b97cfeba700e4b93f4504c", "p_correct": 0.3, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o1", "confidence": 0.29, "n_returned": 255, "prob_sum": 1.0, "top5": [["o1", 0.3], ["o100", 0.03], ["o255", 0.02], ["o2", 0.02], ["o200", 0.02]]}, {"index": 1, "template": "cap", "n": 256, "position": 0, "correct_key": "o1", "status": 400, "attempts": [400], "server_ms": 14.0, "wall_ms": 825.6, "in_tok": null, "out_tok": null, "request_id": "req_01a0aed115c27498b4baefa164d49ffe", "error_body": "{\"detail\": \"Too many choices. Must have at most 255 choices.\"}"}, {"index": 2, "template": "cap", "n": 257, "position": 0, "correct_key": "o1", "status": 400, "attempts": [400], "server_ms": 25.0, "wall_ms": 811.6, "in_tok": null, "out_tok": null, "request_id": "req_01a0aed115ab7c50aed57a380c58b2d5", "error_body": "{\"detail\": \"Too many choices. Must have at most 255 choices.\"}"}, {"index": 3, "template": "cap", "n": 300, "position": 0, "correct_key": "o1", "status": 400, "attempts": [400], "server_ms": 18.0, "wall_ms": 808.4, "in_tok": null, "out_tok": null, "request_id": "req_01a0aed115ac7b3580c17679cc23b2f0", "error_body": "{\"detail\": \"Too many choices. Must have at most 255 choices.\"}"}, {"index": 4, "template": "cap", "n": 512, "position": 0, "correct_key": "o1", "status": 400, "attempts": [400], "server_ms": 3.0, "wall_ms": 201.0, "in_tok": null, "out_tok": null, "request_id": "req_01a0aed11687714b9805ae4fb98975bd", "error_body": "{\"detail\": \"Too many choices. Must have at most 255 choices.\"}"}, {"index": 5, "template": "cap", "n": 513, "position": 0, "correct_key": "o1", "status": 400, "attempts": [400], "server_ms": 5.0, "wall_ms": 198.0, "in_tok": null, "out_tok": null, "request_id": "req_01a0aed11688757bbeeda48e25e555c8", "error_body": "{\"detail\": \"Too many choices. Must have at most 255 choices.\"}"}, {"index": 6, "template": "cap", "n": 1000, "position": 0, "correct_key": "o1", "status": 400, "attempts": [400], "server_ms": 9.0, "wall_ms": 208.0, "in_tok": null, "out_tok": null, "request_id": "req_01a0aed116977ab58dca2ff5ef815f15", "error_body": "{\"detail\": \"Too many choices. Must have at most 255 choices.\"}"}]}, "pilot": {"started": 1789639343.902828, "api_calls": 8, "trials": [{"index": 0, "template": "reasoning", "variant": "mult", "n": 10, "position": 1, "seed": 5000, "keep_probs": true, "correct_key": "546", "status": 200, "attempts": [200], "server_ms": 67.0, "wall_ms": 852.5, "in_tok": 385, "out_tok": 109, "request_id": "req_01a0aed14228799a918c8869d8400d30", "correct_value": "546", "state": "Warehouse log: each case holds 23 cans. Today we received 23 full cases plus 17 loose cans.", "p_correct": 0.97, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "546", "confidence": 0.95, "n_returned": 10, "prob_sum": 1.0, "top5": [["546", 0.97], ["545", 0.02], ["551", 0.01], ["750", 0.0], ["409", 0.0]], "probabilities": {"750": 0.0, "551": 0.01, "409": 0.0, "663": 0.0, "679": 0.0, "497": 0.0, "548": 0.0, "546": 0.97, "446": 0.0, "545": 0.02}}, {"index": 1, "template": "reasoning", "variant": "mult", "n": 10, "position": 1, "seed": 5001, "keep_probs": true, "correct_key": "510", "status": 200, "attempts": [200], "server_ms": 89.0, "wall_ms": 866.8, "in_tok": 384, "out_tok": 109, "request_id": "req_01a0aed142257ea69ce43613bc94624d", "correct_value": "510", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 6 loose bolts.", "p_correct": 0.18, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "516", "confidence": 0.76, "n_returned": 10, "prob_sum": 1.0, "top5": [["516", 0.78], ["510", 0.18], ["531", 0.02], ["586", 0.01], ["585", 0.01]], "probabilities": {"821": 0.0, "510": 0.18, "329": 0.0, "664": 0.0, "500": 0.0, "586": 0.01, "531": 0.02, "585": 0.01, "516": 0.78, "489": 0.0}}, {"index": 2, "template": "reasoning", "variant": "mult", "n": 10, "position": 1, "seed": 5002, "keep_probs": true, "correct_key": "158", "status": 200, "attempts": [200], "server_ms": 67.0, "wall_ms": 865.5, "in_tok": 385, "out_tok": 109, "request_id": "req_01a0aed14233723c88aabb1f601a1638", "correct_value": "158", "state": "Warehouse log: each tray holds 13 eggs. Today we received 11 full trays plus 15 loose eggs.", "p_correct": 0.18, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "148", "confidence": 0.65, "n_returned": 10, "prob_sum": 1.0, "top5": [["148", 0.69], ["158", 0.18], ["164", 0.1], ["168", 0.03], ["294", 0.0]], "probabilities": {"294": 0.0, "148": 0.69, "168": 0.03, "322": 0.0, "265": 0.0, "203": 0.0, "164": 0.1, "158": 0.18, "109": 0.0, "326": 0.0}}, {"index": 3, "template": "reasoning", "variant": "two_step", "n": 10, "position": 1, "seed": 5000, "keep_probs": true, "correct_key": "518", "status": 200, "attempts": [200], "server_ms": 76.0, "wall_ms": 853.1, "in_tok": 403, "out_tok": 109, "request_id": "req_01a0aed1422470e9941dcf7237030256", "correct_value": "518", "state": "Warehouse log: each case holds 23 cans. Today we received 23 full cases plus 17 loose cans. During unloading, 28 cans were damaged and thrown away.", "p_correct": 0.67, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "518", "confidence": 0.62, "n_returned": 10, "prob_sum": 1.0, "top5": [["518", 0.67], ["517", 0.31], ["520", 0.01], ["381", 0.01], ["418", 0.0]], "probabilities": {"418": 0.0, "523": 0.0, "520": 0.01, "722": 0.0, "381": 0.01, "635": 0.0, "469": 0.0, "517": 0.31, "518": 0.67, "651": 0.0}}, {"index": 4, "template": "reasoning", "variant": "two_step", "n": 10, "position": 1, "seed": 5001, "keep_probs": true, "correct_key": "486", "status": 200, "attempts": [200], "server_ms": 62.0, "wall_ms": 266.9, "in_tok": 402, "out_tok": 109, "request_id": "req_01a0aed1433b7f53af56c4850491a61c", "correct_value": "486", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 6 loose bolts. During unloading, 24 bolts were damaged and thrown away.", "p_correct": 0.8, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "486", "confidence": 0.78, "n_returned": 10, "prob_sum": 0.99, "top5": [["486", 0.8], ["492", 0.07], ["561", 0.07], ["507", 0.02], ["562", 0.01]], "probabilities": {"562": 0.01, "797": 0.0, "486": 0.8, "305": 0.0, "476": 0.01, "492": 0.07, "465": 0.01, "507": 0.02, "561": 0.07, "640": 0.0}}, {"index": 5, "template": "reasoning", "variant": "two_step", "n": 10, "position": 1, "seed": 5002, "keep_probs": true, "correct_key": "109", "status": 200, "attempts": [200], "server_ms": 108.0, "wall_ms": 310.6, "in_tok": 400, "out_tok": 106, "request_id": "req_01a0aed1433c70d697aff0834af01459", "correct_value": "109", "state": "Warehouse log: each tray holds 13 eggs. Today we received 11 full trays plus 15 loose eggs. During unloading, 49 eggs were damaged and thrown away.", "p_correct": 0.42, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "109", "confidence": 0.36, "n_returned": 10, "prob_sum": 1.0, "top5": [["109", 0.42], ["106", 0.18], ["87", 0.13], ["60", 0.11], ["99", 0.09]], "probabilities": {"99": 0.09, "168": 0.01, "137": 0.01, "152": 0.02, "87": 0.13, "119": 0.01, "60": 0.11, "166": 0.02, "106": 0.18, "109": 0.42}}, {"index": 6, "template": "lexical", "n": 10, "position": 1, "seed": 6000, "keep_probs": true, "correct_key": "FJ-3784", "status": 200, "attempts": [200], "server_ms": 101.0, "wall_ms": 301.5, "in_tok": 435, "out_tok": 146, "request_id": "req_01a0aed1434272eeb07b0df328675c78", "correct_value": "FJ-3784", "state": "Support ticket from Marcus Oyelaran. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is FJ-3784. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "FJ-3784", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["FJ-3784", 1.0], ["NZ-6518", 0.0], ["JW-8319", 0.0], ["YL-1747", 0.0], ["US-7659", 0.0]], "probabilities": {"NZ-6518": 0.0, "JW-8319": 0.0, "YL-1747": 0.0, "US-7659": 0.0, "ZP-8077": 0.0, "BN-6992": 0.0, "MA-9262": 0.0, "LZ-3194": 0.0, "FJ-3784": 1.0, "TX-5397": 0.0}}, {"index": 7, "template": "lexical", "n": 10, "position": 1, "seed": 6001, "keep_probs": true, "correct_key": "LZ-3839", "status": 200, "attempts": [200], "server_ms": 63.0, "wall_ms": 255.3, "in_tok": 437, "out_tok": 147, "request_id": "req_01a0aed143417833a6c8b75d9b1c66c7", "correct_value": "LZ-3839", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is LZ-3839. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "LZ-3839", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["LZ-3839", 1.0], ["WQ-5123", 0.0], ["VZ-8883", 0.0], ["NA-6711", 0.0], ["WX-9643", 0.0]], "probabilities": {"WQ-5123": 0.0, "VZ-8883": 0.0, "NA-6711": 0.0, "LZ-3839": 1.0, "WX-9643": 0.0, "XQ-3984": 0.0, "VP-7759": 0.0, "LL-4556": 0.0, "PT-7176": 0.0, "TW-3061": 0.0}}]}, "main": {"started": 1789639357.925511, "api_calls": 105, "trials": [{"index": 0, "template": "lexical", "variant": null, "n": 200, "position": 200, "rep": 2, "seed": 30002, "labels": "value", "keep_probs": false, "correct_key": "AV-9496", "status": 200, "attempts": [200], "server_ms": 120.0, "wall_ms": 875.0, "in_tok": 2547, "out_tok": 2447, "request_id": "req_01a0aed178d97b49a7d31e47a734e620", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "AV-9496", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["AV-9496", 1.0], ["NU-4426", 0.0], ["UW-7009", 0.0], ["SZ-8372", 0.0], ["GQ-4805", 0.0]]}, {"index": 1, "template": "lexical", "variant": null, "n": 50, "position": 25, "rep": 2, "seed": 15002, "labels": "value", "keep_probs": true, "correct_key": "SS-8141", "status": 200, "attempts": [200], "server_ms": 112.0, "wall_ms": 900.7, "in_tok": 874, "out_tok": 624, "request_id": "req_01a0aed178f67597abcc279de2966b91", "correct_value": "SS-8141", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is SS-8141. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "SS-8141", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["SS-8141", 1.0], ["TZ-6342", 0.0], ["WB-8736", 0.0], ["MF-9075", 0.0], ["MN-5423", 0.0]], "probabilities": {"TZ-6342": 0.0, "WB-8736": 0.0, "MF-9075": 0.0, "MN-5423": 0.0, "MU-1259": 0.0, "WX-4388": 0.0, "KS-3726": 0.0, "PY-8591": 0.0, "ES-9686": 0.0, "KB-8333": 0.0, "SL-3210": 0.0, "JS-1666": 0.0, "RU-9816": 0.0, "ST-8638": 0.0, "SE-4698": 0.0, "DT-7722": 0.0, "SH-3775": 0.0, "WB-4163": 0.0, "AE-9512": 0.0, "TW-9116": 0.0, "KM-4035": 0.0, "KU-6416": 0.0, "EG-7473": 0.0, "LY-2553": 0.0, "NT-9311": 0.0, "LK-7156": 0.0, "NT-2034": 0.0, "DG-9216": 0.0, "ND-5422": 0.0, "HF-3915": 0.0, "TL-9500": 0.0, "YV-9991": 0.0, "ZS-1140": 0.0, "HR-1686": 0.0, "SF-7642": 0.0, "JJ-1379": 0.0, "BF-7019": 0.0, "YM-3017": 0.0, "FW-8221": 0.0, "AD-7367": 0.0, "RH-9468": 0.0, "KH-8358": 0.0, "VU-6041": 0.0, "SG-1015": 0.0, "NA-7727": 0.0, "SS-8141": 1.0, "EV-3579": 0.0, "SL-8701": 0.0, "FV-2437": 0.0, "YQ-9732": 0.0}}, {"index": 2, "template": "reasoning", "variant": "two_step", "n": 10, "position": 1, "rep": 3, "seed": 61003, "labels": "value", "keep_probs": true, "correct_key": "616", "status": 200, "attempts": [200], "server_ms": 91.0, "wall_ms": 886.4, "in_tok": 402, "out_tok": 109, "request_id": "req_01a0aed178fc7cf5afd09b65bf6196f8", "correct_value": "616", "state": "Warehouse log: each case holds 23 cans. Today we received 29 full cases plus 6 loose cans. During unloading, 57 cans were damaged and thrown away.", "p_correct": 0.19, "rank": 3, "ties_at_pc": 0, "argmax_correct": false, "choice": "610", "confidence": 0.33, "n_returned": 10, "prob_sum": 1.0, "top5": [["610", 0.39], ["614", 0.24], ["616", 0.19], ["607", 0.08], ["313", 0.05]], "probabilities": {"727": 0.0, "616": 0.19, "313": 0.05, "614": 0.24, "610": 0.39, "626": 0.02, "607": 0.08, "399": 0.03, "795": 0.0, "781": 0.0}}, {"index": 3, "template": "reasoning", "variant": "two_step", "n": 50, "position": 1, "rep": 1, "seed": 65001, "labels": "value", "keep_probs": true, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 48.0, "wall_ms": 796.3, "in_tok": 719, "out_tok": 466, "request_id": "req_01a0aed178d57e9b8bf8d22b2f63d1c1", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.24000000000000002, "rank": 2, "ties_at_pc": 1, "argmax_correct": false, "choice": "478", "confidence": 0.38, "n_returned": 50, "prob_sum": 0.99, "top5": [["478", 0.4], ["476", 0.24000000000000002], ["477", 0.24000000000000002], ["475", 0.04], ["467", 0.02]], "probabilities": {"467": 0.02, "142": 0.0, "807": 0.0, "685": 0.0, "130": 0.0, "637": 0.0, "563": 0.0, "487": 0.0, "815": 0.0, "456": 0.0, "91": 0.0, "475": 0.04, "162": 0.0, "850": 0.0, "743": 0.0, "157": 0.0, "631": 0.0, "279": 0.0, "472": 0.0, "512": 0.0, "363": 0.01, "509": 0.0, "96": 0.0, "220": 0.0, "225": 0.0, "773": 0.0, "593": 0.0, "628": 0.0, "393": 0.0, "392": 0.0, "211": 0.0, "159": 0.0, "765": 0.0, "478": 0.4, "476": 0.24000000000000002, "377": 0.0, "822": 0.0, "672": 0.0, "577": 0.0, "479": 0.02, "477": 0.24000000000000002, "445": 0.0, "268": 0.0, "831": 0.0, "85": 0.0, "562": 0.0, "498": 0.02, "482": 0.0, "716": 0.0, "357": 0.0}}, {"index": 4, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 5, "seed": 80005, "labels": "value", "keep_probs": false, "correct_key": "457", "status": 200, "attempts": [200], "server_ms": 75.0, "wall_ms": 260.1, "in_tok": 1917, "out_tok": 1813, "request_id": "req_01a0aed179be7ff5b9887ba86dde408e", "correct_value": "457", "state": "Warehouse log: each crate holds 25 bottles. Today we received 20 full crates plus 12 loose bottles. During unloading, 55 bottles were damaged and thrown away.", "p_correct": 0.01, "rank": 22, "ties_at_pc": 46, "argmax_correct": false, "choice": "295", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["455", 0.03], ["393", 0.03], ["403", 0.03], ["445", 0.03], ["617", 0.03]]}, {"index": 5, "template": "reasoning", "variant": "two_step", "n": 50, "position": 50, "rep": 3, "seed": 65003, "labels": "value", "keep_probs": true, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 89.0, "wall_ms": 277.9, "in_tok": 722, "out_tok": 467, "request_id": "req_01a0aed17a0e78a3bb7fb54b92d6b013", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.73, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "281", "confidence": 0.72, "n_returned": 50, "prob_sum": 1.0, "top5": [["281", 0.73], ["279", 0.09], ["282", 0.05], ["280", 0.02], ["359", 0.02]], "probabilities": {"536": 0.0, "181": 0.0, "280": 0.02, "243": 0.0, "396": 0.0, "262": 0.01, "660": 0.0, "140": 0.0, "217": 0.01, "141": 0.01, "200": 0.0, "617": 0.0, "271": 0.0, "581": 0.0, "666": 0.0, "263": 0.01, "282": 0.05, "198": 0.0, "343": 0.0, "143": 0.0, "368": 0.0, "480": 0.0, "525": 0.0, "236": 0.01, "182": 0.0, "323": 0.0, "266": 0.0, "122": 0.0, "279": 0.09, "610": 0.0, "102": 0.0, "359": 0.02, "598": 0.0, "300": 0.0, "296": 0.0, "283": 0.02, "319": 0.0, "55": 0.0, "281": 0.73, "244": 0.0, "626": 0.0, "291": 0.02, "381": 0.0, "82": 0.0, "178": 0.0, "623": 0.0, "566": 0.0, "382": 0.0, "653": 0.0, "130": 0.0}}, {"index": 6, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 0, "seed": 80000, "labels": "value", "keep_probs": false, "correct_key": "356", "status": 200, "attempts": [200], "server_ms": 102.0, "wall_ms": 305.2, "in_tok": 1898, "out_tok": 1793, "request_id": "req_01a0aed17a20730bbce3909959b954f9", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.0, "rank": 56, "ties_at_pc": 144, "argmax_correct": false, "choice": "88", "confidence": 0.12, "n_returned": 200, "prob_sum": 1.0, "top5": [["88", 0.14], ["44", 0.04], ["196", 0.03], ["369", 0.03], ["40", 0.03]]}, {"index": 7, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 1, "seed": 80001, "labels": "value", "keep_probs": false, "correct_key": "283", "status": 200, "attempts": [200], "server_ms": 61.0, "wall_ms": 256.4, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed17a2973c58236635f2b1e4c07", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.11, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "283", "confidence": 0.1, "n_returned": 200, "prob_sum": 0.99, "top5": [["283", 0.11], ["223", 0.09], ["198", 0.05], ["253", 0.05], ["282", 0.04]]}, {"index": 8, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 0, "seed": 80000, "labels": "value", "keep_probs": false, "correct_key": "356", "status": 200, "attempts": [200], "server_ms": 93.0, "wall_ms": 282.5, "in_tok": 1898, "out_tok": 1793, "request_id": "req_01a0aed17ac67d0d9ba8699c0439cf0a", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 17, "ties_at_pc": 51, "argmax_correct": false, "choice": "88", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["88", 0.060000000000000005], ["240", 0.05], ["369", 0.05], ["44", 0.04], ["196", 0.03]]}, {"index": 9, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 0, "seed": 80000, "labels": "value", "keep_probs": false, "correct_key": "356", "status": 200, "attempts": [200], "server_ms": 70.0, "wall_ms": 263.4, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed17b297e878ec24c39ee272cb1", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 20, "ties_at_pc": 43, "argmax_correct": false, "choice": "240", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["240", 0.060000000000000005], ["196", 0.04], ["214", 0.04], ["88", 0.04], ["369", 0.04]]}, {"index": 10, "template": "lexical", "variant": null, "n": 200, "position": 200, "rep": 0, "seed": 30000, "labels": "value", "keep_probs": false, "correct_key": "GA-8933", "status": 200, "attempts": [200], "server_ms": 84.0, "wall_ms": 280.7, "in_tok": 2535, "out_tok": 2435, "request_id": "req_01a0aed17b2c7cb7baea154f1d6720da", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GA-8933", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GA-8933", 1.0], ["GE-9771", 0.0], ["SP-6865", 0.0], ["LQ-6050", 0.0], ["DW-9337", 0.0]]}, {"index": 11, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 2, "seed": 80002, "labels": "value", "keep_probs": false, "correct_key": "921", "status": 200, "attempts": [200], "server_ms": 75.0, "wall_ms": 275.5, "in_tok": 2000, "out_tok": 1897, "request_id": "req_01a0aed17b52731394b7a2d482a72589", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.02, "rank": 7, "ties_at_pc": 15, "argmax_correct": false, "choice": "729", "confidence": 0.05, "n_returned": 200, "prob_sum": 0.99, "top5": [["729", 0.060000000000000005], ["919", 0.04], ["957", 0.03], ["707", 0.03], ["819", 0.03]]}, {"index": 12, "template": "lexical", "variant": null, "n": 200, "position": 190, "rep": 1, "seed": 30001, "labels": "value", "keep_probs": false, "correct_key": "VU-5695", "status": 200, "attempts": [200], "server_ms": 85.0, "wall_ms": 271.4, "in_tok": 2546, "out_tok": 2445, "request_id": "req_01a0aed17be072d38373fda2c9636b83", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VU-5695", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VU-5695", 1.0], ["NC-3074", 0.0], ["NV-8727", 0.0], ["RW-7130", 0.0], ["LS-8698", 0.0]]}, {"index": 13, "template": "reasoning", "variant": "two_step", "n": 50, "position": 50, "rep": 2, "seed": 65002, "labels": "value", "keep_probs": true, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 116.0, "wall_ms": 306.5, "in_tok": 717, "out_tok": 463, "request_id": "req_01a0aed17c307a47870e6cb8ee57c5e0", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.93, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "267", "confidence": 0.91, "n_returned": 50, "prob_sum": 0.99, "top5": [["267", 0.93], ["265", 0.05], ["266", 0.01], ["264", 0.0], ["283", 0.0]], "probabilities": {"264": 0.0, "283": 0.0, "257": 0.0, "291": 0.0, "477": 0.0, "167": 0.0, "265": 0.05, "37": 0.0, "478": 0.0, "411": 0.0, "400": 0.0, "251": 0.0, "563": 0.0, "266": 0.01, "438": 0.0, "217": 0.0, "259": 0.0, "269": 0.0, "268": 0.0, "596": 0.0, "201": 0.0, "367": 0.0, "243": 0.0, "664": 0.0, "62": 0.0, "487": 0.0, "549": 0.0, "307": 0.0, "81": 0.0, "270": 0.0, "356": 0.0, "128": 0.0, "380": 0.0, "638": 0.0, "27": 0.0, "255": 0.0, "612": 0.0, "630": 0.0, "419": 0.0, "385": 0.0, "193": 0.0, "47": 0.0, "277": 0.0, "124": 0.0, "219": 0.0, "227": 0.0, "353": 0.0, "267": 0.93, "77": 0.0, "133": 0.0}}, {"index": 14, "template": "lexical", "variant": null, "n": 50, "position": 1, "rep": 1, "seed": 15001, "labels": "value", "keep_probs": true, "correct_key": "ZS-5465", "status": 200, "attempts": [200], "server_ms": 56.0, "wall_ms": 251.7, "in_tok": 877, "out_tok": 628, "request_id": "req_01a0aed17c457fa49f704a13b8af75a5", "correct_value": "ZS-5465", "state": "Support ticket from Elena Brandt. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is ZS-5465. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "ZS-5465", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["ZS-5465", 1.0], ["YU-9436", 0.0], ["CS-9022", 0.0], ["SQ-8294", 0.0], ["DL-2281", 0.0]], "probabilities": {"YU-9436": 0.0, "CS-9022": 0.0, "SQ-8294": 0.0, "DL-2281": 0.0, "YC-2256": 0.0, "ZM-8230": 0.0, "DV-2939": 0.0, "FV-4057": 0.0, "AN-2754": 0.0, "VD-4687": 0.0, "LH-8546": 0.0, "DA-9161": 0.0, "GH-8434": 0.0, "PB-4779": 0.0, "CP-4922": 0.0, "TR-5200": 0.0, "ZG-6390": 0.0, "MJ-8925": 0.0, "HN-1409": 0.0, "VQ-4725": 0.0, "PP-3232": 0.0, "GM-7288": 0.0, "WN-7040": 0.0, "BA-8802": 0.0, "ZE-4016": 0.0, "QZ-8317": 0.0, "MU-3899": 0.0, "XG-3601": 0.0, "VL-9183": 0.0, "TQ-7252": 0.0, "UG-6734": 0.0, "AY-8082": 0.0, "LQ-3836": 0.0, "KW-5623": 0.0, "WS-9676": 0.0, "BH-6847": 0.0, "CC-7674": 0.0, "QD-4155": 0.0, "ZS-5465": 1.0, "RH-8863": 0.0, "RS-1008": 0.0, "KQ-2965": 0.0, "WC-5737": 0.0, "XF-5580": 0.0, "MJ-8931": 0.0, "DY-3944": 0.0, "HY-8204": 0.0, "BU-1426": 0.0, "KB-6796": 0.0, "ZX-2850": 0.0}}, {"index": 15, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 2, "seed": 80002, "labels": "value", "keep_probs": false, "correct_key": "921", "status": 200, "attempts": [200], "server_ms": 67.0, "wall_ms": 266.8, "in_tok": 2000, "out_tok": 1897, "request_id": "req_01a0aed17c66774086aafbdff48835fd", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.02, "rank": 5, "ties_at_pc": 15, "argmax_correct": false, "choice": "729", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["729", 0.05], ["731", 0.03], ["603", 0.03], ["819", 0.03], ["930", 0.02]]}, {"index": 16, "template": "reasoning", "variant": "two_step", "n": 50, "position": 1, "rep": 0, "seed": 65000, "labels": "value", "keep_probs": true, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 122.0, "wall_ms": 308.8, "in_tok": 716, "out_tok": 461, "request_id": "req_01a0aed17cf37f5290fce29e00aa8967", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.62, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "279", "confidence": 0.6, "n_returned": 50, "prob_sum": 1.0, "top5": [["279", 0.62], ["278", 0.21000000000000002], ["280", 0.060000000000000005], ["376", 0.02], ["379", 0.02]], "probabilities": {"528": 0.0, "381": 0.0, "281": 0.01, "394": 0.0, "30": 0.0, "625": 0.0, "590": 0.0, "61": 0.0, "525": 0.0, "376": 0.02, "196": 0.0, "280": 0.060000000000000005, "269": 0.01, "379": 0.02, "123": 0.0, "398": 0.0, "253": 0.01, "508": 0.0, "277": 0.01, "56": 0.0, "279": 0.62, "273": 0.0, "190": 0.0, "500": 0.0, "233": 0.01, "104": 0.0, "192": 0.0, "12": 0.0, "591": 0.0, "81": 0.0, "278": 0.21000000000000002, "266": 0.0, "289": 0.02, "103": 0.0, "473": 0.0, "305": 0.0, "419": 0.0, "116": 0.0, "67": 0.0, "39": 0.0, "364": 0.0, "292": 0.0, "246": 0.0, "325": 0.0, "159": 0.0, "539": 0.0, "132": 0.0, "38": 0.0, "179": 0.0, "651": 0.0}}, {"index": 17, "template": "lexical", "variant": null, "n": 50, "position": 25, "rep": 1, "seed": 15001, "labels": "value", "keep_probs": true, "correct_key": "ZS-5465", "status": 200, "attempts": [200], "server_ms": 85.0, "wall_ms": 281.3, "in_tok": 877, "out_tok": 628, "request_id": "req_01a0aed17d417afc810be5cf9bd5e93c", "correct_value": "ZS-5465", "state": "Support ticket from Elena Brandt. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is ZS-5465. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "ZS-5465", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["ZS-5465", 1.0], ["MJ-8931", 0.0], ["QZ-8317", 0.0], ["BH-6847", 0.0], ["ZE-4016", 0.0]], "probabilities": {"MJ-8931": 0.0, "QZ-8317": 0.0, "BH-6847": 0.0, "ZE-4016": 0.0, "WS-9676": 0.0, "CP-4922": 0.0, "LQ-3836": 0.0, "KQ-2965": 0.0, "RS-1008": 0.0, "VQ-4725": 0.0, "DA-9161": 0.0, "DY-3944": 0.0, "HN-1409": 0.0, "GH-8434": 0.0, "VD-4687": 0.0, "DV-2939": 0.0, "ZM-8230": 0.0, "AN-2754": 0.0, "ZG-6390": 0.0, "ZS-5465": 1.0, "PP-3232": 0.0, "PB-4779": 0.0, "AY-8082": 0.0, "DL-2281": 0.0, "MU-3899": 0.0, "ZX-2850": 0.0, "SQ-8294": 0.0, "YC-2256": 0.0, "QD-4155": 0.0, "UG-6734": 0.0, "XG-3601": 0.0, "TQ-7252": 0.0, "XF-5580": 0.0, "MJ-8925": 0.0, "GM-7288": 0.0, "LH-8546": 0.0, "CC-7674": 0.0, "TR-5200": 0.0, "WC-5737": 0.0, "BU-1426": 0.0, "FV-4057": 0.0, "WN-7040": 0.0, "KB-6796": 0.0, "KW-5623": 0.0, "CS-9022": 0.0, "BA-8802": 0.0, "HY-8204": 0.0, "YU-9436": 0.0, "RH-8863": 0.0, "VL-9183": 0.0}}, {"index": 18, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 3, "seed": 80003, "labels": "value", "keep_probs": false, "correct_key": "418", "status": 200, "attempts": [200], "server_ms": 78.0, "wall_ms": 266.2, "in_tok": 1905, "out_tok": 1801, "request_id": "req_01a0aed17d627b31977408a8a70176f3", "correct_value": "418", "state": "Warehouse log: each crate holds 15 bottles. Today we received 29 full crates plus 13 loose bottles. During unloading, 30 bottles were damaged and thrown away.", "p_correct": 0.0, "rank": 62, "ties_at_pc": 138, "argmax_correct": false, "choice": "328", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["328", 0.04], ["520", 0.03], ["500", 0.03], ["187", 0.03], ["313", 0.03]]}, {"index": 19, "template": "lexical", "variant": null, "n": 200, "position": 190, "rep": 2, "seed": 30002, "labels": "value", "keep_probs": false, "correct_key": "AV-9496", "status": 200, "attempts": [200], "server_ms": 119.0, "wall_ms": 321.3, "in_tok": 2547, "out_tok": 2447, "request_id": "req_01a0aed17d727d68a2e727efe2c6991c", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "AV-9496", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["AV-9496", 1.0], ["SZ-8372", 0.0], ["PK-6182", 0.0], ["NA-6202", 0.0], ["MV-7988", 0.0]]}, {"index": 20, "template": "lexical", "variant": null, "n": 200, "position": 50, "rep": 2, "seed": 30002, "labels": "value", "keep_probs": false, "correct_key": "AV-9496", "status": 200, "attempts": [200], "server_ms": 200.0, "wall_ms": 388.7, "in_tok": 2547, "out_tok": 2447, "request_id": "req_01a0aed17e2778aebd0594d79206aa04", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "AV-9496", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["AV-9496", 1.0], ["GD-2467", 0.0], ["SZ-8372", 0.0], ["KR-3297", 0.0], ["XM-2285", 0.0]]}, {"index": 21, "template": "reasoning", "variant": "two_step", "n": 50, "position": 25, "rep": 3, "seed": 65003, "labels": "value", "keep_probs": true, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 86.0, "wall_ms": 281.5, "in_tok": 722, "out_tok": 467, "request_id": "req_01a0aed17e5a7d35a233d953b216005f", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.11, "rank": 4, "ties_at_pc": 0, "argmax_correct": false, "choice": "279", "confidence": 0.29, "n_returned": 50, "prob_sum": 0.99, "top5": [["279", 0.31], ["280", 0.21000000000000002], ["282", 0.12], ["281", 0.11], ["262", 0.05]], "probabilities": {"198": 0.0, "626": 0.0, "283": 0.03, "343": 0.0, "141": 0.01, "182": 0.01, "382": 0.0, "236": 0.01, "623": 0.0, "598": 0.0, "300": 0.0, "359": 0.0, "525": 0.0, "200": 0.0, "266": 0.01, "617": 0.0, "181": 0.01, "281": 0.11, "280": 0.21000000000000002, "666": 0.0, "566": 0.0, "262": 0.05, "55": 0.0, "130": 0.01, "279": 0.31, "271": 0.02, "368": 0.0, "263": 0.02, "243": 0.01, "323": 0.0, "296": 0.01, "660": 0.0, "217": 0.02, "178": 0.0, "82": 0.0, "291": 0.01, "122": 0.0, "244": 0.0, "581": 0.0, "282": 0.12, "143": 0.01, "610": 0.0, "536": 0.0, "653": 0.0, "319": 0.0, "102": 0.0, "381": 0.0, "140": 0.0, "396": 0.0, "480": 0.0}}, {"index": 22, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 3, "seed": 80003, "labels": "value", "keep_probs": false, "correct_key": "418", "status": 200, "attempts": [200], "server_ms": 144.0, "wall_ms": 332.8, "in_tok": 1905, "out_tok": 1801, "request_id": "req_01a0aed17e707cd8b4ff2f1fb12e7898", "correct_value": "418", "state": "Warehouse log: each crate holds 15 bottles. Today we received 29 full crates plus 13 loose bottles. During unloading, 30 bottles were damaged and thrown away.", "p_correct": 0.0, "rank": 62, "ties_at_pc": 138, "argmax_correct": false, "choice": "385", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["328", 0.04], ["385", 0.04], ["520", 0.04], ["492", 0.03], ["495", 0.03]]}, {"index": 23, "template": "lexical", "variant": null, "n": 200, "position": 100, "rep": 2, "seed": 30002, "labels": "value", "keep_probs": false, "correct_key": "AV-9496", "status": 200, "attempts": [200], "server_ms": 71.0, "wall_ms": 272.4, "in_tok": 2547, "out_tok": 2447, "request_id": "req_01a0aed17eb67e6a99de9d8a8ef169e7", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "AV-9496", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["AV-9496", 1.0], ["UW-7009", 0.0], ["SQ-6348", 0.0], ["KQ-7883", 0.0], ["MR-4220", 0.0]]}, {"index": 24, "template": "reasoning", "variant": "two_step", "n": 200, "position": 190, "rep": 4, "seed": 80004, "labels": "value", "keep_probs": false, "correct_key": "650", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 273.6, "in_tok": 1938, "out_tok": 1834, "request_id": "req_01a0aed17f797628a4013b28089d1021", "correct_value": "650", "state": "Warehouse log: each box holds 31 pens. Today we received 22 full boxs plus 5 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.05, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "651", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["651", 0.060000000000000005], ["650", 0.05], ["627", 0.04], ["649", 0.04], ["636", 0.03]]}, {"index": 25, "template": "reasoning", "variant": "two_step", "n": 200, "position": 10, "rep": 0, "seed": 80000, "labels": "value", "keep_probs": false, "correct_key": "356", "status": 200, "attempts": [200], "server_ms": 111.0, "wall_ms": 319.7, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed17fd07b4694d39aec746eff80", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.0, "rank": 60, "ties_at_pc": 140, "argmax_correct": false, "choice": "357", "confidence": 0.04, "n_returned": 200, "prob_sum": 1.0, "top5": [["369", 0.05], ["357", 0.05], ["88", 0.05], ["196", 0.04], ["44", 0.04]]}, {"index": 26, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 5, "seed": 80005, "labels": "value", "keep_probs": false, "correct_key": "457", "status": 200, "attempts": [200], "server_ms": 78.0, "wall_ms": 266.1, "in_tok": 1917, "out_tok": 1813, "request_id": "req_01a0aed17fb971319e5842a7e3d81a38", "correct_value": "457", "state": "Warehouse log: each crate holds 25 bottles. Today we received 20 full crates plus 12 loose bottles. During unloading, 55 bottles were damaged and thrown away.", "p_correct": 0.01, "rank": 17, "ties_at_pc": 55, "argmax_correct": false, "choice": "445", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["445", 0.05], ["437", 0.04], ["343", 0.04], ["295", 0.03], ["305", 0.03]]}, {"index": 27, "template": "lexical", "variant": null, "n": 50, "position": 50, "rep": 2, "seed": 15002, "labels": "value", "keep_probs": true, "correct_key": "SS-8141", "status": 200, "attempts": [200], "server_ms": 112.0, "wall_ms": 318.5, "in_tok": 874, "out_tok": 624, "request_id": "req_01a0aed17fd8721483143c4071c69b24", "correct_value": "SS-8141", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is SS-8141. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "SS-8141", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["SS-8141", 1.0], ["DT-7722", 0.0], ["KS-3726", 0.0], ["WB-4163", 0.0], ["FW-8221", 0.0]], "probabilities": {"DT-7722": 0.0, "KS-3726": 0.0, "WB-4163": 0.0, "FW-8221": 0.0, "LK-7156": 0.0, "WB-8736": 0.0, "ES-9686": 0.0, "SL-3210": 0.0, "DG-9216": 0.0, "SF-7642": 0.0, "SG-1015": 0.0, "KM-4035": 0.0, "FV-2437": 0.0, "RU-9816": 0.0, "MN-5423": 0.0, "ND-5422": 0.0, "SS-8141": 1.0, "SE-4698": 0.0, "LY-2553": 0.0, "TL-9500": 0.0, "EV-3579": 0.0, "TW-9116": 0.0, "ZS-1140": 0.0, "HF-3915": 0.0, "YV-9991": 0.0, "MU-1259": 0.0, "HR-1686": 0.0, "KH-8358": 0.0, "NT-2034": 0.0, "JS-1666": 0.0, "JJ-1379": 0.0, "YQ-9732": 0.0, "MF-9075": 0.0, "NT-9311": 0.0, "BF-7019": 0.0, "KU-6416": 0.0, "AD-7367": 0.0, "YM-3017": 0.0, "PY-8591": 0.0, "VU-6041": 0.0, "SL-8701": 0.0, "WX-4388": 0.0, "SH-3775": 0.0, "NA-7727": 0.0, "ST-8638": 0.0, "KB-8333": 0.0, "TZ-6342": 0.0, "AE-9512": 0.0, "EG-7473": 0.0, "RH-9468": 0.0}}, {"index": 28, "template": "reasoning", "variant": "two_step", "n": 50, "position": 25, "rep": 0, "seed": 65000, "labels": "value", "keep_probs": true, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 84.0, "wall_ms": 283.3, "in_tok": 716, "out_tok": 461, "request_id": "req_01a0aed1808c7359aa68023902cd9ecc", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.34, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "279", "confidence": 0.32, "n_returned": 50, "prob_sum": 1.0, "top5": [["279", 0.34], ["278", 0.31], ["281", 0.12], ["277", 0.05], ["269", 0.04]], "probabilities": {"281": 0.12, "591": 0.0, "269": 0.04, "590": 0.0, "394": 0.0, "196": 0.0, "325": 0.01, "38": 0.0, "500": 0.0, "381": 0.0, "67": 0.0, "364": 0.0, "398": 0.0, "379": 0.01, "190": 0.01, "273": 0.01, "508": 0.0, "419": 0.0, "280": 0.02, "292": 0.0, "253": 0.01, "525": 0.0, "103": 0.01, "159": 0.0, "277": 0.05, "132": 0.0, "539": 0.0, "625": 0.0, "116": 0.0, "30": 0.0, "278": 0.31, "123": 0.01, "473": 0.0, "651": 0.0, "192": 0.0, "81": 0.0, "61": 0.0, "233": 0.01, "528": 0.0, "56": 0.01, "279": 0.34, "305": 0.0, "246": 0.0, "12": 0.0, "266": 0.0, "179": 0.0, "376": 0.0, "39": 0.0, "104": 0.01, "289": 0.02}}, {"index": 29, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 4, "seed": 80004, "labels": "value", "keep_probs": false, "correct_key": "650", "status": 200, "attempts": [200], "server_ms": 176.0, "wall_ms": 362.3, "in_tok": 1938, "out_tok": 1834, "request_id": "req_01a0aed180cd768a881176eedbcec6d8", "correct_value": "650", "state": "Warehouse log: each box holds 31 pens. Today we received 22 full boxs plus 5 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 18, "ties_at_pc": 49, "argmax_correct": false, "choice": "651", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["651", 0.060000000000000005], ["627", 0.05], ["629", 0.04], ["649", 0.04], ["620", 0.03]]}, {"index": 30, "template": "lexical", "variant": null, "n": 10, "position": 10, "rep": 1, "seed": 11001, "labels": "value", "keep_probs": true, "correct_key": "HW-4944", "status": 200, "attempts": [200], "server_ms": 65.0, "wall_ms": 254.7, "in_tok": 432, "out_tok": 142, "request_id": "req_01a0aed180f37dfe8ef9e04c638c3dd4", "correct_value": "HW-4944", "state": "Support ticket from Tomasz Kowal. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is HW-4944. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "HW-4944", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["HW-4944", 1.0], ["PA-2699", 0.0], ["GK-1329", 0.0], ["LY-8273", 0.0], ["TK-3090", 0.0]], "probabilities": {"PA-2699": 0.0, "GK-1329": 0.0, "LY-8273": 0.0, "TK-3090": 0.0, "SD-7093": 0.0, "PH-8850": 0.0, "UD-6618": 0.0, "HW-4944": 1.0, "EH-3342": 0.0, "AF-5927": 0.0}}, {"index": 31, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 4, "seed": 80004, "labels": "value", "keep_probs": false, "correct_key": "650", "status": 200, "attempts": [200], "server_ms": 136.0, "wall_ms": 340.0, "in_tok": 1938, "out_tok": 1834, "request_id": "req_01a0aed1810972d6a91b9a44582a87a6", "correct_value": "650", "state": "Warehouse log: each box holds 31 pens. Today we received 22 full boxs plus 5 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.12, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "650", "confidence": 0.11, "n_returned": 200, "prob_sum": 1.0, "top5": [["650", 0.12], ["651", 0.04], ["627", 0.04], ["640", 0.03], ["647", 0.03]]}, {"index": 32, "template": "reasoning", "variant": "two_step", "n": 10, "position": 10, "rep": 3, "seed": 61003, "labels": "value", "keep_probs": true, "correct_key": "616", "status": 200, "attempts": [200], "server_ms": 79.0, "wall_ms": 275.5, "in_tok": 402, "out_tok": 109, "request_id": "req_01a0aed181a57e169ae1ecfebb935221", "correct_value": "616", "state": "Warehouse log: each case holds 23 cans. Today we received 29 full cases plus 6 loose cans. During unloading, 57 cans were damaged and thrown away.", "p_correct": 0.13, "rank": 3, "ties_at_pc": 1, "argmax_correct": false, "choice": "610", "confidence": 0.27, "n_returned": 10, "prob_sum": 1.0, "top5": [["610", 0.34], ["614", 0.32], ["616", 0.13], ["607", 0.13], ["313", 0.04]], "probabilities": {"727": 0.01, "626": 0.01, "399": 0.01, "616": 0.13, "795": 0.0, "781": 0.01, "610": 0.34, "313": 0.04, "607": 0.13, "614": 0.32}}, {"index": 33, "template": "lexical", "variant": null, "n": 200, "position": 50, "rep": 0, "seed": 30000, "labels": "value", "keep_probs": false, "correct_key": "GA-8933", "status": 200, "attempts": [200], "server_ms": 151.0, "wall_ms": 337.4, "in_tok": 2535, "out_tok": 2435, "request_id": "req_01a0aed181ed7ce88bc382a781d37587", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GA-8933", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GA-8933", 1.0], ["EA-2082", 0.0], ["LQ-6050", 0.0], ["VP-5300", 0.0], ["ZM-4427", 0.0]]}, {"index": 34, "template": "reasoning", "variant": "two_step", "n": 10, "position": 5, "rep": 1, "seed": 61001, "labels": "value", "keep_probs": true, "correct_key": "635", "status": 200, "attempts": [200], "server_ms": 54.0, "wall_ms": 242.1, "in_tok": 403, "out_tok": 109, "request_id": "req_01a0aed1822e7d549a1cc00454bbf15d", "correct_value": "635", "state": "Warehouse log: each case holds 23 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.04, "rank": 3, "ties_at_pc": 0, "argmax_correct": false, "choice": "634", "confidence": 0.83, "n_returned": 10, "prob_sum": 1.0, "top5": [["634", 0.85], ["633", 0.060000000000000005], ["635", 0.04], ["617", 0.03], ["434", 0.01]], "probabilities": {"634": 0.85, "617": 0.03, "635": 0.04, "633": 0.060000000000000005, "750": 0.0, "434": 0.01, "239": 0.0, "412": 0.0, "815": 0.0, "601": 0.01}}, {"index": 35, "template": "reasoning", "variant": "two_step", "n": 200, "position": 190, "rep": 3, "seed": 80003, "labels": "value", "keep_probs": false, "correct_key": "418", "status": 200, "attempts": [200], "server_ms": 133.0, "wall_ms": 331.0, "in_tok": 1905, "out_tok": 1801, "request_id": "req_01a0aed182567b7db7d061b5fa22f426", "correct_value": "418", "state": "Warehouse log: each crate holds 15 bottles. Today we received 29 full crates plus 13 loose bottles. During unloading, 30 bottles were damaged and thrown away.", "p_correct": 0.01, "rank": 26, "ties_at_pc": 35, "argmax_correct": false, "choice": "327", "confidence": 0.06, "n_returned": 200, "prob_sum": 1.0, "top5": [["327", 0.06999999999999999], ["249", 0.03], ["507", 0.03], ["171", 0.03], ["520", 0.03]]}, {"index": 36, "template": "reasoning", "variant": "two_step", "n": 200, "position": 190, "rep": 0, "seed": 80000, "labels": "value", "keep_probs": false, "correct_key": "356", "status": 200, "attempts": [200], "server_ms": 83.0, "wall_ms": 281.2, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed182b87318adaf9d1bde4010d4", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.04, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "357", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["357", 0.05], ["356", 0.04], ["369", 0.03], ["240", 0.03], ["88", 0.03]]}, {"index": 37, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 5, "seed": 80005, "labels": "value", "keep_probs": false, "correct_key": "457", "status": 200, "attempts": [200], "server_ms": 83.0, "wall_ms": 277.7, "in_tok": 1917, "out_tok": 1813, "request_id": "req_01a0aed183267269be0cdf912c246ceb", "correct_value": "457", "state": "Warehouse log: each crate holds 25 bottles. Today we received 20 full crates plus 12 loose bottles. During unloading, 55 bottles were damaged and thrown away.", "p_correct": 0.01, "rank": 21, "ties_at_pc": 52, "argmax_correct": false, "choice": "295", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["455", 0.03], ["393", 0.03], ["445", 0.03], ["305", 0.03], ["458", 0.03]]}, {"index": 38, "template": "lexical", "variant": null, "n": 200, "position": 10, "rep": 2, "seed": 30002, "labels": "value", "keep_probs": false, "correct_key": "AV-9496", "status": 200, "attempts": [200], "server_ms": 105.0, "wall_ms": 292.9, "in_tok": 2547, "out_tok": 2447, "request_id": "req_01a0aed18343709c8d046503c989edb0", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "AV-9496", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["AV-9496", 1.0], ["SC-5640", 0.0], ["SQ-6348", 0.0], ["KQ-7883", 0.0], ["KC-8328", 0.0]]}, {"index": 39, "template": "lexical", "variant": null, "n": 50, "position": 50, "rep": 1, "seed": 15001, "labels": "value", "keep_probs": true, "correct_key": "ZS-5465", "status": 200, "attempts": [200], "server_ms": 66.0, "wall_ms": 268.3, "in_tok": 877, "out_tok": 628, "request_id": "req_01a0aed183a97986845c1476516d7f1e", "correct_value": "ZS-5465", "state": "Support ticket from Elena Brandt. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is ZS-5465. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "ZS-5465", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["ZS-5465", 1.0], ["VL-9183", 0.0], ["TR-5200", 0.0], ["ZE-4016", 0.0], ["MJ-8925", 0.0]], "probabilities": {"VL-9183": 0.0, "TR-5200": 0.0, "ZE-4016": 0.0, "MJ-8925": 0.0, "KB-6796": 0.0, "ZM-8230": 0.0, "MJ-8931": 0.0, "VQ-4725": 0.0, "WS-9676": 0.0, "CS-9022": 0.0, "ZS-5465": 1.0, "ZG-6390": 0.0, "TQ-7252": 0.0, "KW-5623": 0.0, "LQ-3836": 0.0, "DL-2281": 0.0, "UG-6734": 0.0, "KQ-2965": 0.0, "GH-8434": 0.0, "SQ-8294": 0.0, "WN-7040": 0.0, "LH-8546": 0.0, "RS-1008": 0.0, "MU-3899": 0.0, "GM-7288": 0.0, "AN-2754": 0.0, "BH-6847": 0.0, "BA-8802": 0.0, "AY-8082": 0.0, "YC-2256": 0.0, "DY-3944": 0.0, "HY-8204": 0.0, "FV-4057": 0.0, "QZ-8317": 0.0, "VD-4687": 0.0, "CC-7674": 0.0, "WC-5737": 0.0, "QD-4155": 0.0, "ZX-2850": 0.0, "DA-9161": 0.0, "RH-8863": 0.0, "BU-1426": 0.0, "YU-9436": 0.0, "CP-4922": 0.0, "HN-1409": 0.0, "DV-2939": 0.0, "XF-5580": 0.0, "PB-4779": 0.0, "XG-3601": 0.0, "PP-3232": 0.0}}, {"index": 40, "template": "lexical", "variant": null, "n": 50, "position": 50, "rep": 0, "seed": 15000, "labels": "value", "keep_probs": true, "correct_key": "UP-4104", "status": 200, "attempts": [200], "server_ms": 64.0, "wall_ms": 261.7, "in_tok": 877, "out_tok": 627, "request_id": "req_01a0aed183d77763ae03c85f1ef3697a", "correct_value": "UP-4104", "state": "Support ticket from Tomasz Kowal. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is UP-4104. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "UP-4104", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["UP-4104", 1.0], ["EF-3632", 0.0], ["WZ-1138", 0.0], ["RW-3131", 0.0], ["XL-3525", 0.0]], "probabilities": {"EF-3632": 0.0, "WZ-1138": 0.0, "RW-3131": 0.0, "XL-3525": 0.0, "VF-6343": 0.0, "UZ-9533": 0.0, "KE-3677": 0.0, "HB-4149": 0.0, "CJ-1676": 0.0, "BR-1314": 0.0, "QH-8566": 0.0, "BC-1079": 0.0, "ZW-5226": 0.0, "HR-7860": 0.0, "EL-7342": 0.0, "ZX-8990": 0.0, "GD-7424": 0.0, "QW-8129": 0.0, "ZA-3692": 0.0, "QU-3773": 0.0, "PG-3751": 0.0, "SC-9026": 0.0, "ZY-7820": 0.0, "MB-9859": 0.0, "XB-3976": 0.0, "QS-4387": 0.0, "GT-2763": 0.0, "VC-7210": 0.0, "ZY-9031": 0.0, "XH-4616": 0.0, "KD-5589": 0.0, "YG-8095": 0.0, "AZ-7541": 0.0, "CG-3947": 0.0, "UG-3857": 0.0, "NL-3312": 0.0, "UP-4104": 1.0, "SE-6280": 0.0, "ZY-2844": 0.0, "SW-1294": 0.0, "ZK-5437": 0.0, "SV-7637": 0.0, "WH-7681": 0.0, "MY-2610": 0.0, "PR-1653": 0.0, "GM-6470": 0.0, "FW-4993": 0.0, "MT-3365": 0.0, "HV-9164": 0.0, "LB-9442": 0.0}}, {"index": 41, "template": "reasoning", "variant": "two_step", "n": 50, "position": 25, "rep": 2, "seed": 65002, "labels": "value", "keep_probs": true, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 107.0, "wall_ms": 296.5, "in_tok": 717, "out_tok": 463, "request_id": "req_01a0aed18445764bb9c94e42da5b78a7", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.67, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "267", "confidence": 0.65, "n_returned": 50, "prob_sum": 0.99, "top5": [["267", 0.67], ["265", 0.22], ["266", 0.08], ["270", 0.01], ["268", 0.01]], "probabilities": {"37": 0.0, "630": 0.0, "356": 0.0, "167": 0.0, "259": 0.0, "367": 0.0, "380": 0.0, "267": 0.67, "270": 0.01, "596": 0.0, "77": 0.0, "549": 0.0, "265": 0.22, "269": 0.0, "385": 0.0, "563": 0.0, "400": 0.0, "283": 0.0, "264": 0.0, "353": 0.0, "638": 0.0, "291": 0.0, "438": 0.0, "219": 0.0, "47": 0.0, "268": 0.01, "193": 0.0, "217": 0.0, "612": 0.0, "257": 0.0, "81": 0.0, "277": 0.0, "266": 0.08, "419": 0.0, "307": 0.0, "62": 0.0, "128": 0.0, "243": 0.0, "664": 0.0, "201": 0.0, "227": 0.0, "124": 0.0, "133": 0.0, "487": 0.0, "411": 0.0, "477": 0.0, "255": 0.0, "478": 0.0, "251": 0.0, "27": 0.0}}, {"index": 42, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 0, "seed": 80000, "labels": "value", "keep_probs": false, "correct_key": "356", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 260.1, "in_tok": 1898, "out_tok": 1793, "request_id": "req_01a0aed184687cc1b6848621c069be58", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.03, "rank": 5, "ties_at_pc": 6, "argmax_correct": false, "choice": "88", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["88", 0.05], ["357", 0.05], ["369", 0.04], ["238", 0.04], ["196", 0.03]]}, {"index": 43, "template": "lexical", "variant": null, "n": 200, "position": 150, "rep": 0, "seed": 30000, "labels": "value", "keep_probs": false, "correct_key": "GA-8933", "status": 200, "attempts": [200], "server_ms": 63.0, "wall_ms": 263.5, "in_tok": 2535, "out_tok": 2435, "request_id": "req_01a0aed184b2765ab8893a0a15eda4c5", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GA-8933", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GA-8933", 1.0], ["LR-8874", 0.0], ["AT-2444", 0.0], ["MU-9308", 0.0], ["XU-9030", 0.0]]}, {"index": 44, "template": "reasoning", "variant": "two_step", "n": 200, "position": 10, "rep": 1, "seed": 80001, "labels": "value", "keep_probs": false, "correct_key": "283", "status": 200, "attempts": [200], "server_ms": 163.0, "wall_ms": 357.3, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed184dc7f38a901abc1d528f465", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.04, "rank": 4, "ties_at_pc": 1, "argmax_correct": false, "choice": "293", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["293", 0.060000000000000005], ["223", 0.05], ["256", 0.05], ["283", 0.04], ["297", 0.04]]}, {"index": 45, "template": "reasoning", "variant": "two_step", "n": 10, "position": 5, "rep": 3, "seed": 61003, "labels": "value", "keep_probs": true, "correct_key": "616", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 259.9, "in_tok": 402, "out_tok": 109, "request_id": "req_01a0aed18562792b969f4e6fc6e490cf", "correct_value": "616", "state": "Warehouse log: each case holds 23 cans. Today we received 29 full cases plus 6 loose cans. During unloading, 57 cans were damaged and thrown away.", "p_correct": 0.09, "rank": 4, "ties_at_pc": 0, "argmax_correct": false, "choice": "610", "confidence": 0.4, "n_returned": 10, "prob_sum": 1.0, "top5": [["610", 0.46], ["614", 0.19], ["607", 0.14], ["616", 0.09], ["313", 0.06]], "probabilities": {"399": 0.03, "614": 0.19, "626": 0.02, "313": 0.06, "795": 0.0, "610": 0.46, "727": 0.01, "781": 0.0, "616": 0.09, "607": 0.14}}, {"index": 46, "template": "reasoning", "variant": "two_step", "n": 10, "position": 1, "rep": 1, "seed": 61001, "labels": "value", "keep_probs": true, "correct_key": "635", "status": 200, "attempts": [200], "server_ms": 48.0, "wall_ms": 232.5, "in_tok": 403, "out_tok": 109, "request_id": "req_01a0aed1856a7097974a26b89f77fdb0", "correct_value": "635", "state": "Warehouse log: each case holds 23 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.08, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "634", "confidence": 0.84, "n_returned": 10, "prob_sum": 1.0, "top5": [["634", 0.87], ["635", 0.08], ["633", 0.03], ["601", 0.01], ["617", 0.01]], "probabilities": {"634": 0.87, "601": 0.01, "635": 0.08, "633": 0.03, "434": 0.0, "750": 0.0, "239": 0.0, "412": 0.0, "815": 0.0, "617": 0.01}}, {"index": 47, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 3, "seed": 80003, "labels": "value", "keep_probs": false, "correct_key": "418", "status": 200, "attempts": [200], "server_ms": 61.0, "wall_ms": 257.7, "in_tok": 1905, "out_tok": 1801, "request_id": "req_01a0aed185b77b20ba93e76d9599f725", "correct_value": "418", "state": "Warehouse log: each crate holds 15 bottles. Today we received 29 full crates plus 13 loose bottles. During unloading, 30 bottles were damaged and thrown away.", "p_correct": 0.02, "rank": 9, "ties_at_pc": 18, "argmax_correct": false, "choice": "327", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["327", 0.060000000000000005], ["328", 0.05], ["507", 0.04], ["385", 0.04], ["495", 0.03]]}, {"index": 48, "template": "lexical", "variant": null, "n": 200, "position": 1, "rep": 1, "seed": 30001, "labels": "value", "keep_probs": false, "correct_key": "VU-5695", "status": 200, "attempts": [200], "server_ms": 92.0, "wall_ms": 286.1, "in_tok": 2546, "out_tok": 2445, "request_id": "req_01a0aed1863e7a278f9a7e556c14d02d", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VU-5695", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VU-5695", 1.0], ["XB-6521", 0.0], ["PD-9349", 0.0], ["BL-7527", 0.0], ["DF-4287", 0.0]]}, {"index": 49, "template": "reasoning", "variant": "two_step", "n": 50, "position": 50, "rep": 0, "seed": 65000, "labels": "value", "keep_probs": true, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 438.0, "wall_ms": 623.7, "in_tok": 716, "out_tok": 461, "request_id": "req_01a0aed1877b71f99dd70d0c4ac5f3b4", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.71, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "279", "confidence": 0.7, "n_returned": 50, "prob_sum": 1.0, "top5": [["279", 0.71], ["277", 0.06999999999999999], ["281", 0.06999999999999999], ["278", 0.04], ["280", 0.04]], "probabilities": {"159": 0.0, "525": 0.0, "590": 0.0, "500": 0.0, "398": 0.0, "67": 0.0, "278": 0.04, "30": 0.0, "394": 0.0, "233": 0.01, "473": 0.0, "123": 0.0, "381": 0.0, "39": 0.0, "292": 0.0, "279": 0.71, "625": 0.0, "81": 0.0, "179": 0.0, "280": 0.04, "12": 0.0, "364": 0.0, "419": 0.0, "104": 0.0, "528": 0.0, "61": 0.0, "376": 0.0, "246": 0.0, "289": 0.01, "277": 0.06999999999999999, "651": 0.0, "305": 0.0, "192": 0.0, "132": 0.0, "281": 0.06999999999999999, "103": 0.0, "325": 0.0, "253": 0.02, "273": 0.01, "196": 0.0, "266": 0.01, "56": 0.0, "591": 0.0, "539": 0.0, "269": 0.0, "38": 0.0, "116": 0.0, "190": 0.0, "508": 0.0, "379": 0.01}}, {"index": 50, "template": "reasoning", "variant": "two_step", "n": 200, "position": 10, "rep": 5, "seed": 80005, "labels": "value", "keep_probs": false, "correct_key": "457", "status": 200, "attempts": [200], "server_ms": 96.0, "wall_ms": 283.3, "in_tok": 1917, "out_tok": 1813, "request_id": "req_01a0aed186657dddbed8f8b9947045c0", "correct_value": "457", "state": "Warehouse log: each crate holds 25 bottles. Today we received 20 full crates plus 12 loose bottles. During unloading, 55 bottles were damaged and thrown away.", "p_correct": 0.01, "rank": 26, "ties_at_pc": 37, "argmax_correct": false, "choice": "393", "confidence": 0.02, "n_returned": 200, "prob_sum": 1.0, "top5": [["393", 0.04], ["455", 0.04], ["445", 0.04], ["617", 0.03], ["295", 0.03]]}, {"index": 51, "template": "lexical", "variant": null, "n": 200, "position": 10, "rep": 1, "seed": 30001, "labels": "value", "keep_probs": false, "correct_key": "VU-5695", "status": 200, "attempts": [200], "server_ms": 110.0, "wall_ms": 308.4, "in_tok": 2546, "out_tok": 2445, "request_id": "req_01a0aed186ba71e9b535bdeb97d3c87f", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VU-5695", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VU-5695", 1.0], ["XB-6521", 0.0], ["PD-9349", 0.0], ["BL-7527", 0.0], ["DF-4287", 0.0]]}, {"index": 52, "template": "lexical", "variant": null, "n": 200, "position": 200, "rep": 1, "seed": 30001, "labels": "value", "keep_probs": false, "correct_key": "VU-5695", "status": 200, "attempts": [200], "server_ms": 78.0, "wall_ms": 276.7, "in_tok": 2546, "out_tok": 2445, "request_id": "req_01a0aed187647b54891deeafc31828be", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VU-5695", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VU-5695", 1.0], ["FX-4725", 0.0], ["KV-3354", 0.0], ["EG-4930", 0.0], ["WK-1549", 0.0]]}, {"index": 53, "template": "lexical", "variant": null, "n": 200, "position": 10, "rep": 0, "seed": 30000, "labels": "value", "keep_probs": false, "correct_key": "GA-8933", "status": 200, "attempts": [200], "server_ms": 56.0, "wall_ms": 246.9, "in_tok": 2535, "out_tok": 2435, "request_id": "req_01a0aed1878374008b3edce60ab862e7", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GA-8933", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GA-8933", 1.0], ["EN-1600", 0.0], ["YD-2719", 0.0], ["QP-6357", 0.0], ["DQ-6877", 0.0]]}, {"index": 54, "template": "reasoning", "variant": "two_step", "n": 200, "position": 10, "rep": 3, "seed": 80003, "labels": "value", "keep_probs": false, "correct_key": "418", "status": 200, "attempts": [200], "server_ms": 54.0, "wall_ms": 255.3, "in_tok": 1905, "out_tok": 1801, "request_id": "req_01a0aed187f379dea1d6d46fed3aefd4", "correct_value": "418", "state": "Warehouse log: each crate holds 15 bottles. Today we received 29 full crates plus 13 loose bottles. During unloading, 30 bottles were damaged and thrown away.", "p_correct": 0.01, "rank": 19, "ties_at_pc": 47, "argmax_correct": false, "choice": "495", "confidence": 0.03, "n_returned": 200, "prob_sum": 0.99, "top5": [["495", 0.04], ["327", 0.04], ["417", 0.04], ["313", 0.04], ["328", 0.03]]}, {"index": 55, "template": "reasoning", "variant": "two_step", "n": 50, "position": 1, "rep": 3, "seed": 65003, "labels": "value", "keep_probs": true, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 72.0, "wall_ms": 277.2, "in_tok": 722, "out_tok": 467, "request_id": "req_01a0aed188817267a21dea85619a8ca5", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.09999999999999999, "rank": 3, "ties_at_pc": 0, "argmax_correct": false, "choice": "280", "confidence": 0.53, "n_returned": 50, "prob_sum": 0.99, "top5": [["280", 0.55], ["282", 0.14], ["281", 0.09999999999999999], ["279", 0.05], ["262", 0.04]], "probabilities": {"217": 0.0, "102": 0.0, "610": 0.0, "382": 0.0, "243": 0.02, "262": 0.04, "282": 0.14, "566": 0.0, "181": 0.0, "143": 0.01, "617": 0.0, "381": 0.0, "281": 0.09999999999999999, "55": 0.0, "581": 0.0, "480": 0.0, "182": 0.0, "626": 0.0, "279": 0.05, "623": 0.0, "200": 0.0, "653": 0.0, "343": 0.0, "536": 0.0, "82": 0.0, "271": 0.01, "319": 0.0, "178": 0.0, "666": 0.0, "140": 0.0, "660": 0.0, "263": 0.01, "141": 0.01, "323": 0.0, "122": 0.0, "280": 0.55, "266": 0.01, "283": 0.01, "368": 0.0, "130": 0.0, "198": 0.0, "525": 0.0, "244": 0.0, "359": 0.0, "300": 0.01, "291": 0.01, "598": 0.0, "296": 0.01, "396": 0.0, "236": 0.0}}, {"index": 56, "template": "reasoning", "variant": "two_step", "n": 10, "position": 5, "rep": 2, "seed": 61002, "labels": "value", "keep_probs": true, "correct_key": "748", "status": 200, "attempts": [200], "server_ms": 48.0, "wall_ms": 241.3, "in_tok": 403, "out_tok": 110, "request_id": "req_01a0aed188827d2f8d7aca7454b98aae", "correct_value": "748", "state": "Warehouse log: each bag holds 34 bolts. Today we received 23 full bags plus 7 loose bolts. During unloading, 41 bolts were damaged and thrown away.", "p_correct": 0.51, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "748", "confidence": 0.45, "n_returned": 10, "prob_sum": 1.0, "top5": [["748", 0.51], ["701", 0.23], ["714", 0.14], ["750", 0.04], ["586", 0.03]], "probabilities": {"586": 0.03, "981": 0.0, "400": 0.01, "621": 0.03, "848": 0.01, "1054": 0.0, "748": 0.51, "701": 0.23, "750": 0.04, "714": 0.14}}, {"index": 57, "template": "lexical", "variant": null, "n": 50, "position": 1, "rep": 0, "seed": 15000, "labels": "value", "keep_probs": true, "correct_key": "UP-4104", "status": 200, "attempts": [200], "server_ms": 90.0, "wall_ms": 277.5, "in_tok": 877, "out_tok": 627, "request_id": "req_01a0aed188c67ab3a657d880d10fb9cf", "correct_value": "UP-4104", "state": "Support ticket from Tomasz Kowal. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is UP-4104. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "UP-4104", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["UP-4104", 1.0], ["MY-2610", 0.0], ["XB-3976", 0.0], ["ZY-9031", 0.0], ["NL-3312", 0.0]], "probabilities": {"MY-2610": 0.0, "XB-3976": 0.0, "ZY-9031": 0.0, "NL-3312": 0.0, "KD-5589": 0.0, "HV-9164": 0.0, "EL-7342": 0.0, "ZA-3692": 0.0, "XL-3525": 0.0, "WH-7681": 0.0, "UG-3857": 0.0, "GT-2763": 0.0, "LB-9442": 0.0, "WZ-1138": 0.0, "UZ-9533": 0.0, "YG-8095": 0.0, "SC-9026": 0.0, "QU-3773": 0.0, "QS-4387": 0.0, "QW-8129": 0.0, "MT-3365": 0.0, "EF-3632": 0.0, "CJ-1676": 0.0, "FW-4993": 0.0, "RW-3131": 0.0, "SW-1294": 0.0, "ZY-7820": 0.0, "PR-1653": 0.0, "ZY-2844": 0.0, "SV-7637": 0.0, "GD-7424": 0.0, "CG-3947": 0.0, "QH-8566": 0.0, "GM-6470": 0.0, "ZK-5437": 0.0, "ZX-8990": 0.0, "VC-7210": 0.0, "SE-6280": 0.0, "UP-4104": 1.0, "XH-4616": 0.0, "HB-4149": 0.0, "BC-1079": 0.0, "BR-1314": 0.0, "HR-7860": 0.0, "AZ-7541": 0.0, "PG-3751": 0.0, "ZW-5226": 0.0, "MB-9859": 0.0, "KE-3677": 0.0, "VF-6343": 0.0}}, {"index": 58, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 1, "seed": 80001, "labels": "value", "keep_probs": false, "correct_key": "283", "status": 200, "attempts": [200], "server_ms": 70.0, "wall_ms": 271.0, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed188f47d5592ae72ea93b38aaa", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.060000000000000005, "rank": 2, "ties_at_pc": 1, "argmax_correct": false, "choice": "297", "confidence": 0.05, "n_returned": 200, "prob_sum": 0.99, "top5": [["297", 0.06999999999999999], ["283", 0.060000000000000005], ["293", 0.060000000000000005], ["316", 0.04], ["223", 0.04]]}, {"index": 59, "template": "reasoning", "variant": "two_step", "n": 10, "position": 10, "rep": 1, "seed": 61001, "labels": "value", "keep_probs": true, "correct_key": "635", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 263.9, "in_tok": 403, "out_tok": 109, "request_id": "req_01a0aed18971764fa134f1935d640e99", "correct_value": "635", "state": "Warehouse log: each case holds 23 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.09, "rank": 4, "ties_at_pc": 0, "argmax_correct": false, "choice": "633", "confidence": 0.43, "n_returned": 10, "prob_sum": 1.0, "top5": [["633", 0.49], ["634", 0.27], ["617", 0.14], ["635", 0.09], ["601", 0.01]], "probabilities": {"635": 0.09, "601": 0.01, "617": 0.14, "750": 0.0, "634": 0.27, "239": 0.0, "434": 0.0, "412": 0.0, "633": 0.49, "815": 0.0}}, {"index": 60, "template": "reasoning", "variant": "two_step", "n": 10, "position": 5, "rep": 0, "seed": 61000, "labels": "value", "keep_probs": true, "correct_key": "271", "status": 200, "attempts": [200], "server_ms": 93.0, "wall_ms": 290.1, "in_tok": 402, "out_tok": 107, "request_id": "req_01a0aed189917f57a5d5d43c96e2c014", "correct_value": "271", "state": "Warehouse log: each box holds 17 pens. Today we received 17 full boxs plus 16 loose pens. During unloading, 34 pens were damaged and thrown away.", "p_correct": 0.15, "rank": 3, "ties_at_pc": 0, "argmax_correct": false, "choice": "237", "confidence": 0.27, "n_returned": 10, "prob_sum": 1.0, "top5": [["237", 0.35000000000000003], ["254", 0.25], ["271", 0.15], ["289", 0.11], ["228", 0.08]], "probabilities": {"237": 0.35000000000000003, "228": 0.08, "427": 0.0, "289": 0.11, "305": 0.03, "425": 0.01, "48": 0.01, "271": 0.15, "73": 0.01, "254": 0.25}}, {"index": 61, "template": "reasoning", "variant": "two_step", "n": 10, "position": 1, "rep": 2, "seed": 61002, "labels": "value", "keep_probs": true, "correct_key": "748", "status": 200, "attempts": [200], "server_ms": 85.0, "wall_ms": 273.8, "in_tok": 403, "out_tok": 110, "request_id": "req_01a0aed189de76249ef8cf7e0a7682f4", "correct_value": "748", "state": "Warehouse log: each bag holds 34 bolts. Today we received 23 full bags plus 7 loose bolts. During unloading, 41 bolts were damaged and thrown away.", "p_correct": 0.51, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "748", "confidence": 0.46, "n_returned": 10, "prob_sum": 1.0, "top5": [["748", 0.51], ["701", 0.32], ["714", 0.07], ["750", 0.04], ["586", 0.03]], "probabilities": {"586": 0.03, "400": 0.01, "621": 0.02, "701": 0.32, "750": 0.04, "714": 0.07, "1054": 0.0, "981": 0.0, "748": 0.51, "848": 0.0}}, {"index": 62, "template": "lexical", "variant": null, "n": 200, "position": 100, "rep": 1, "seed": 30001, "labels": "value", "keep_probs": false, "correct_key": "VU-5695", "status": 200, "attempts": [200], "server_ms": 116.0, "wall_ms": 315.1, "in_tok": 2546, "out_tok": 2445, "request_id": "req_01a0aed18a0779b79f8a756806f1a345", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VU-5695", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VU-5695", 1.0], ["WQ-1997", 0.0], ["CQ-2463", 0.0], ["NA-8047", 0.0], ["YD-6329", 0.0]]}, {"index": 63, "template": "reasoning", "variant": "two_step", "n": 200, "position": 190, "rep": 1, "seed": 80001, "labels": "value", "keep_probs": false, "correct_key": "283", "status": 200, "attempts": [200], "server_ms": 54.0, "wall_ms": 240.9, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed18a767ed1980f2723712e7788", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.18000000000000002, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "283", "confidence": 0.16, "n_returned": 200, "prob_sum": 1.0, "top5": [["283", 0.18000000000000002], ["223", 0.09], ["253", 0.04], ["293", 0.04], ["198", 0.03]]}, {"index": 64, "template": "reasoning", "variant": "two_step", "n": 10, "position": 1, "rep": 0, "seed": 61000, "labels": "value", "keep_probs": true, "correct_key": "271", "status": 200, "attempts": [200], "server_ms": 50.0, "wall_ms": 243.2, "in_tok": 402, "out_tok": 107, "request_id": "req_01a0aed18aaf78bd9ad04ea19858db23", "correct_value": "271", "state": "Warehouse log: each box holds 17 pens. Today we received 17 full boxs plus 16 loose pens. During unloading, 34 pens were damaged and thrown away.", "p_correct": 0.17, "rank": 2, "ties_at_pc": 1, "argmax_correct": false, "choice": "237", "confidence": 0.47, "n_returned": 10, "prob_sum": 1.0, "top5": [["237", 0.51], ["254", 0.17], ["271", 0.17], ["228", 0.07], ["289", 0.03]], "probabilities": {"254": 0.17, "228": 0.07, "237": 0.51, "425": 0.01, "305": 0.02, "73": 0.01, "427": 0.0, "271": 0.17, "289": 0.03, "48": 0.01}}, {"index": 65, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 1, "seed": 80001, "labels": "value", "keep_probs": false, "correct_key": "283", "status": 200, "attempts": [200], "server_ms": 84.0, "wall_ms": 269.5, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed18aec7fb0b742c54e3702f827", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.14, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "283", "confidence": 0.13, "n_returned": 200, "prob_sum": 1.0, "top5": [["283", 0.14], ["273", 0.060000000000000005], ["223", 0.060000000000000005], ["253", 0.05], ["297", 0.04]]}, {"index": 66, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 2, "seed": 80002, "labels": "value", "keep_probs": false, "correct_key": "921", "status": 200, "attempts": [200], "server_ms": 56.0, "wall_ms": 254.5, "in_tok": 2000, "out_tok": 1897, "request_id": "req_01a0aed18b3e780eb08ac216c6e99feb", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.0, "rank": 72, "ties_at_pc": 128, "argmax_correct": false, "choice": "919", "confidence": 0.03, "n_returned": 200, "prob_sum": 0.99, "top5": [["919", 0.04], ["607", 0.04], ["729", 0.04], ["731", 0.04], ["603", 0.03]]}, {"index": 67, "template": "lexical", "variant": null, "n": 10, "position": 5, "rep": 1, "seed": 11001, "labels": "value", "keep_probs": true, "correct_key": "HW-4944", "status": 200, "attempts": [200], "server_ms": 250.0, "wall_ms": 438.8, "in_tok": 432, "out_tok": 142, "request_id": "req_01a0aed18b687064b7b36ac6b73d9987", "correct_value": "HW-4944", "state": "Support ticket from Tomasz Kowal. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is HW-4944. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "HW-4944", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["HW-4944", 1.0], ["SD-7093", 0.0], ["EH-3342", 0.0], ["GK-1329", 0.0], ["PA-2699", 0.0]], "probabilities": {"SD-7093": 0.0, "EH-3342": 0.0, "GK-1329": 0.0, "PA-2699": 0.0, "TK-3090": 0.0, "AF-5927": 0.0, "UD-6618": 0.0, "HW-4944": 1.0, "LY-8273": 0.0, "PH-8850": 0.0}}, {"index": 68, "template": "reasoning", "variant": "two_step", "n": 50, "position": 25, "rep": 1, "seed": 65001, "labels": "value", "keep_probs": true, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 55.0, "wall_ms": 251.1, "in_tok": 719, "out_tok": 466, "request_id": "req_01a0aed18ba57a15b1c3b2b455f45fba", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.7, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "477", "confidence": 0.68, "n_returned": 50, "prob_sum": 1.0, "top5": [["477", 0.7], ["478", 0.16], ["476", 0.08], ["475", 0.04], ["479", 0.01]], "probabilities": {"477": 0.7, "512": 0.0, "363": 0.0, "509": 0.0, "815": 0.0, "479": 0.01, "498": 0.0, "487": 0.0, "672": 0.0, "157": 0.0, "211": 0.0, "159": 0.0, "268": 0.0, "475": 0.04, "445": 0.0, "850": 0.0, "357": 0.0, "220": 0.0, "765": 0.0, "393": 0.0, "130": 0.0, "577": 0.0, "472": 0.0, "85": 0.0, "392": 0.0, "377": 0.01, "91": 0.0, "142": 0.0, "456": 0.0, "716": 0.0, "162": 0.0, "637": 0.0, "822": 0.0, "478": 0.16, "279": 0.0, "773": 0.0, "685": 0.0, "831": 0.0, "593": 0.0, "631": 0.0, "467": 0.0, "628": 0.0, "96": 0.0, "225": 0.0, "562": 0.0, "563": 0.0, "743": 0.0, "476": 0.08, "482": 0.0, "807": 0.0}}, {"index": 69, "template": "lexical", "variant": null, "n": 200, "position": 50, "rep": 1, "seed": 30001, "labels": "value", "keep_probs": false, "correct_key": "VU-5695", "status": 200, "attempts": [200], "server_ms": 86.0, "wall_ms": 274.7, "in_tok": 2546, "out_tok": 2445, "request_id": "req_01a0aed18c0272ecaae8054734fa50b2", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VU-5695", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VU-5695", 1.0], ["AN-2724", 0.0], ["UC-2236", 0.0], ["MZ-9371", 0.0], ["XL-7077", 0.0]]}, {"index": 70, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 1, "seed": 80001, "labels": "value", "keep_probs": false, "correct_key": "283", "status": 200, "attempts": [200], "server_ms": 62.0, "wall_ms": 264.4, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed18c4575ef841779d74e46b956", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.08, "rank": 1, "ties_at_pc": 1, "argmax_correct": false, "choice": "293", "confidence": 0.07, "n_returned": 200, "prob_sum": 0.99, "top5": [["293", 0.08], ["283", 0.08], ["223", 0.06999999999999999], ["273", 0.04], ["256", 0.04]]}, {"index": 71, "template": "lexical", "variant": null, "n": 10, "position": 1, "rep": 1, "seed": 11001, "labels": "value", "keep_probs": true, "correct_key": "HW-4944", "status": 200, "attempts": [200], "server_ms": 96.0, "wall_ms": 291.4, "in_tok": 432, "out_tok": 142, "request_id": "req_01a0aed18c9e713baf0bf586d9ab622c", "correct_value": "HW-4944", "state": "Support ticket from Tomasz Kowal. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is HW-4944. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "HW-4944", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["HW-4944", 1.0], ["AF-5927", 0.0], ["EH-3342", 0.0], ["GK-1329", 0.0], ["PA-2699", 0.0]], "probabilities": {"AF-5927": 0.0, "EH-3342": 0.0, "GK-1329": 0.0, "PA-2699": 0.0, "TK-3090": 0.0, "HW-4944": 1.0, "SD-7093": 0.0, "LY-8273": 0.0, "PH-8850": 0.0, "UD-6618": 0.0}}, {"index": 72, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 5, "seed": 80005, "labels": "value", "keep_probs": false, "correct_key": "457", "status": 200, "attempts": [200], "server_ms": 94.0, "wall_ms": 282.1, "in_tok": 1917, "out_tok": 1813, "request_id": "req_01a0aed18d107b09a29e06e4e4e2e792", "correct_value": "457", "state": "Warehouse log: each crate holds 25 bottles. Today we received 20 full crates plus 12 loose bottles. During unloading, 55 bottles were damaged and thrown away.", "p_correct": 0.03, "rank": 1, "ties_at_pc": 6, "argmax_correct": false, "choice": "617", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["393", 0.03], ["295", 0.03], ["617", 0.03], ["457", 0.03], ["357", 0.03]]}, {"index": 73, "template": "lexical", "variant": null, "n": 50, "position": 25, "rep": 0, "seed": 15000, "labels": "value", "keep_probs": true, "correct_key": "UP-4104", "status": 200, "attempts": [200], "server_ms": 74.0, "wall_ms": 262.5, "in_tok": 877, "out_tok": 627, "request_id": "req_01a0aed18d20790e858c3a12a3e201e7", "correct_value": "UP-4104", "state": "Support ticket from Tomasz Kowal. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is UP-4104. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "UP-4104", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["UP-4104", 1.0], ["MY-2610", 0.0], ["XL-3525", 0.0], ["ZY-9031", 0.0], ["NL-3312", 0.0]], "probabilities": {"MY-2610": 0.0, "XL-3525": 0.0, "ZY-9031": 0.0, "NL-3312": 0.0, "KD-5589": 0.0, "HV-9164": 0.0, "XB-3976": 0.0, "ZA-3692": 0.0, "VF-6343": 0.0, "WZ-1138": 0.0, "UG-3857": 0.0, "GT-2763": 0.0, "LB-9442": 0.0, "WH-7681": 0.0, "UZ-9533": 0.0, "YG-8095": 0.0, "QU-3773": 0.0, "SC-9026": 0.0, "FW-4993": 0.0, "QW-8129": 0.0, "MT-3365": 0.0, "EF-3632": 0.0, "CJ-1676": 0.0, "QS-4387": 0.0, "RW-3131": 0.0, "SW-1294": 0.0, "PR-1653": 0.0, "ZY-7820": 0.0, "ZY-2844": 0.0, "SV-7637": 0.0, "GD-7424": 0.0, "GM-6470": 0.0, "ZX-8990": 0.0, "ZK-5437": 0.0, "CG-3947": 0.0, "QH-8566": 0.0, "SE-6280": 0.0, "VC-7210": 0.0, "UP-4104": 1.0, "XH-4616": 0.0, "HB-4149": 0.0, "BC-1079": 0.0, "BR-1314": 0.0, "HR-7860": 0.0, "AZ-7541": 0.0, "PG-3751": 0.0, "ZW-5226": 0.0, "MB-9859": 0.0, "KE-3677": 0.0, "EL-7342": 0.0}}, {"index": 74, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 4, "seed": 80004, "labels": "value", "keep_probs": false, "correct_key": "650", "status": 200, "attempts": [200], "server_ms": 93.0, "wall_ms": 294.4, "in_tok": 1938, "out_tok": 1834, "request_id": "req_01a0aed18d4b7b2792914f71be198708", "correct_value": "650", "state": "Warehouse log: each box holds 31 pens. Today we received 22 full boxs plus 5 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 27, "ties_at_pc": 28, "argmax_correct": false, "choice": "651", "confidence": 0.09, "n_returned": 200, "prob_sum": 1.0, "top5": [["651", 0.11], ["629", 0.05], ["647", 0.04], ["640", 0.04], ["627", 0.03]]}, {"index": 75, "template": "lexical", "variant": null, "n": 200, "position": 150, "rep": 1, "seed": 30001, "labels": "value", "keep_probs": false, "correct_key": "VU-5695", "status": 200, "attempts": [200], "server_ms": 65.0, "wall_ms": 261.3, "in_tok": 2546, "out_tok": 2445, "request_id": "req_01a0aed18dc57d5ea8a1afb208fe8b9d", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VU-5695", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VU-5695", 1.0], ["VM-2792", 0.0], ["NV-8727", 0.0], ["PP-5872", 0.0], ["KU-5589", 0.0]]}, {"index": 76, "template": "reasoning", "variant": "two_step", "n": 50, "position": 50, "rep": 1, "seed": 65001, "labels": "value", "keep_probs": true, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 85.0, "wall_ms": 279.8, "in_tok": 719, "out_tok": 466, "request_id": "req_01a0aed18e2e787a91ab7798fca8427c", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.9400000000000001, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "477", "confidence": 0.93, "n_returned": 50, "prob_sum": 0.99, "top5": [["477", 0.9400000000000001], ["478", 0.02], ["475", 0.01], ["377", 0.01], ["476", 0.01]], "probabilities": {"479": 0.0, "363": 0.0, "475": 0.01, "482": 0.0, "822": 0.0, "467": 0.0, "807": 0.0, "393": 0.0, "765": 0.0, "445": 0.0, "593": 0.0, "773": 0.0, "487": 0.0, "509": 0.0, "743": 0.0, "477": 0.9400000000000001, "162": 0.0, "91": 0.0, "850": 0.0, "628": 0.0, "512": 0.0, "279": 0.0, "268": 0.0, "225": 0.0, "637": 0.0, "159": 0.0, "716": 0.0, "392": 0.0, "562": 0.0, "631": 0.0, "377": 0.01, "357": 0.0, "815": 0.0, "157": 0.0, "472": 0.0, "563": 0.0, "672": 0.0, "85": 0.0, "142": 0.0, "130": 0.0, "220": 0.0, "96": 0.0, "456": 0.0, "476": 0.01, "478": 0.02, "498": 0.0, "831": 0.0, "685": 0.0, "577": 0.0, "211": 0.0}}, {"index": 77, "template": "lexical", "variant": null, "n": 10, "position": 1, "rep": 0, "seed": 11000, "labels": "value", "keep_probs": true, "correct_key": "BS-7321", "status": 200, "attempts": [200], "server_ms": 102.0, "wall_ms": 288.9, "in_tok": 430, "out_tok": 142, "request_id": "req_01a0aed18e2c723a88dd33bd65096247", "correct_value": "BS-7321", "state": "Support ticket from Elena Brandt. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is BS-7321. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "BS-7321", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["BS-7321", 1.0], ["MB-9208", 0.0], ["WY-3070", 0.0], ["GY-3999", 0.0], ["TL-4461", 0.0]], "probabilities": {"MB-9208": 0.0, "WY-3070": 0.0, "GY-3999": 0.0, "TL-4461": 0.0, "EY-6117": 0.0, "BS-7321": 1.0, "BM-2721": 0.0, "KE-8778": 0.0, "TK-1302": 0.0, "TH-4469": 0.0}}, {"index": 78, "template": "reasoning", "variant": "two_step", "n": 200, "position": 190, "rep": 5, "seed": 80005, "labels": "value", "keep_probs": false, "correct_key": "457", "status": 200, "attempts": [200], "server_ms": 351.0, "wall_ms": 551.2, "in_tok": 1917, "out_tok": 1813, "request_id": "req_01a0aed18e747445a525bcb461863473", "correct_value": "457", "state": "Warehouse log: each crate holds 25 bottles. Today we received 20 full crates plus 12 loose bottles. During unloading, 55 bottles were damaged and thrown away.", "p_correct": 0.03, "rank": 3, "ties_at_pc": 5, "argmax_correct": false, "choice": "445", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["393", 0.04], ["445", 0.04], ["437", 0.03], ["465", 0.03], ["343", 0.03]]}, {"index": 79, "template": "lexical", "variant": null, "n": 200, "position": 1, "rep": 0, "seed": 30000, "labels": "value", "keep_probs": false, "correct_key": "GA-8933", "status": 200, "attempts": [200], "server_ms": 56.0, "wall_ms": 252.7, "in_tok": 2535, "out_tok": 2435, "request_id": "req_01a0aed18ec97e1581a0974d0a1e0289", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GA-8933", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GA-8933", 1.0], ["XL-3256", 0.0], ["AH-7712", 0.0], ["VF-6215", 0.0], ["GK-8173", 0.0]]}, {"index": 80, "template": "reasoning", "variant": "two_step", "n": 200, "position": 190, "rep": 2, "seed": 80002, "labels": "value", "keep_probs": false, "correct_key": "921", "status": 200, "attempts": [200], "server_ms": 106.0, "wall_ms": 294.4, "in_tok": 2000, "out_tok": 1897, "request_id": "req_01a0aed18f3f71f98fe3dd11ed0896c3", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.03, "rank": 2, "ties_at_pc": 5, "argmax_correct": false, "choice": "729", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["729", 0.04], ["919", 0.03], ["829", 0.03], ["731", 0.03], ["603", 0.03]]}, {"index": 81, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 3, "seed": 80003, "labels": "value", "keep_probs": false, "correct_key": "418", "status": 200, "attempts": [200], "server_ms": 83.0, "wall_ms": 268.2, "in_tok": 1905, "out_tok": 1801, "request_id": "req_01a0aed18f4c79b7b4a9aa8c8832d6d7", "correct_value": "418", "state": "Warehouse log: each crate holds 15 bottles. Today we received 29 full crates plus 13 loose bottles. During unloading, 30 bottles were damaged and thrown away.", "p_correct": 0.0, "rank": 63, "ties_at_pc": 137, "argmax_correct": false, "choice": "495", "confidence": 0.02, "n_returned": 200, "prob_sum": 1.0, "top5": [["495", 0.04], ["327", 0.04], ["317", 0.03], ["500", 0.03], ["318", 0.03]]}, {"index": 82, "template": "lexical", "variant": null, "n": 200, "position": 100, "rep": 0, "seed": 30000, "labels": "value", "keep_probs": false, "correct_key": "GA-8933", "status": 200, "attempts": [200], "server_ms": 76.0, "wall_ms": 277.6, "in_tok": 2535, "out_tok": 2435, "request_id": "req_01a0aed18fcc732b87c89e0d1d8defb4", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GA-8933", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GA-8933", 1.0], ["GE-9771", 0.0], ["SP-6865", 0.0], ["LQ-6050", 0.0], ["DW-9337", 0.0]]}, {"index": 83, "template": "reasoning", "variant": "two_step", "n": 200, "position": 10, "rep": 2, "seed": 80002, "labels": "value", "keep_probs": false, "correct_key": "921", "status": 200, "attempts": [200], "server_ms": 130.0, "wall_ms": 320.0, "in_tok": 2000, "out_tok": 1897, "request_id": "req_01a0aed1906672d5bd726e1eb41a3607", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.02, "rank": 6, "ties_at_pc": 14, "argmax_correct": false, "choice": "729", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["819", 0.04], ["729", 0.04], ["603", 0.03], ["919", 0.03], ["731", 0.03]]}, {"index": 84, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 3, "seed": 80003, "labels": "value", "keep_probs": false, "correct_key": "418", "status": 200, "attempts": [200], "server_ms": 55.0, "wall_ms": 244.6, "in_tok": 1905, "out_tok": 1801, "request_id": "req_01a0aed19067768d94aa85c9e6e53eb4", "correct_value": "418", "state": "Warehouse log: each crate holds 15 bottles. Today we received 29 full crates plus 13 loose bottles. During unloading, 30 bottles were damaged and thrown away.", "p_correct": 0.04, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "417", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["417", 0.060000000000000005], ["418", 0.04], ["318", 0.03], ["500", 0.03], ["313", 0.03]]}, {"index": 85, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 0, "seed": 80000, "labels": "value", "keep_probs": false, "correct_key": "356", "status": 200, "attempts": [200], "server_ms": 66.0, "wall_ms": 265.4, "in_tok": 1898, "out_tok": 1793, "request_id": "req_01a0aed1909b76a9baf6a80ff70b844b", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.04, "rank": 2, "ties_at_pc": 1, "argmax_correct": false, "choice": "88", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["88", 0.05], ["357", 0.04], ["356", 0.04], ["171", 0.03], ["126", 0.03]]}, {"index": 86, "template": "lexical", "variant": null, "n": 10, "position": 10, "rep": 0, "seed": 11000, "labels": "value", "keep_probs": true, "correct_key": "BS-7321", "status": 200, "attempts": [200], "server_ms": 75.0, "wall_ms": 274.8, "in_tok": 430, "out_tok": 142, "request_id": "req_01a0aed190e47eb6b4b48a715f6be31a", "correct_value": "BS-7321", "state": "Support ticket from Elena Brandt. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is BS-7321. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "BS-7321", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["BS-7321", 1.0], ["GY-3999", 0.0], ["TK-1302", 0.0], ["TL-4461", 0.0], ["TH-4469", 0.0]], "probabilities": {"GY-3999": 0.0, "TK-1302": 0.0, "BS-7321": 1.0, "TL-4461": 0.0, "TH-4469": 0.0, "EY-6117": 0.0, "BM-2721": 0.0, "MB-9208": 0.0, "WY-3070": 0.0, "KE-8778": 0.0}}, {"index": 87, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 1, "seed": 80001, "labels": "value", "keep_probs": false, "correct_key": "283", "status": 200, "attempts": [200], "server_ms": 105.0, "wall_ms": 295.0, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed191627bcaa47990a1ba328dcd", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.060000000000000005, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "293", "confidence": 0.07, "n_returned": 200, "prob_sum": 1.0, "top5": [["293", 0.08], ["283", 0.060000000000000005], ["297", 0.04], ["282", 0.04], ["256", 0.03]]}, {"index": 88, "template": "reasoning", "variant": "two_step", "n": 10, "position": 10, "rep": 2, "seed": 61002, "labels": "value", "keep_probs": true, "correct_key": "748", "status": 200, "attempts": [200], "server_ms": 55.0, "wall_ms": 244.5, "in_tok": 403, "out_tok": 110, "request_id": "req_01a0aed191a27a08826210d0de870bb8", "correct_value": "748", "state": "Warehouse log: each bag holds 34 bolts. Today we received 23 full bags plus 7 loose bolts. During unloading, 41 bolts were damaged and thrown away.", "p_correct": 0.13, "rank": 3, "ties_at_pc": 0, "argmax_correct": false, "choice": "701", "confidence": 0.34, "n_returned": 10, "prob_sum": 1.0, "top5": [["701", 0.4], ["714", 0.36], ["748", 0.13], ["621", 0.03], ["750", 0.03]], "probabilities": {"586": 0.02, "714": 0.36, "848": 0.01, "701": 0.4, "748": 0.13, "981": 0.01, "621": 0.03, "1054": 0.0, "400": 0.01, "750": 0.03}}, {"index": 89, "template": "reasoning", "variant": "two_step", "n": 10, "position": 10, "rep": 0, "seed": 61000, "labels": "value", "keep_probs": true, "correct_key": "271", "status": 200, "attempts": [200], "server_ms": 109.0, "wall_ms": 307.7, "in_tok": 402, "out_tok": 107, "request_id": "req_01a0aed191a3703996b4f20500876279", "correct_value": "271", "state": "Warehouse log: each box holds 17 pens. Today we received 17 full boxs plus 16 loose pens. During unloading, 34 pens were damaged and thrown away.", "p_correct": 0.31, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "271", "confidence": 0.22, "n_returned": 10, "prob_sum": 1.0, "top5": [["271", 0.31], ["237", 0.29], ["254", 0.21], ["228", 0.07], ["289", 0.05]], "probabilities": {"425": 0.01, "237": 0.29, "254": 0.21, "305": 0.03, "271": 0.31, "48": 0.01, "73": 0.02, "289": 0.05, "427": 0.0, "228": 0.07}}, {"index": 90, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 2, "seed": 80002, "labels": "value", "keep_probs": false, "correct_key": "921", "status": 200, "attempts": [200], "server_ms": 70.0, "wall_ms": 268.0, "in_tok": 2000, "out_tok": 1897, "request_id": "req_01a0aed191f5740faa0623c8fad012a9", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.02, "rank": 9, "ties_at_pc": 12, "argmax_correct": false, "choice": "729", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["729", 0.04], ["1143", 0.03], ["731", 0.03], ["829", 0.03], ["919", 0.03]]}, {"index": 91, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 2, "seed": 80002, "labels": "value", "keep_probs": false, "correct_key": "921", "status": 200, "attempts": [200], "server_ms": 128.0, "wall_ms": 316.0, "in_tok": 2000, "out_tok": 1897, "request_id": "req_01a0aed19295798e9031a9a08ff42b8a", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.0, "rank": 70, "ties_at_pc": 130, "argmax_correct": false, "choice": "729", "confidence": 0.03, "n_returned": 200, "prob_sum": 0.99, "top5": [["729", 0.05], ["819", 0.04], ["919", 0.03], ["829", 0.03], ["603", 0.03]]}, {"index": 92, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 4, "seed": 80004, "labels": "value", "keep_probs": false, "correct_key": "650", "status": 200, "attempts": [200], "server_ms": 62.0, "wall_ms": 246.5, "in_tok": 1938, "out_tok": 1834, "request_id": "req_01a0aed192917327959b497d56bcfdbb", "correct_value": "650", "state": "Warehouse log: each box holds 31 pens. Today we received 22 full boxs plus 5 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 28, "ties_at_pc": 27, "argmax_correct": false, "choice": "651", "confidence": 0.09, "n_returned": 200, "prob_sum": 1.0, "top5": [["651", 0.09999999999999999], ["629", 0.05], ["627", 0.04], ["647", 0.04], ["557", 0.03]]}, {"index": 93, "template": "lexical", "variant": null, "n": 50, "position": 1, "rep": 2, "seed": 15002, "labels": "value", "keep_probs": true, "correct_key": "SS-8141", "status": 200, "attempts": [200], "server_ms": 85.0, "wall_ms": 286.0, "in_tok": 874, "out_tok": 624, "request_id": "req_01a0aed192db78d8afab2fec822e0832", "correct_value": "SS-8141", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is SS-8141. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "SS-8141", "confidence": 1.0, "n_returned": 50, "prob_sum": 1.0, "top5": [["SS-8141", 1.0], ["KH-8358", 0.0], ["AD-7367", 0.0], ["WB-4163", 0.0], ["YM-3017", 0.0]], "probabilities": {"KH-8358": 0.0, "AD-7367": 0.0, "WB-4163": 0.0, "YM-3017": 0.0, "DG-9216": 0.0, "SL-8701": 0.0, "JS-1666": 0.0, "ZS-1140": 0.0, "MF-9075": 0.0, "EG-7473": 0.0, "RH-9468": 0.0, "SS-8141": 1.0, "KU-6416": 0.0, "PY-8591": 0.0, "TW-9116": 0.0, "MU-1259": 0.0, "TZ-6342": 0.0, "TL-9500": 0.0, "FW-8221": 0.0, "LK-7156": 0.0, "JJ-1379": 0.0, "KS-3726": 0.0, "WB-8736": 0.0, "NT-9311": 0.0, "MN-5423": 0.0, "ND-5422": 0.0, "NT-2034": 0.0, "RU-9816": 0.0, "DT-7722": 0.0, "HF-3915": 0.0, "KM-4035": 0.0, "SG-1015": 0.0, "KB-8333": 0.0, "SE-4698": 0.0, "AE-9512": 0.0, "EV-3579": 0.0, "NA-7727": 0.0, "SH-3775": 0.0, "FV-2437": 0.0, "HR-1686": 0.0, "YQ-9732": 0.0, "YV-9991": 0.0, "WX-4388": 0.0, "ST-8638": 0.0, "SL-3210": 0.0, "ES-9686": 0.0, "LY-2553": 0.0, "BF-7019": 0.0, "VU-6041": 0.0, "SF-7642": 0.0}}, {"index": 94, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 4, "seed": 80004, "labels": "value", "keep_probs": false, "correct_key": "650", "status": 200, "attempts": [200], "server_ms": 64.0, "wall_ms": 257.2, "in_tok": 1938, "out_tok": 1834, "request_id": "req_01a0aed192fd7b879ba82e143ede17dd", "correct_value": "650", "state": "Warehouse log: each box holds 31 pens. Today we received 22 full boxs plus 5 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.02, "rank": 7, "ties_at_pc": 23, "argmax_correct": false, "choice": "651", "confidence": 0.15, "n_returned": 200, "prob_sum": 1.0, "top5": [["651", 0.16], ["647", 0.04], ["629", 0.03], ["640", 0.03], ["653", 0.03]]}, {"index": 95, "template": "lexical", "variant": null, "n": 200, "position": 150, "rep": 2, "seed": 30002, "labels": "value", "keep_probs": false, "correct_key": "AV-9496", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 261.0, "in_tok": 2547, "out_tok": 2447, "request_id": "req_01a0aed1938b759790e3174e1a77c932", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "AV-9496", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["AV-9496", 1.0], ["HU-7886", 0.0], ["WD-6554", 0.0], ["UZ-2201", 0.0], ["QB-1047", 0.0]]}, {"index": 96, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 5, "seed": 80005, "labels": "value", "keep_probs": false, "correct_key": "457", "status": 200, "attempts": [200], "server_ms": 65.0, "wall_ms": 252.7, "in_tok": 1917, "out_tok": 1813, "request_id": "req_01a0aed193c37f7896287b7e880b674a", "correct_value": "457", "state": "Warehouse log: each crate holds 25 bottles. Today we received 20 full crates plus 12 loose bottles. During unloading, 55 bottles were damaged and thrown away.", "p_correct": 0.01, "rank": 16, "ties_at_pc": 61, "argmax_correct": false, "choice": "295", "confidence": 0.02, "n_returned": 200, "prob_sum": 1.0, "top5": [["433", 0.03], ["357", 0.03], ["393", 0.03], ["316", 0.03], ["447", 0.03]]}, {"index": 97, "template": "lexical", "variant": null, "n": 10, "position": 5, "rep": 2, "seed": 11002, "labels": "value", "keep_probs": true, "correct_key": "LX-8614", "status": 200, "attempts": [200], "server_ms": 100.0, "wall_ms": 300.6, "in_tok": 435, "out_tok": 145, "request_id": "req_01a0aed193fb7f35b5b99a63d4a48e7e", "correct_value": "LX-8614", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is LX-8614. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "LX-8614", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["LX-8614", 1.0], ["VB-3704", 0.0], ["LZ-5363", 0.0], ["QS-1005", 0.0], ["CG-5481", 0.0]], "probabilities": {"VB-3704": 0.0, "LZ-5363": 0.0, "QS-1005": 0.0, "LX-8614": 1.0, "CG-5481": 0.0, "WQ-6752": 0.0, "YW-7088": 0.0, "JS-5834": 0.0, "KQ-5489": 0.0, "MY-8856": 0.0}}, {"index": 98, "template": "reasoning", "variant": "two_step", "n": 200, "position": 10, "rep": 4, "seed": 80004, "labels": "value", "keep_probs": false, "correct_key": "650", "status": 200, "attempts": [200], "server_ms": 67.0, "wall_ms": 262.2, "in_tok": 1938, "out_tok": 1834, "request_id": "req_01a0aed19401762c8c6e92666d18e923", "correct_value": "650", "state": "Warehouse log: each box holds 31 pens. Today we received 22 full boxs plus 5 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 54, "ties_at_pc": 146, "argmax_correct": false, "choice": "651", "confidence": 0.11, "n_returned": 200, "prob_sum": 1.0, "top5": [["651", 0.13], ["647", 0.04], ["629", 0.04], ["640", 0.03], ["645", 0.03]]}, {"index": 99, "template": "lexical", "variant": null, "n": 200, "position": 190, "rep": 0, "seed": 30000, "labels": "value", "keep_probs": false, "correct_key": "GA-8933", "status": 200, "attempts": [200], "server_ms": 86.0, "wall_ms": 274.6, "in_tok": 2535, "out_tok": 2435, "request_id": "req_01a0aed1948e77fab190dee79e10fc85", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GA-8933", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GA-8933", 1.0], ["ZC-8824", 0.0], ["GK-8173", 0.0], ["SX-5775", 0.0], ["YP-7195", 0.0]]}, {"index": 100, "template": "lexical", "variant": null, "n": 200, "position": 1, "rep": 2, "seed": 30002, "labels": "value", "keep_probs": false, "correct_key": "AV-9496", "status": 200, "attempts": [200], "server_ms": 68.0, "wall_ms": 256.7, "in_tok": 2547, "out_tok": 2447, "request_id": "req_01a0aed194bf78309edff0c6ceabbd10", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "AV-9496", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["AV-9496", 1.0], ["HU-7886", 0.0], ["WD-6554", 0.0], ["QB-1047", 0.0], ["UZ-2201", 0.0]]}, {"index": 101, "template": "lexical", "variant": null, "n": 10, "position": 1, "rep": 2, "seed": 11002, "labels": "value", "keep_probs": true, "correct_key": "LX-8614", "status": 200, "attempts": [200], "server_ms": 70.0, "wall_ms": 265.7, "in_tok": 435, "out_tok": 145, "request_id": "req_01a0aed1950872d0a425149fe5a659c9", "correct_value": "LX-8614", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is LX-8614. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "LX-8614", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["LX-8614", 1.0], ["CG-5481", 0.0], ["WQ-6752", 0.0], ["QS-1005", 0.0], ["MY-8856", 0.0]], "probabilities": {"CG-5481": 0.0, "WQ-6752": 0.0, "LX-8614": 1.0, "QS-1005": 0.0, "MY-8856": 0.0, "LZ-5363": 0.0, "VB-3704": 0.0, "JS-5834": 0.0, "KQ-5489": 0.0, "YW-7088": 0.0}}, {"index": 102, "template": "lexical", "variant": null, "n": 10, "position": 10, "rep": 2, "seed": 11002, "labels": "value", "keep_probs": true, "correct_key": "LX-8614", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 275.6, "in_tok": 435, "out_tok": 145, "request_id": "req_01a0aed19523749088ed2e33b6af89c1", "correct_value": "LX-8614", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is LX-8614. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "LX-8614", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["LX-8614", 1.0], ["QS-1005", 0.0], ["JS-5834", 0.0], ["VB-3704", 0.0], ["YW-7088", 0.0]], "probabilities": {"QS-1005": 0.0, "JS-5834": 0.0, "VB-3704": 0.0, "YW-7088": 0.0, "KQ-5489": 0.0, "MY-8856": 0.0, "LX-8614": 1.0, "LZ-5363": 0.0, "WQ-6752": 0.0, "CG-5481": 0.0}}, {"index": 103, "template": "lexical", "variant": null, "n": 10, "position": 5, "rep": 0, "seed": 11000, "labels": "value", "keep_probs": true, "correct_key": "BS-7321", "status": 200, "attempts": [200], "server_ms": 119.0, "wall_ms": 308.7, "in_tok": 430, "out_tok": 142, "request_id": "req_01a0aed195a37898b34b4504919580ef", "correct_value": "BS-7321", "state": "Support ticket from Elena Brandt. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is BS-7321. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "BS-7321", "confidence": 1.0, "n_returned": 10, "prob_sum": 1.0, "top5": [["BS-7321", 1.0], ["TL-4461", 0.0], ["EY-6117", 0.0], ["GY-3999", 0.0], ["KE-8778", 0.0]], "probabilities": {"TL-4461": 0.0, "BS-7321": 1.0, "EY-6117": 0.0, "GY-3999": 0.0, "KE-8778": 0.0, "BM-2721": 0.0, "TK-1302": 0.0, "TH-4469": 0.0, "MB-9208": 0.0, "WY-3070": 0.0}}, {"index": 104, "template": "reasoning", "variant": "two_step", "n": 50, "position": 1, "rep": 2, "seed": 65002, "labels": "value", "keep_probs": true, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 264.9, "in_tok": 717, "out_tok": 463, "request_id": "req_01a0aed195c475959eb8fbd21f0a35ee", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.39, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "266", "confidence": 0.51, "n_returned": 50, "prob_sum": 1.0, "top5": [["266", 0.53], ["267", 0.39], ["265", 0.05], ["268", 0.01], ["257", 0.01]], "probabilities": {"419": 0.0, "270": 0.0, "243": 0.0, "477": 0.0, "487": 0.0, "128": 0.0, "283": 0.0, "37": 0.0, "291": 0.0, "367": 0.0, "356": 0.0, "77": 0.0, "269": 0.0, "307": 0.0, "268": 0.01, "62": 0.0, "193": 0.0, "277": 0.0, "265": 0.05, "124": 0.0, "411": 0.0, "438": 0.0, "353": 0.0, "81": 0.0, "219": 0.0, "267": 0.39, "612": 0.0, "227": 0.0, "266": 0.53, "257": 0.01, "264": 0.01, "664": 0.0, "400": 0.0, "201": 0.0, "380": 0.0, "167": 0.0, "27": 0.0, "478": 0.0, "133": 0.0, "385": 0.0, "549": 0.0, "47": 0.0, "630": 0.0, "259": 0.0, "251": 0.0, "563": 0.0, "596": 0.0, "255": 0.0, "638": 0.0, "217": 0.0}}]}, "labels": {"started": 1789639370.160407, "api_calls": 27, "trials": [{"index": 0, "template": "reasoning", "variant": "two_step", "n": 255, "position": 1, "rep": 2, "seed": 90002, "labels": "value", "correct_key": "363", "status": 200, "attempts": [200], "server_ms": 133.0, "wall_ms": 888.6, "in_tok": 2335, "out_tok": 2287, "request_id": "req_01a0aed1a8a77766b1bab8c0b0238746", "correct_value": "363", "state": "Warehouse log: each tray holds 27 eggs. Today we received 15 full trays plus 6 loose eggs. During unloading, 48 eggs were damaged and thrown away.", "p_correct": 0.03, "rank": 3, "ties_at_pc": 4, "argmax_correct": false, "choice": "345", "confidence": 0.05, "n_returned": 255, "prob_sum": 1.0, "top5": [["345", 0.060000000000000005], ["357", 0.05], ["296", 0.03], ["315", 0.03], ["363", 0.03]]}, {"index": 1, "template": "lexical", "variant": null, "n": 200, "position": 100, "rep": 0, "seed": 30000, "labels": "ordinal", "correct_key": "o100", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 832.0, "in_tok": 4233, "out_tok": 1912, "request_id": "req_01a0aed1a8a8768f97dc1fe589fe877e", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o100", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o100", 1.0], ["o38", 0.0], ["o83", 0.0], ["o99", 0.0], ["o109", 0.0]]}, {"index": 2, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 0, "seed": 80000, "labels": "ordinal", "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 74.0, "wall_ms": 834.6, "in_tok": 3596, "out_tok": 1912, "request_id": "req_01a0aed1a8aa74df9c063cb5f2bc9b11", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 25, "ties_at_pc": 33, "argmax_correct": false, "choice": "o147", "confidence": 0.04, "n_returned": 200, "prob_sum": 1.0, "top5": [["o147", 0.060000000000000005], ["o186", 0.05], ["o54", 0.04], ["o179", 0.04], ["o76", 0.03]]}, {"index": 3, "template": "lexical", "variant": null, "n": 200, "position": 100, "rep": 1, "seed": 30001, "labels": "ordinal", "correct_key": "o100", "status": 200, "attempts": [200], "server_ms": 250.0, "wall_ms": 1005.6, "in_tok": 4244, "out_tok": 1912, "request_id": "req_01a0aed1a92b7d1ea929a07b5f4ad09d", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o100", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o100", 1.0], ["o131", 0.0], ["o145", 0.0], ["o38", 0.0], ["o151", 0.0]]}, {"index": 4, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 2, "seed": 80002, "labels": "ordinal", "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 172.0, "wall_ms": 359.4, "in_tok": 3698, "out_tok": 1912, "request_id": "req_01a0aed1a9ad7e2ca0fe8e13cebc0dd7", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.0, "rank": 59, "ties_at_pc": 141, "argmax_correct": false, "choice": "o176", "confidence": 0.07, "n_returned": 200, "prob_sum": 1.0, "top5": [["o176", 0.09], ["o194", 0.060000000000000005], ["o36", 0.04], ["o199", 0.04], ["o197", 0.03]]}, {"index": 5, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 0, "seed": 80000, "labels": "ordinal", "correct_key": "o100", "status": 200, "attempts": [200], "server_ms": 85.0, "wall_ms": 272.7, "in_tok": 3596, "out_tok": 1912, "request_id": "req_01a0aed1a9af753a9ee81be4c04212cd", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 26, "ties_at_pc": 38, "argmax_correct": false, "choice": "o187", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["o187", 0.04], ["o94", 0.03], ["o159", 0.03], ["o111", 0.03], ["o186", 0.03]]}, {"index": 6, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 1, "seed": 80001, "labels": "ordinal", "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 79.0, "wall_ms": 265.8, "in_tok": 3596, "out_tok": 1911, "request_id": "req_01a0aed1a9e37adfad72c088c7d65dfb", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 19, "ties_at_pc": 35, "argmax_correct": false, "choice": "o51", "confidence": 0.12, "n_returned": 200, "prob_sum": 0.99, "top5": [["o51", 0.13], ["o132", 0.08], ["o19", 0.04], ["o89", 0.03], ["o191", 0.03]]}, {"index": 7, "template": "reasoning", "variant": "two_step", "n": 255, "position": 128, "rep": 1, "seed": 90001, "labels": "value", "correct_key": "657", "status": 200, "attempts": [200], "server_ms": 62.0, "wall_ms": 252.8, "in_tok": 2376, "out_tok": 2326, "request_id": "req_01a0aed1aa5d74c5b6f21b6dc95a3c01", "correct_value": "657", "state": "Warehouse log: each box holds 26 pens. Today we received 26 full boxs plus 18 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.02, "rank": 3, "ties_at_pc": 24, "argmax_correct": false, "choice": "658", "confidence": 0.02, "n_returned": 255, "prob_sum": 1.0, "top5": [["658", 0.03], ["695", 0.03], ["675", 0.02], ["657", 0.02], ["698", 0.02]]}, {"index": 8, "template": "reasoning", "variant": "two_step", "n": 255, "position": 1, "rep": 0, "seed": 90000, "labels": "value", "correct_key": "455", "status": 200, "attempts": [200], "server_ms": 53.0, "wall_ms": 244.1, "in_tok": 2348, "out_tok": 2300, "request_id": "req_01a0aed1aac373edacdb3e193c903349", "correct_value": "455", "state": "Warehouse log: each tray holds 28 eggs. Today we received 18 full trays plus 3 loose eggs. During unloading, 52 eggs were damaged and thrown away.", "p_correct": 0.0, "rank": 52, "ties_at_pc": 203, "argmax_correct": false, "choice": "440", "confidence": 0.03, "n_returned": 255, "prob_sum": 1.0, "top5": [["440", 0.04], ["403", 0.03], ["506", 0.03], ["454", 0.03], ["373", 0.03]]}, {"index": 9, "template": "lexical", "variant": null, "n": 200, "position": 1, "rep": 1, "seed": 30001, "labels": "ordinal", "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 66.0, "wall_ms": 253.8, "in_tok": 4244, "out_tok": 1910, "request_id": "req_01a0aed1aaf37ce8bdbb4ef193b92fc8", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o1", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o1", 1.0], ["o131", 0.0], ["o145", 0.0], ["o38", 0.0], ["o151", 0.0]]}, {"index": 10, "template": "reasoning", "variant": "two_step", "n": 255, "position": 128, "rep": 0, "seed": 90000, "labels": "value", "correct_key": "455", "status": 200, "attempts": [200], "server_ms": 75.0, "wall_ms": 268.4, "in_tok": 2348, "out_tok": 2300, "request_id": "req_01a0aed1ab1d700fbb785926dda07f5f", "correct_value": "455", "state": "Warehouse log: each tray holds 28 eggs. Today we received 18 full trays plus 3 loose eggs. During unloading, 52 eggs were damaged and thrown away.", "p_correct": 0.02, "rank": 8, "ties_at_pc": 37, "argmax_correct": false, "choice": "440", "confidence": 0.02, "n_returned": 255, "prob_sum": 0.99, "top5": [["440", 0.04], ["404", 0.03], ["506", 0.03], ["437", 0.03], ["310", 0.03]]}, {"index": 11, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 1, "seed": 80001, "labels": "ordinal", "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 178.0, "wall_ms": 376.7, "in_tok": 3596, "out_tok": 1912, "request_id": "req_01a0aed1ab9e7c19a2821957c35150ff", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 26, "ties_at_pc": 34, "argmax_correct": false, "choice": "o192", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["o192", 0.060000000000000005], ["o166", 0.04], ["o61", 0.04], ["o20", 0.04], ["o165", 0.03]]}, {"index": 12, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 2, "seed": 80002, "labels": "ordinal", "correct_key": "o100", "status": 200, "attempts": [200], "server_ms": 76.0, "wall_ms": 265.3, "in_tok": 3698, "out_tok": 1912, "request_id": "req_01a0aed1abbb7539a280de5dd52954c8", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 23, "ties_at_pc": 39, "argmax_correct": false, "choice": "o176", "confidence": 0.07, "n_returned": 200, "prob_sum": 0.99, "top5": [["o176", 0.09], ["o194", 0.04], ["o94", 0.03], ["o190", 0.03], ["o127", 0.03]]}, {"index": 13, "template": "reasoning", "variant": "two_step", "n": 255, "position": 255, "rep": 1, "seed": 90001, "labels": "value", "correct_key": "657", "status": 200, "attempts": [200], "server_ms": 91.0, "wall_ms": 278.6, "in_tok": 2376, "out_tok": 2326, "request_id": "req_01a0aed1abf373d082d0e9fd7c66ba1a", "correct_value": "657", "state": "Warehouse log: each box holds 26 pens. Today we received 26 full boxs plus 18 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.03, "rank": 1, "ties_at_pc": 1, "argmax_correct": true, "choice": "657", "confidence": 0.02, "n_returned": 255, "prob_sum": 1.0, "top5": [["695", 0.03], ["657", 0.03], ["800", 0.02], ["656", 0.02], ["716", 0.02]]}, {"index": 14, "template": "lexical", "variant": null, "n": 200, "position": 1, "rep": 0, "seed": 30000, "labels": "ordinal", "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 106.0, "wall_ms": 295.8, "in_tok": 4233, "out_tok": 1910, "request_id": "req_01a0aed1ac2f782894e3bf7d4ae6892a", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o1", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o1", 1.0], ["o33", 0.0], ["o47", 0.0], ["o142", 0.0], ["o134", 0.0]]}, {"index": 15, "template": "lexical", "variant": null, "n": 200, "position": 100, "rep": 2, "seed": 30002, "labels": "ordinal", "correct_key": "o100", "status": 200, "attempts": [200], "server_ms": 134.0, "wall_ms": 321.5, "in_tok": 4245, "out_tok": 1912, "request_id": "req_01a0aed1acc27f93829c1a1f3a708d5a", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o100", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o100", 1.0], ["o163", 0.0], ["o144", 0.0], ["o51", 0.0], ["o67", 0.0]]}, {"index": 16, "template": "reasoning", "variant": "two_step", "n": 255, "position": 255, "rep": 2, "seed": 90002, "labels": "value", "correct_key": "363", "status": 200, "attempts": [200], "server_ms": 81.0, "wall_ms": 268.4, "in_tok": 2335, "out_tok": 2287, "request_id": "req_01a0aed1acd373e395d8a51201d1c531", "correct_value": "363", "state": "Warehouse log: each tray holds 27 eggs. Today we received 15 full trays plus 6 loose eggs. During unloading, 48 eggs were damaged and thrown away.", "p_correct": 0.09, "rank": 1, "ties_at_pc": 1, "argmax_correct": true, "choice": "363", "confidence": 0.08, "n_returned": 255, "prob_sum": 0.99, "top5": [["363", 0.09], ["357", 0.09], ["345", 0.060000000000000005], ["136", 0.03], ["336", 0.03]]}, {"index": 17, "template": "lexical", "variant": null, "n": 200, "position": 200, "rep": 2, "seed": 30002, "labels": "ordinal", "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 94.0, "wall_ms": 283.9, "in_tok": 4245, "out_tok": 1912, "request_id": "req_01a0aed1ad0b7476a2af7b09120ea498", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o200", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o200", 1.0], ["o30", 0.0], ["o124", 0.0], ["o196", 0.0], ["o197", 0.0]]}, {"index": 18, "template": "lexical", "variant": null, "n": 200, "position": 1, "rep": 2, "seed": 30002, "labels": "ordinal", "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 75.0, "wall_ms": 264.1, "in_tok": 4245, "out_tok": 1910, "request_id": "req_01a0aed1ad527204a75efe2aa6f28f7d", "correct_value": "AV-9496", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is AV-9496. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o1", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o1", 1.0], ["o90", 0.0], ["o55", 0.0], ["o128", 0.0], ["o159", 0.0]]}, {"index": 19, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 0, "seed": 80000, "labels": "ordinal", "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 253.0, "wall_ms": 443.6, "in_tok": 3596, "out_tok": 1911, "request_id": "req_01a0aed1ade877d69537b8ddf20adb51", "correct_value": "356", "state": "Warehouse log: each case holds 14 cans. Today we received 28 full cases plus 18 loose cans. During unloading, 54 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 22, "ties_at_pc": 37, "argmax_correct": false, "choice": "o56", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["o56", 0.060000000000000005], ["o187", 0.060000000000000005], ["o44", 0.04], ["o64", 0.04], ["o186", 0.03]]}, {"index": 20, "template": "reasoning", "variant": "two_step", "n": 255, "position": 1, "rep": 1, "seed": 90001, "labels": "value", "correct_key": "657", "status": 200, "attempts": [200], "server_ms": 132.0, "wall_ms": 322.7, "in_tok": 2376, "out_tok": 2326, "request_id": "req_01a0aed1ae187b148fc113bc628de82a", "correct_value": "657", "state": "Warehouse log: each box holds 26 pens. Today we received 26 full boxs plus 18 loose pens. During unloading, 37 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 59, "ties_at_pc": 196, "argmax_correct": false, "choice": "658", "confidence": 0.02, "n_returned": 255, "prob_sum": 1.0, "top5": [["658", 0.04], ["677", 0.03], ["656", 0.03], ["684", 0.02], ["596", 0.02]]}, {"index": 21, "template": "reasoning", "variant": "two_step", "n": 255, "position": 255, "rep": 0, "seed": 90000, "labels": "value", "correct_key": "455", "status": 200, "attempts": [200], "server_ms": 86.0, "wall_ms": 273.8, "in_tok": 2348, "out_tok": 2300, "request_id": "req_01a0aed1ae2870b6a6612f76ca2f5b01", "correct_value": "455", "state": "Warehouse log: each tray holds 28 eggs. Today we received 18 full trays plus 3 loose eggs. During unloading, 52 eggs were damaged and thrown away.", "p_correct": 0.03, "rank": 2, "ties_at_pc": 4, "argmax_correct": false, "choice": "310", "confidence": 0.02, "n_returned": 255, "prob_sum": 0.99, "top5": [["310", 0.04], ["455", 0.03], ["439", 0.03], ["323", 0.03], ["370", 0.03]]}, {"index": 22, "template": "lexical", "variant": null, "n": 200, "position": 200, "rep": 0, "seed": 30000, "labels": "ordinal", "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 106.0, "wall_ms": 295.3, "in_tok": 4233, "out_tok": 1912, "request_id": "req_01a0aed1ae5c75cea58b8e7c85e8a211", "correct_value": "GA-8933", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GA-8933. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o200", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o200", 1.0], ["o30", 0.0], ["o124", 0.0], ["o196", 0.0], ["o197", 0.0]]}, {"index": 23, "template": "lexical", "variant": null, "n": 200, "position": 200, "rep": 1, "seed": 30001, "labels": "ordinal", "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 91.0, "wall_ms": 280.8, "in_tok": 4244, "out_tok": 1912, "request_id": "req_01a0aed1af3f7224924add5b631be16c", "correct_value": "VU-5695", "state": "Support ticket from Priya Natarajan. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VU-5695. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o200", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["o200", 1.0], ["o65", 0.0], ["o85", 0.0], ["o9", 0.0], ["o148", 0.0]]}, {"index": 24, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 2, "seed": 80002, "labels": "ordinal", "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 104.0, "wall_ms": 298.8, "in_tok": 3698, "out_tok": 1912, "request_id": "req_01a0aed1af4e7ab4b0f9d5fb5e5425e2", "correct_value": "921", "state": "Warehouse log: each case holds 36 cans. Today we received 27 full cases plus 4 loose cans. During unloading, 55 cans were damaged and thrown away.", "p_correct": 0.01, "rank": 18, "ties_at_pc": 51, "argmax_correct": false, "choice": "o198", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["o198", 0.05], ["o35", 0.04], ["o175", 0.04], ["o94", 0.04], ["o176", 0.04]]}, {"index": 25, "template": "reasoning", "variant": "two_step", "n": 255, "position": 128, "rep": 2, "seed": 90002, "labels": "value", "correct_key": "363", "status": 200, "attempts": [200], "server_ms": 87.0, "wall_ms": 281.5, "in_tok": 2335, "out_tok": 2287, "request_id": "req_01a0aed1af897092b2d4fba255547b4e", "correct_value": "363", "state": "Warehouse log: each tray holds 27 eggs. Today we received 15 full trays plus 6 loose eggs. During unloading, 48 eggs were damaged and thrown away.", "p_correct": 0.03, "rank": 3, "ties_at_pc": 1, "argmax_correct": false, "choice": "357", "confidence": 0.1, "n_returned": 255, "prob_sum": 1.0, "top5": [["357", 0.12], ["345", 0.06999999999999999], ["360", 0.03], ["363", 0.03], ["211", 0.02]]}, {"index": 26, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 1, "seed": 80001, "labels": "ordinal", "correct_key": "o100", "status": 200, "attempts": [200], "server_ms": 70.0, "wall_ms": 259.2, "in_tok": 3596, "out_tok": 1912, "request_id": "req_01a0aed1afa2793aae71af3c0dd833af", "correct_value": "283", "state": "Warehouse log: each case holds 15 cans. Today we received 20 full cases plus 10 loose cans. During unloading, 27 cans were damaged and thrown away.", "p_correct": 0.060000000000000005, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o100", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["o100", 0.060000000000000005], ["o114", 0.05], ["o19", 0.04], ["o192", 0.04], ["o133", 0.04]]}]}, "near_pilot": {"started": 1789639414.817502, "api_calls": 3, "trials": [{"index": 0, "template": "lexical_near", "n": 200, "position": 1, "rep": 0, "seed": 70000, "labels": "value", "correct_key": "SM-1462", "status": 200, "attempts": [200], "server_ms": 279.0, "wall_ms": 1046.3, "in_tok": 2533, "out_tok": 2433, "request_id": "req_01a0aed257c070aea3fff3cd1a2ba40b", "correct_value": "SM-1462", "state": "Support ticket from Tomasz Kowal. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is SM-1462. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "SM-1462", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["SM-1462", 1.0], ["RX-2648", 0.0], ["VZ-8779", 0.0], ["KG-6157", 0.0], ["QM-1462", 0.0]]}, {"index": 1, "template": "lexical_near", "n": 200, "position": 200, "rep": 1, "seed": 70001, "labels": "value", "correct_key": "GL-3575", "status": 200, "attempts": [200], "server_ms": 322.0, "wall_ms": 1088.3, "in_tok": 2543, "out_tok": 2444, "request_id": "req_01a0aed258007267a30094e3a77ed2b6", "correct_value": "GL-3575", "state": "Support ticket from Marcus Oyelaran. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is GL-3575. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "GL-3575", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["GL-3575", 1.0], ["ZN-2265", 0.0], ["QK-5595", 0.0], ["WL-3575", 0.0], ["TL-7336", 0.0]]}, {"index": 2, "template": "lexical_near", "n": 200, "position": 100, "rep": 2, "seed": 70002, "labels": "value", "correct_key": "VT-6825", "status": 200, "attempts": [200], "server_ms": 101.0, "wall_ms": 857.4, "in_tok": 2533, "out_tok": 2433, "request_id": "req_01a0aed2571979838319a14f15965c21", "correct_value": "VT-6825", "state": "Support ticket from Dana Whitfield. The customer says their parcel arrived damaged and asks for a replacement. The customer's account code is VT-6825. Their previous order was placed in March.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "VT-6825", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["VT-6825", 1.0], ["ZV-5068", 0.0], ["PP-2902", 0.0], ["VU-6825", 0.0], ["VT-6828", 0.0]]}]}, "binding_pilot": {"started": 1789639452.526724, "api_calls": 3, "trials": [{"index": 0, "template": "binding", "n": 200, "position": 1, "rep": 0, "seed": 80000, "labels": "value", "correct_key": "CT-9118", "status": 200, "attempts": [200], "server_ms": 74.0, "wall_ms": 837.0, "in_tok": 2586, "out_tok": 2437, "request_id": "req_01a0aed2ea6d712d81611185f9c87b5f", "correct_value": "CT-9118", "state": "Account review note. Codes found on file for this customer: YX-4201 (closed (duplicate)); CT-9118 (active); SQ-1699 (closed); KY-8166 (suspended); BA-2825 (pending transfer). The customer is asking why their last invoice was billed to the wrong account and wants it moved to the account they currently use.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "CT-9118", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["CT-9118", 1.0], ["SQ-1699", 0.0], ["DD-6774", 0.0], ["MC-6482", 0.0], ["WG-4352", 0.0]]}, {"index": 1, "template": "binding", "n": 200, "position": 200, "rep": 1, "seed": 80001, "labels": "value", "correct_key": "NK-2435", "status": 200, "attempts": [200], "server_ms": 63.0, "wall_ms": 856.2, "in_tok": 2582, "out_tok": 2434, "request_id": "req_01a0aed2ea8179c5abf81c69762c5ba3", "correct_value": "NK-2435", "state": "Account review note. Codes found on file for this customer: YE-2585 (closed); KH-2631 (suspended); DB-7150 (pending transfer); NK-2435 (active); XT-5785 (closed (duplicate)). The customer is asking why their last invoice was billed to the wrong account and wants it moved to the account they currently use.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "NK-2435", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["NK-2435", 1.0], ["SK-8271", 0.0], ["PK-4019", 0.0], ["BK-5787", 0.0], ["BY-8429", 0.0]]}, {"index": 2, "template": "binding", "n": 200, "position": 100, "rep": 2, "seed": 80002, "labels": "value", "correct_key": "YZ-9861", "status": 200, "attempts": [200], "server_ms": 84.0, "wall_ms": 840.9, "in_tok": 2589, "out_tok": 2440, "request_id": "req_01a0aed2ea69721e9d1e6624e4006c78", "correct_value": "YZ-9861", "state": "Account review note. Codes found on file for this customer: GS-1871 (closed (duplicate)); YZ-9861 (active); XP-9082 (pending transfer); CM-7018 (closed); GH-6612 (suspended). The customer is asking why their last invoice was billed to the wrong account and wants it moved to the account they currently use.", "p_correct": 1.0, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "YZ-9861", "confidence": 1.0, "n_returned": 200, "prob_sum": 1.0, "top5": [["YZ-9861", 1.0], ["KB-7599", 0.0], ["NJ-8094", 0.0], ["JN-9582", 0.0], ["NY-3128", 0.0]]}]}, "crossn": {"started": 1789639469.84495, "api_calls": 28, "trials": [{"index": 0, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 1, "seed": 65001, "labels": "value", "instance_from_n": 50, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 60.0, "wall_ms": 849.6, "in_tok": 1915, "out_tok": 1812, "request_id": "req_01a0aed32e277b2ebb1df4ed6e87d0d9", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.01, "rank": 19, "ties_at_pc": 53, "argmax_correct": false, "choice": "476", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["476", 0.060000000000000005], ["729", 0.04], ["441", 0.03], ["478", 0.03], ["437", 0.03]]}, {"index": 1, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 3, "seed": 65003, "labels": "value", "instance_from_n": 50, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 58.0, "wall_ms": 837.9, "in_tok": 1901, "out_tok": 1796, "request_id": "req_01a0aed32e1c792fa49f0a8516a96b0b", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 60, "ties_at_pc": 140, "argmax_correct": false, "choice": "282", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["282", 0.060000000000000005], ["243", 0.060000000000000005], ["217", 0.04], ["221", 0.04], ["260", 0.03]]}, {"index": 2, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 3, "seed": 65003, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 851.7, "in_tok": 3599, "out_tok": 1912, "request_id": "req_01a0aed32e217083acedc398ba1d6b21", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 25, "ties_at_pc": 38, "argmax_correct": false, "choice": "o142", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["o142", 0.060000000000000005], ["o148", 0.05], ["o191", 0.03], ["o133", 0.03], ["o153", 0.03]]}, {"index": 3, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 2, "seed": 65002, "labels": "value", "instance_from_n": 50, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 92.0, "wall_ms": 841.6, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed32e09725ca89b955097543e4e", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.03, "rank": 4, "ties_at_pc": 2, "argmax_correct": false, "choice": "264", "confidence": 0.28, "n_returned": 200, "prob_sum": 1.0, "top5": [["264", 0.29000000000000004], ["270", 0.09], ["265", 0.06999999999999999], ["260", 0.03], ["267", 0.03]]}, {"index": 4, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 1, "seed": 65001, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 73.0, "wall_ms": 268.9, "in_tok": 3613, "out_tok": 1912, "request_id": "req_01a0aed32f1b79758770f7289b3e6541", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.01, "rank": 15, "ties_at_pc": 65, "argmax_correct": false, "choice": "o187", "confidence": 0.02, "n_returned": 200, "prob_sum": 1.0, "top5": [["o59", 0.03], ["o28", 0.03], ["o187", 0.03], ["o129", 0.03], ["o51", 0.03]]}, {"index": 5, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 0, "seed": 65000, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 84.0, "wall_ms": 268.6, "in_tok": 3597, "out_tok": 1911, "request_id": "req_01a0aed32f1e7b8ea9ceb1001afcc396", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 22, "ties_at_pc": 48, "argmax_correct": false, "choice": "o34", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["o138", 0.04], ["o34", 0.04], ["o122", 0.03], ["o153", 0.03], ["o26", 0.03]]}, {"index": 6, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 3, "seed": 65003, "labels": "value", "instance_from_n": 50, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 116.0, "wall_ms": 315.4, "in_tok": 1901, "out_tok": 1796, "request_id": "req_01a0aed32f2b78b9986584c8bf85e47b", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 65, "ties_at_pc": 135, "argmax_correct": false, "choice": "282", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["282", 0.05], ["259", 0.05], ["243", 0.03], ["144", 0.03], ["263", 0.03]]}, {"index": 7, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 2, "seed": 65002, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 122.0, "wall_ms": 317.5, "in_tok": 3596, "out_tok": 1911, "request_id": "req_01a0aed32f2e7e648e7c3e58c1a43437", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 19, "ties_at_pc": 45, "argmax_correct": false, "choice": "o52", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["o52", 0.060000000000000005], ["o132", 0.04], ["o36", 0.04], ["o153", 0.04], ["o43", 0.03]]}, {"index": 8, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 2, "seed": 65002, "labels": "value", "instance_from_n": 50, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 93.0, "wall_ms": 290.1, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed33026793d911354b54ec7ff82", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.14, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "264", "confidence": 0.24, "n_returned": 200, "prob_sum": 1.0, "top5": [["264", 0.26], ["267", 0.14], ["270", 0.05], ["268", 0.04], ["260", 0.03]]}, {"index": 9, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 0, "seed": 65000, "labels": "value", "instance_from_n": 50, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 135.0, "wall_ms": 319.5, "in_tok": 1899, "out_tok": 1793, "request_id": "req_01a0aed3303e706fae5f0fdcef097d99", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 19, "ties_at_pc": 57, "argmax_correct": false, "choice": "61", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["61", 0.04], ["325", 0.03], ["56", 0.03], ["262", 0.03], ["67", 0.02]]}, {"index": 10, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 1, "seed": 65001, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o200", "status": 200, "attempts": [200], "server_ms": 68.0, "wall_ms": 269.0, "in_tok": 3613, "out_tok": 1912, "request_id": "req_01a0aed330677f1da95ed30e76bb9361", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.06999999999999999, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "o200", "confidence": 0.06, "n_returned": 200, "prob_sum": 1.0, "top5": [["o200", 0.06999999999999999], ["o17", 0.04], ["o76", 0.03], ["o146", 0.03], ["o198", 0.03]]}, {"index": 11, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 3, "seed": 65003, "labels": "value", "instance_from_n": 50, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 113.0, "wall_ms": 306.0, "in_tok": 1901, "out_tok": 1796, "request_id": "req_01a0aed3306976a9a7e3d22c5f52a7b2", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 59, "ties_at_pc": 141, "argmax_correct": false, "choice": "243", "confidence": 0.03, "n_returned": 200, "prob_sum": 0.99, "top5": [["243", 0.05], ["222", 0.04], ["282", 0.04], ["259", 0.04], ["200", 0.04]]}, {"index": 12, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 0, "seed": 65000, "labels": "value", "instance_from_n": 50, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 87.0, "wall_ms": 287.0, "in_tok": 1899, "out_tok": 1793, "request_id": "req_01a0aed3315274718ee802f93f6ae71a", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 67, "ties_at_pc": 133, "argmax_correct": false, "choice": "61", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["66", 0.04], ["61", 0.04], ["325", 0.04], ["274", 0.04], ["219", 0.03]]}, {"index": 13, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 0, "seed": 65000, "labels": "value", "instance_from_n": 50, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 55.0, "wall_ms": 247.7, "in_tok": 1899, "out_tok": 1794, "request_id": "req_01a0aed33172770a887c06d81fdb616f", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 66, "ties_at_pc": 134, "argmax_correct": false, "choice": "280", "confidence": 0.05, "n_returned": 200, "prob_sum": 1.0, "top5": [["280", 0.06999999999999999], ["274", 0.04], ["253", 0.03], ["325", 0.03], ["262", 0.03]]}, {"index": 14, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 0, "seed": 65000, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 84.0, "wall_ms": 283.0, "in_tok": 3597, "out_tok": 1912, "request_id": "req_01a0aed33175782187b1f40acbd1f6ba", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 23, "ties_at_pc": 41, "argmax_correct": false, "choice": "o139", "confidence": 0.05, "n_returned": 200, "prob_sum": 0.99, "top5": [["o139", 0.060000000000000005], ["o63", 0.04], ["o12", 0.04], ["o100", 0.03], ["o190", 0.03]]}, {"index": 15, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 3, "seed": 65003, "labels": "value", "instance_from_n": 50, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 89.0, "wall_ms": 284.2, "in_tok": 1901, "out_tok": 1796, "request_id": "req_01a0aed3319f7ae4b534683a43d7bd2c", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 51, "ties_at_pc": 149, "argmax_correct": false, "choice": "200", "confidence": 0.04, "n_returned": 200, "prob_sum": 0.99, "top5": [["200", 0.060000000000000005], ["243", 0.05], ["260", 0.04], ["259", 0.04], ["217", 0.04]]}, {"index": 16, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 0, "seed": 65000, "labels": "value", "instance_from_n": 50, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 71.0, "wall_ms": 261.7, "in_tok": 1899, "out_tok": 1793, "request_id": "req_01a0aed3326a7506b4d186b2cdf7bb56", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 17, "ties_at_pc": 57, "argmax_correct": false, "choice": "61", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["61", 0.04], ["262", 0.03], ["207", 0.03], ["325", 0.03], ["57", 0.03]]}, {"index": 17, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 1, "seed": 65001, "labels": "value", "instance_from_n": 50, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 52.0, "wall_ms": 250.3, "in_tok": 1915, "out_tok": 1812, "request_id": "req_01a0aed3326b7b27bf1b15854d3fbda6", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.060000000000000005, "rank": 1, "ties_at_pc": 0, "argmax_correct": true, "choice": "477", "confidence": 0.05, "n_returned": 200, "prob_sum": 0.99, "top5": [["477", 0.060000000000000005], ["437", 0.03], ["427", 0.03], ["476", 0.03], ["467", 0.03]]}, {"index": 18, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 1, "seed": 65001, "labels": "value", "instance_from_n": 50, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 60.0, "wall_ms": 266.0, "in_tok": 1915, "out_tok": 1812, "request_id": "req_01a0aed3329974e493355cafbf396ad7", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.03, "rank": 2, "ties_at_pc": 2, "argmax_correct": false, "choice": "476", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["476", 0.04], ["441", 0.03], ["477", 0.03], ["455", 0.03], ["602", 0.02]]}, {"index": 19, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 0, "seed": 65000, "labels": "value", "instance_from_n": 50, "correct_key": "279", "status": 200, "attempts": [200], "server_ms": 92.0, "wall_ms": 287.7, "in_tok": 1899, "out_tok": 1793, "request_id": "req_01a0aed332bd74cf94ae69113fac76d5", "correct_value": "279", "state": "Warehouse log: each box holds 26 pens. Today we received 12 full boxs plus 13 loose pens. During unloading, 46 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 27, "ties_at_pc": 42, "argmax_correct": false, "choice": "61", "confidence": 0.03, "n_returned": 200, "prob_sum": 1.0, "top5": [["61", 0.04], ["282", 0.03], ["262", 0.03], ["241", 0.03], ["591", 0.02]]}, {"index": 20, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 3, "seed": 65003, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 94.0, "wall_ms": 293.0, "in_tok": 3599, "out_tok": 1911, "request_id": "req_01a0aed3336c7b5195b8dc9f22767d4c", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.01, "rank": 22, "ties_at_pc": 42, "argmax_correct": false, "choice": "o76", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["o76", 0.04], ["o143", 0.04], ["o9", 0.03], ["o154", 0.03], ["o163", 0.03]]}, {"index": 21, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 2, "seed": 65002, "labels": "value", "instance_from_n": 50, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 81.0, "wall_ms": 265.5, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed3336b73888ad1cd1d8a7a3a20", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.03, "rank": 5, "ties_at_pc": 1, "argmax_correct": false, "choice": "265", "confidence": 0.11, "n_returned": 200, "prob_sum": 1.0, "top5": [["265", 0.13], ["264", 0.06999999999999999], ["270", 0.060000000000000005], ["204", 0.04], ["268", 0.03]]}, {"index": 22, "template": "reasoning", "variant": "two_step", "n": 200, "position": 50, "rep": 3, "seed": 65003, "labels": "value", "instance_from_n": 50, "correct_key": "281", "status": 200, "attempts": [200], "server_ms": 101.0, "wall_ms": 298.7, "in_tok": 1901, "out_tok": 1796, "request_id": "req_01a0aed3339d7d68b0008ef9cd3f79a1", "correct_value": "281", "state": "Warehouse log: each box holds 19 pens. Today we received 16 full boxs plus 15 loose pens. During unloading, 38 pens were damaged and thrown away.", "p_correct": 0.0, "rank": 56, "ties_at_pc": 144, "argmax_correct": false, "choice": "282", "confidence": 0.04, "n_returned": 200, "prob_sum": 1.0, "top5": [["282", 0.05], ["243", 0.05], ["259", 0.05], ["260", 0.04], ["263", 0.04]]}, {"index": 23, "template": "reasoning", "variant": "two_step", "n": 200, "position": 150, "rep": 1, "seed": 65001, "labels": "value", "instance_from_n": 50, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 110.0, "wall_ms": 316.8, "in_tok": 1915, "out_tok": 1812, "request_id": "req_01a0aed333df757d8ca34dfaf672e261", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.01, "rank": 20, "ties_at_pc": 53, "argmax_correct": false, "choice": "437", "confidence": 0.02, "n_returned": 200, "prob_sum": 0.99, "top5": [["455", 0.03], ["476", 0.03], ["563", 0.03], ["437", 0.03], ["636", 0.03]]}, {"index": 24, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 2, "seed": 65002, "labels": "ordinal", "instance_from_n": 50, "correct_key": "o1", "status": 200, "attempts": [200], "server_ms": 146.0, "wall_ms": 330.0, "in_tok": 3596, "out_tok": 1912, "request_id": "req_01a0aed334777cbfa810c557db1b4a8a", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.03, "rank": 4, "ties_at_pc": 5, "argmax_correct": false, "choice": "o139", "confidence": 0.09, "n_returned": 200, "prob_sum": 0.99, "top5": [["o139", 0.09999999999999999], ["o154", 0.04], ["o187", 0.04], ["o44", 0.03], ["o189", 0.03]]}, {"index": 25, "template": "reasoning", "variant": "two_step", "n": 200, "position": 200, "rep": 2, "seed": 65002, "labels": "value", "instance_from_n": 50, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 65.0, "wall_ms": 260.8, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed3348e7650afd0059179201027", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.04, "rank": 4, "ties_at_pc": 0, "argmax_correct": false, "choice": "264", "confidence": 0.06, "n_returned": 200, "prob_sum": 1.0, "top5": [["264", 0.06999999999999999], ["265", 0.060000000000000005], ["270", 0.05], ["267", 0.04], ["268", 0.03]]}, {"index": 26, "template": "reasoning", "variant": "two_step", "n": 200, "position": 1, "rep": 2, "seed": 65002, "labels": "value", "instance_from_n": 50, "correct_key": "267", "status": 200, "attempts": [200], "server_ms": 128.0, "wall_ms": 324.9, "in_tok": 1898, "out_tok": 1794, "request_id": "req_01a0aed334de717d8ee52c96b26cd4be", "correct_value": "267", "state": "Warehouse log: each box holds 16 pens. Today we received 18 full boxs plus 3 loose pens. During unloading, 24 pens were damaged and thrown away.", "p_correct": 0.06999999999999999, "rank": 2, "ties_at_pc": 0, "argmax_correct": false, "choice": "264", "confidence": 0.11, "n_returned": 200, "prob_sum": 1.0, "top5": [["264", 0.12], ["267", 0.06999999999999999], ["265", 0.03], ["257", 0.03], ["266", 0.03]]}, {"index": 27, "template": "reasoning", "variant": "two_step", "n": 200, "position": 100, "rep": 1, "seed": 65001, "labels": "value", "instance_from_n": 50, "correct_key": "477", "status": 200, "attempts": [200], "server_ms": 76.0, "wall_ms": 271.7, "in_tok": 1915, "out_tok": 1812, "request_id": "req_01a0aed3351f70f0a966f2e2df77f4a9", "correct_value": "477", "state": "Warehouse log: each bag holds 21 bolts. Today we received 24 full bags plus 5 loose bolts. During unloading, 32 bolts were damaged and thrown away.", "p_correct": 0.01, "rank": 18, "ties_at_pc": 60, "argmax_correct": false, "choice": "441", "confidence": 0.01, "n_returned": 200, "prob_sum": 0.99, "top5": [["461", 0.03], ["437", 0.03], ["476", 0.03], ["441", 0.03], ["602", 0.02]]}]}}}