diff --git a/.github/script/assets/d3_goldens.jsonl b/.github/script/assets/d3_goldens.jsonl new file mode 100644 index 000000000..72559faf0 --- /dev/null +++ b/.github/script/assets/d3_goldens.jsonl @@ -0,0 +1,15 @@ +{"id": "scen-choice-0000", "request": {"state": "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today.", "questions": {"route": {"type": "choice", "instructions": "Which team should handle this?", "criteria": {"returns": "Refunds, replacements and damaged deliveries", "billing": "Payments, invoices and charges", "technical": "Product setup and faults", "logistics": "Shipping, customs and delivery delays", "facilities": "Buildings, plumbing and maintenance", "security": "Abuse, prompt injection and fraud", "general": "Anything else"}}}}, "response": {"model": "d3-lite", "answers": {"route": {"type": "choice", "choice": "returns", "probabilities": {"returns": 0.997228170414022, "billing": 0.0002742457086480882, "technical": 0.0015224572395775566, "logistics": 0.00029111344619764834, "facilities": 6.157099208476648e-05, "security": 1.9958108039337077e-05, "general": 0.0006024840914306099}, "confidence": 0.988414547194079}}, "usage": {"input_tokens": 166, "output_tokens": 0}}} +{"id": "scen-noul-0001", "request": {"state": "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today.", "questions": {"needs_human": {"type": "noul", "instructions": "Does this need a human agent?"}}}, "response": {"model": "d3-lite", "answers": {"needs_human": {"type": "noul", "noul": 0.2591443878667239}}, "usage": {"input_tokens": 109, "output_tokens": 0}}} +{"id": "scen-score5-0004", "request": {"state": "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today.", "questions": {"sentiment": {"type": "score", "instructions": "Sentiment of the author?", "criteria": ["very negative", "negative", "neutral", "positive", "very positive"]}}}, "response": {"model": "d3-lite", "answers": {"sentiment": {"type": "score", "score": 0.7762791400127715, "probabilities": {"0": 0.407738325891494, "1": 0.4364548899228667, "2": 0.13345733216589753, "3": 0.01648822232085725, "4": 0.005861229698884521}, "confidence": 0.32011383540681515, "legend": {"0": "very negative", "1": "negative", "2": "neutral", "3": "positive", "4": "very positive"}}}, "usage": {"input_tokens": 118, "output_tokens": 0}}} +{"id": "nopt-27-0145", "request": {"state": "Weather: 2°C, freezing rain expected after 16:00; the delivery route includes two mountain passes.", "questions": {"q": {"type": "choice", "instructions": "Pick the best option.", "criteria": {"opt000": null, "opt001": "Option number 1", "opt002": "Option number 2", "opt003": null, "opt004": "Option number 4", "opt005": "Option number 5", "opt006": null, "opt007": "Option number 7", "opt008": "Option number 8", "opt009": null, "opt010": "Option number 10", "opt011": "Option number 11", "opt012": null, "opt013": "Option number 13", "opt014": "Option number 14", "opt015": null, "opt016": "Option number 16", "opt017": "Option number 17", "opt018": null, "opt019": "Option number 19", "opt020": "Option number 20", "opt021": null, "opt022": "Option number 22", "opt023": "Option number 23", "opt024": null, "opt025": "Option number 25", "opt026": "Option number 26"}}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "choice", "choice": "opt020", "probabilities": {"opt000": 0.007468117807545698, "opt001": 0.01203468618700762, "opt002": 0.023210409844956842, "opt003": 0.03476704257026328, "opt004": 0.030536591764948304, "opt005": 0.02308941427966578, "opt006": 0.035202335287612074, "opt007": 0.02683737107727895, "opt008": 0.03700550988824402, "opt009": 0.020943255929259885, "opt010": 0.05319521906584561, "opt011": 0.024478073969424117, "opt012": 0.054040755648774634, "opt013": 0.030355803428195112, "opt014": 0.05820679246004882, "opt015": 0.07587550520436671, "opt016": 0.04848683587076319, "opt017": 0.023169992308183635, "opt018": 0.04119197991664351, "opt019": 0.0442707944882086, "opt020": 0.0787847706399201, "opt021": 0.019919110304385017, "opt022": 0.046109255038268245, "opt023": 0.03429219844612881, "opt024": 0.05926635069422053, "opt025": 0.03877396850885941, "opt026": 0.018487859370981526}, "confidence": 0.034082073308846494}}, "usage": {"input_tokens": 393, "output_tokens": 0}}} +{"id": "nopt-100-0148", "request": {"state": "顧客は商品が届いていないと言っています。注文番号は 58821 です。", "questions": {"q": {"type": "choice", "instructions": "Pick the best option.", "criteria": {"opt000": null, "opt001": "Option number 1", "opt002": "Option number 2", "opt003": null, "opt004": "Option number 4", "opt005": "Option number 5", "opt006": null, "opt007": "Option number 7", "opt008": "Option number 8", "opt009": null, "opt010": "Option number 10", "opt011": "Option number 11", "opt012": null, "opt013": "Option number 13", "opt014": "Option number 14", "opt015": null, "opt016": "Option number 16", "opt017": "Option number 17", "opt018": null, "opt019": "Option number 19", "opt020": "Option number 20", "opt021": null, "opt022": "Option number 22", "opt023": "Option number 23", "opt024": null, "opt025": "Option number 25", "opt026": "Option number 26", "opt027": null, "opt028": "Option number 28", "opt029": "Option number 29", "opt030": null, "opt031": "Option number 31", "opt032": "Option number 32", "opt033": null, "opt034": "Option number 34", "opt035": "Option number 35", "opt036": null, "opt037": "Option number 37", "opt038": "Option number 38", "opt039": null, "opt040": "Option number 40", "opt041": "Option number 41", "opt042": null, "opt043": "Option number 43", "opt044": "Option number 44", "opt045": null, "opt046": "Option number 46", "opt047": "Option number 47", "opt048": null, "opt049": "Option number 49", "opt050": "Option number 50", "opt051": null, "opt052": "Option number 52", "opt053": "Option number 53", "opt054": null, "opt055": "Option number 55", "opt056": "Option number 56", "opt057": null, "opt058": "Option number 58", "opt059": "Option number 59", "opt060": null, "opt061": "Option number 61", "opt062": "Option number 62", "opt063": null, "opt064": "Option number 64", "opt065": "Option number 65", "opt066": null, "opt067": "Option number 67", "opt068": "Option number 68", "opt069": null, "opt070": "Option number 70", "opt071": "Option number 71", "opt072": null, "opt073": "Option number 73", "opt074": "Option number 74", "opt075": null, "opt076": "Option number 76", "opt077": "Option number 77", "opt078": null, "opt079": "Option number 79", "opt080": "Option number 80", "opt081": null, "opt082": "Option number 82", "opt083": "Option number 83", "opt084": null, "opt085": "Option number 85", "opt086": "Option number 86", "opt087": null, "opt088": "Option number 88", "opt089": "Option number 89", "opt090": null, "opt091": "Option number 91", "opt092": "Option number 92", "opt093": null, "opt094": "Option number 94", "opt095": "Option number 95", "opt096": null, "opt097": "Option number 97", "opt098": "Option number 98", "opt099": null}}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "choice", "choice": "opt058", "probabilities": {"opt000": 0.0012552937992036102, "opt001": 0.002197073900691935, "opt002": 0.0030539836187434607, "opt003": 0.0027804529984722555, "opt004": 0.003305759240894739, "opt005": 0.0015294175555741177, "opt006": 0.003933207361322931, "opt007": 0.002394396252882728, "opt008": 0.005358568705227566, "opt009": 0.0019344163556982057, "opt010": 0.0070122922203920785, "opt011": 0.0024423249093355587, "opt012": 0.005196822178970081, "opt013": 0.006085675099040757, "opt014": 0.006855145496598898, "opt015": 0.00662740451753252, "opt016": 0.006269660678354557, "opt017": 0.0032088511632329822, "opt018": 0.0032626499462159843, "opt019": 0.007916340134815127, "opt020": 0.01850178945686162, "opt021": 0.007328403609750118, "opt022": 0.004425326620355323, "opt023": 0.005763237700725683, "opt024": 0.005486146401213406, "opt025": 0.004949260791714729, "opt026": 0.0046568664644251224, "opt027": 0.0018624658618466227, "opt028": 0.0028604377798129954, "opt029": 0.0035761187076144237, "opt030": 0.0066658457903644, "opt031": 0.0024764460093224714, "opt032": 0.010291449394544603, "opt033": 0.00346288151746891, "opt034": 0.0035309146355125393, "opt035": 0.004296325863392471, "opt036": 0.004945717574772949, "opt037": 0.004905442062023383, "opt038": 0.008149848794922494, "opt039": 0.006093114038539367, "opt040": 0.008482470671846275, "opt041": 0.004960817108555856, "opt042": 0.010529447961046061, "opt043": 0.010049118315173345, "opt044": 0.005293016630667417, "opt045": 0.003525586538752408, "opt046": 0.004058987604634928, "opt047": 0.005908963615120245, "opt048": 0.008172837562695101, "opt049": 0.0074167935778169875, "opt050": 0.021436564011932183, "opt051": 0.005417335628774616, "opt052": 0.006187325238698389, "opt053": 0.008113258060154305, "opt054": 0.006978551798529743, "opt055": 0.01009101479495857, "opt056": 0.013698658773562411, "opt057": 0.006284129241056362, "opt058": 0.08358830701067832, "opt059": 0.01561829445706279, "opt060": 0.013918734967392661, "opt061": 0.009847385451842577, "opt062": 0.022013211044089254, "opt063": 0.005597378919400139, "opt064": 0.014636546295707776, "opt065": 0.014466619031477735, "opt066": 0.010103513144638865, "opt067": 0.0138458468644844, "opt068": 0.03213325330274644, "opt069": 0.008411826125151116, "opt070": 0.039118396537182465, "opt071": 0.022276030289974432, "opt072": 0.009120163108362086, "opt073": 0.01767236097411402, "opt074": 0.03892032284046078, "opt075": 0.016234478928785996, "opt076": 0.008400558052663643, "opt077": 0.003838510709243578, "opt078": 0.010771424206495695, "opt079": 0.0021134956109874533, "opt080": 0.012425435500352548, "opt081": 0.0103707375465589, "opt082": 0.017488124869013036, "opt083": 0.007769016536450629, "opt084": 0.012516113728118708, "opt085": 0.012305834117022215, "opt086": 0.006442841983850095, "opt087": 0.011644447901503695, "opt088": 0.009265613351258483, "opt089": 0.02112976771150874, "opt090": 0.005656382864556254, "opt091": 0.010416258734015938, "opt092": 0.007163278710656175, "opt093": 0.02437242076389757, "opt094": 0.010479698568853665, "opt095": 0.009273918886463317, "opt096": 0.01822413796678804, "opt097": 0.023077874859098263, "opt098": 0.010637291396186103, "opt099": 0.003241493788546462}, "confidence": 0.07328034892724522}}, "usage": {"input_tokens": 1187, "output_tokens": 0}}} +{"id": "score-json-levels-0161", "request": {"state": "A 34-year-old reports a mild headache since this morning, no fever, slept four hours last night.", "questions": {"q": {"type": "score", "criteria": [0, 1.5, {"high": true}, [3, "x"], "n/a"]}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "score", "score": 1.9523753968200201, "probabilities": {"0": 0.2573348000124179, "1": 0.26869976670686024, "2": 0.024500051650218953, "3": 0.16318599970928951, "4": 0.2862793819212134}, "confidence": 0.10080631409854535, "legend": {"0": "0", "1": "1.5", "2": "{\"high\":true}", "3": "[3,\"x\"]", "4": "n/a"}}}, "usage": {"input_tokens": 135, "output_tokens": 0}}} +{"id": "json-state-0171", "request": {"state": [1, 2.0, -0.0, 1e-05, 1e+20, 1e+16, 1000000000000000.0, 123456789012345678, 9223372036854775808, 18446744073709551617, 0.1, 0.3333333333333333, -1e-07], "questions": {"q": {"type": "choice", "instructions": "Is the state well-formed data?", "criteria": {"yes": null, "no": null, "unsure": "Cannot tell"}}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "choice", "choice": "no", "probabilities": {"yes": 0.14062434644437352, "no": 0.6877791388001909, "unsure": 0.1715965147554355}, "confidence": 0.23927450987800158}}, "usage": {"input_tokens": 243, "output_tokens": 0}}} +{"id": "empty-instr-0192", "request": {"state": "User: my laptop won't boot after the update, the screen stays black but the fan spins.", "questions": {"q": {"type": "noul", "instructions": ""}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "noul", "noul": 0.5306217828578288}}, "usage": {"input_tokens": 110, "output_tokens": 0}}} +{"id": "false-instr-0194", "request": {"state": "User: my laptop won't boot after the update, the screen stays black but the fan spins.", "questions": {"q": {"type": "noul", "instructions": false}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "noul", "noul": 0.5306217828578288}}, "usage": {"input_tokens": 110, "output_tokens": 0}}} +{"id": "key-unicode-0196", "request": {"state": "顧客は商品が届いていないと言っています。注文番号は 58821 です。", "questions": {"q": {"type": "choice", "criteria": {"返品": "Returns", "配送": "Delivery", "🙂": null, " spaced key ": null}}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "choice", "choice": "配送", "probabilities": {"返品": 0.015092023434853082, "配送": 0.9829923371512479, "🙂": 0.000833609304241425, " spaced key ": 0.0010820301096575965}, "confidence": 0.9325892670937882}}, "usage": {"input_tokens": 122, "output_tokens": 0}}} +{"id": "literal-image-pad-0197", "request": {"state": "look at <|image_pad|> here", "questions": {"q": {"type": "noul"}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "noul", "noul": 0.557546079158783}}, "usage": {"input_tokens": 95, "output_tokens": 0}}} +{"id": "multi-3-0199", "request": {"state": "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today.", "questions": {"route": {"type": "choice", "instructions": "Which team should handle this request?", "criteria": {"returns": "Refunds, replacements and damaged deliveries", "billing": "Payments, invoices and charges", "technical": "Product setup and faults"}}, "receipt": {"type": "noul", "instructions": "Does the customer have a receipt?"}, "urgency": {"type": "score", "instructions": "How urgent is this request?", "criteria": ["Routine", "Soon", "Today"]}}}, "response": {"model": "d3-lite", "answers": {"route": {"type": "choice", "choice": "returns", "probabilities": {"returns": 0.9948883703647983, "billing": 0.001305922639726265, "technical": 0.0038057069954754675}, "confidence": 0.9681657194482413}, "receipt": {"type": "noul", "noul": 0.9961495411071805}, "urgency": {"type": "score", "score": 1.1458984889701234, "probabilities": {"0": 0.3125005019828602, "1": 0.22910050706415613, "2": 0.45839899095298364}, "confidence": 0.03638053584761969, "legend": {"0": "Routine", "1": "Soon", "2": "Today"}}}, "usage": {"input_tokens": 345, "output_tokens": 0}}} +{"id": "multi-16-0200", "request": {"state": "Log: 2026-10-01T03:12:44Z ERROR db-pool exhausted (max=50) while serving /api/v2/orders; p99 latency 9.8s.", "questions": {"q00": {"type": "noul", "instructions": "Is component 0 involved?"}, "q01": {"type": "choice", "criteria": {"low": null, "mid": null, "high": null}}, "q02": {"type": "score", "criteria": ["a", "b", "c", "d"]}, "q03": {"type": "noul", "instructions": "Is component 3 involved?"}, "q04": {"type": "choice", "criteria": {"low": null, "mid": null, "high": null}}, "q05": {"type": "score", "criteria": ["a", "b", "c", "d"]}, "q06": {"type": "noul", "instructions": "Is component 6 involved?"}, "q07": {"type": "choice", "criteria": {"low": null, "mid": null, "high": null}}, "q08": {"type": "score", "criteria": ["a", "b", "c", "d"]}, "q09": {"type": "noul", "instructions": "Is component 9 involved?"}, "q10": {"type": "choice", "criteria": {"low": null, "mid": null, "high": null}}, "q11": {"type": "score", "criteria": ["a", "b", "c", "d"]}, "q12": {"type": "noul", "instructions": "Is component 12 involved?"}, "q13": {"type": "choice", "criteria": {"low": null, "mid": null, "high": null}}, "q14": {"type": "score", "criteria": ["a", "b", "c", "d"]}, "q15": {"type": "noul", "instructions": "Is component 15 involved?"}}}, "response": {"model": "d3-lite", "answers": {"q00": {"type": "noul", "noul": 0.20675245493266933}, "q01": {"type": "choice", "choice": "mid", "probabilities": {"low": 0.1992160677909851, "mid": 0.5181268453598022, "high": 0.28265708684921265}, "confidence": 0.07224955867479488}, "q02": {"type": "score", "score": 1.4385067540852203, "probabilities": {"0": 0.2317957212985837, "1": 0.31430735247411273, "2": 0.23749137707080314, "3": 0.21640554915650043}, "confidence": 0.007937943291352001, "legend": {"0": "a", "1": "b", "2": "c", "3": "d"}}, "q03": {"type": "noul", "noul": 0.03161033172286305}, "q04": {"type": "choice", "choice": "mid", "probabilities": {"low": 0.1992160677909851, "mid": 0.5181268453598022, "high": 0.28265708684921265}, "confidence": 0.07224955867479488}, "q05": {"type": "score", "score": 1.4385067540852203, "probabilities": {"0": 0.2317957212985837, "1": 0.31430735247411273, "2": 0.23749137707080314, "3": 0.21640554915650043}, "confidence": 0.007937943291352001, "legend": {"0": "a", "1": "b", "2": "c", "3": "d"}}, "q06": {"type": "noul", "noul": 0.016506596661809238}, "q07": {"type": "choice", "choice": "mid", "probabilities": {"low": 0.1992160677909851, "mid": 0.5181268453598022, "high": 0.28265708684921265}, "confidence": 0.07224955867479488}, "q08": {"type": "score", "score": 1.4385067540852203, "probabilities": {"0": 0.2317957212985837, "1": 0.31430735247411273, "2": 0.23749137707080314, "3": 0.21640554915650043}, "confidence": 0.007937943291352001, "legend": {"0": "a", "1": "b", "2": "c", "3": "d"}}, "q09": {"type": "noul", "noul": 0.20954438490618996}, "q10": {"type": "choice", "choice": "mid", "probabilities": {"low": 0.1992160677909851, "mid": 0.5181268453598022, "high": 0.28265708684921265}, "confidence": 0.07224955867479488}, "q11": {"type": "score", "score": 1.4385067540852203, "probabilities": {"0": 0.2317957212985837, "1": 0.31430735247411273, "2": 0.23749137707080314, "3": 0.21640554915650043}, "confidence": 0.007937943291352001, "legend": {"0": "a", "1": "b", "2": "c", "3": "d"}}, "q12": {"type": "noul", "noul": 0.016962776622575684}, "q13": {"type": "choice", "choice": "mid", "probabilities": {"low": 0.1992160677909851, "mid": 0.5181268453598022, "high": 0.28265708684921265}, "confidence": 0.07224955867479488}, "q14": {"type": "score", "score": 1.4385067540852203, "probabilities": {"0": 0.2317957212985837, "1": 0.31430735247411273, "2": 0.23749137707080314, "3": 0.21640554915650043}, "confidence": 0.007937943291352001, "legend": {"0": "a", "1": "b", "2": "c", "3": "d"}}, "q15": {"type": "noul", "noul": 0.026299577696241024}}, "usage": {"input_tokens": 2294, "output_tokens": 0}}} +{"id": "len-330-0230", "request": {"state": "ticket invoice request latency system update ticket service order retry retry retry order region ticket system latency invoice invoice service customer request ticket update update invoice queue error delivery queue retry account order ticket request retry region account invoice system retry region delivery error delivery order retry invoice latency queue account account account account delivery customer ticket error error retry update order request queue error status error region delivery update latency system error invoice system status request account queue account invoice invoice service status region update invoice request latency account customer retry delivery system request request error region queue delivery retry status delivery invoice latency order delivery retry customer region customer error order order customer request invoice error request system request invoice queue request status update latency system account ticket region status queue latency error invoice retry status error queue retry customer region order update system region account request customer order delivery error update region status retry system delivery region latency latency order queue status error update latency order request customer region update region update invoice service service order update system invoice ticket latency customer invoice queue status latency region queue status update request account queue ticket status invoice account error service invoice order order status retry ticket service customer request ticket update system region latency update region system ticket customer error service request service account invoice customer update customer order customer account", "questions": {"q": {"type": "choice", "instructions": "Which word appears most often in the state?", "criteria": {"order": null, "invoice": null, "ticket": null, "latency": null}}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "choice", "choice": "latency", "probabilities": {"order": 0.20152367062227447, "invoice": 0.23827409319314777, "ticket": 0.17886898721168878, "latency": 0.381333248972889}, "confidence": 0.03334887559968103}}, "usage": {"input_tokens": 330, "output_tokens": 0}}} +{"id": "len-1024-0239", "request": {"state": "customer service invoice latency request update invoice status error error latency update region region request latency ticket latency status latency request error retry error error region invoice update delivery ticket delivery account service request request ticket customer service delivery update order status update region system order request order system order update retry update customer retry queue invoice system order latency ticket queue request error service update region update latency system queue update system latency queue retry error system queue request status queue delivery delivery retry latency order invoice region delivery region region ticket error queue account service delivery service status error update service account order order order order latency system retry invoice ticket request system service ticket retry ticket customer queue region region ticket retry request status region latency customer system queue customer order invoice error status latency system error error retry status latency latency latency ticket update customer system delivery region latency order status system error account service invoice latency invoice system delivery invoice error delivery retry invoice system error service system ticket invoice system error request request order region status latency delivery invoice error status update delivery region region order customer invoice latency queue invoice service account delivery system request update region latency customer service service ticket service account system delivery update update invoice region customer system system error latency system request service invoice order order status region account delivery order status order order status region status latency service latency queue customer retry queue customer latency retry region customer status status region queue status delivery order error update delivery service queue queue retry update service queue customer region ticket status customer latency error order order order region retry queue service update account order error latency delivery delivery ticket status queue customer region region system retry delivery request service account system update account error service latency account error account invoice account system order latency request request ticket system status system retry service region error system region update request customer region latency invoice region system ticket latency error system delivery delivery region system service status queue delivery status invoice system retry delivery order retry order status latency system service customer system delivery customer order order customer latency latency retry request error service update queue account ticket system account latency service account region order ticket request latency retry order service retry delivery delivery status status ticket status queue request delivery request account request update order service retry order invoice error update latency region customer region invoice region request ticket account order queue ticket error system update delivery status order update system customer queue customer system invoice error retry account queue system invoice order latency update service invoice error latency latency update system ticket queue system order delivery queue region account queue update status region status system latency customer account retry delivery system account ticket delivery status customer region error status account retry invoice account invoice retry status service order invoice retry service status service customer customer update invoice update update account queue customer account order customer update retry delivery queue error latency delivery order delivery system system status delivery status error order service latency error retry service customer request ticket account account customer retry region order service queue order delivery queue service service invoice ticket service invoice queue request region queue error system queue customer ticket ticket status queue queue delivery delivery customer region region error queue invoice latency retry update region system delivery error ticket update error latency latency service queue system update update account error order retry latency retry update region request order latency request update delivery ticket error service queue ticket retry error account invoice order order queue invoice customer queue status account queue delivery service invoice delivery status status error queue order queue delivery queue error invoice update queue update request customer account queue update order queue invoice region system status retry invoice order ticket status ticket request invoice customer order update region update queue system update account error ticket ticket request latency region delivery order retry invoice region update invoice status update order account region customer status latency region latency retry customer customer update invoice retry system queue status delivery delivery service customer order status order order request latency delivery delivery retry error status request update status queue region latency delivery latency delivery status retry status latency request order invoice request latency error status queue order queue status account account update system update system system delivery customer invoice invoice account status status latency order system customer account service request status status order customer request delivery status ticket invoice retry retry error queue request order delivery region request error service region retry service customer request latency queue system update system invoice latency queue region delivery ticket status invoice update system order retry queue order error latency invoice update ticket error order ticket delivery system system ticket latency region invoice ticket customer retry error order delivery region status status account invoice request ticket queue queue service queue system error ticket request region request queue retry system latency error account delivery system queue error order customer delivery retry system error retry status request request retry region system update request error status delivery customer account delivery invoice region service latency update customer error system status delivery region status latency customer latency update region request account update status delivery retry error queue delivery latency customer update queue latency invoice ticket order region invoice service ticket order customer customer ticket queue error retry delivery invoice queue request invoice ticket status delivery status queue update latency request service queue account", "questions": {"q": {"type": "choice", "instructions": "Which word appears most often in the state?", "criteria": {"order": null, "invoice": null, "ticket": null, "latency": null}}}}, "response": {"model": "d3-lite", "answers": {"q": {"type": "choice", "choice": "latency", "probabilities": {"order": 0.031174795527897054, "invoice": 0.06244289840137829, "ticket": 0.016227770506062895, "latency": 0.8901545355646617}, "confidence": 0.6741255017734924}}, "usage": {"input_tokens": 1024, "output_tokens": 0}}} diff --git a/.github/script/sweep_common.py b/.github/script/sweep_common.py index b25725ff1..db2543f4e 100644 --- a/.github/script/sweep_common.py +++ b/.github/script/sweep_common.py @@ -223,6 +223,8 @@ class SweepTask: name = "sweep" needs_asr = False needs_embed = False + # A decision model tag (e.g. "d3:lite") served with --decision; empty for none. + needs_decision = "" csv_header: list[str] = [] @staticmethod @@ -320,6 +322,8 @@ def start_server(self, append_log: bool = False) -> None: "-a", "1" if self.needs_asr else "0", "-e", "1" if self.needs_embed else "0", ] + if self.needs_decision: + cmd += ["--decision", self.needs_decision] self.log_event( "server_start", detail=" ".join(cmd), diff --git a/.github/script/sweep_decision.py b/.github/script/sweep_decision.py new file mode 100644 index 000000000..158ce0d5a --- /dev/null +++ b/.github/script/sweep_decision.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Sweep the decision models through POST /v1/systemone. + +Per model (served with `--decision `): + goldens stored requests answered by the upstream BF16 model (assets/d3_goldens.jsonl): every answer has + the upstream shape, top-1 agrees on at least --min-agreement of them, mean |dp| at most --max-dp + types one choice, one noul, one score, and all three in one request + two-letter a 27-option choice (the 27th option's code is "AA") + invalid a malformed question -> 422 {"detail": {"invalid_questions": ...}}, nothing run + too-long a prompt over the input limit -> max_length_exceeded inside a 200 reply + docs the example request of docs/docs/models/d3.md: the server's reply must decide what the documented + reply decides (choice, noul side, most likely score), so the docs cannot drift from the server + +Decision models are absent from /v1/models, so the model set comes from `flm list` (family "d3"). Each decision +model gets its own server (--decision takes one tag). +""" + +from __future__ import annotations + +import json +import re +import time +import urllib.error +import urllib.request +from pathlib import Path + +from sweep_common import SweepTask, run_task + +REPO = Path(__file__).resolve().parents[2] +STATE = "The order arrived damaged yesterday. The customer has a receipt and asks for a replacement today." + + +def answer_shape_ok(answer: dict) -> bool: + kind = answer.get("type") + if "error" in answer: + return set(answer) == {"type", "error", "message"} + if kind == "choice": + return list(answer) == ["type", "choice", "probabilities", "confidence"] + if kind == "noul": + return list(answer) == ["type", "noul"] + if kind == "score": + return list(answer) == ["type", "score", "probabilities", "confidence", "legend"] + return False + + +class DecisionSweep(SweepTask): + name = "decision" + csv_header = ["Model", "Case", "Detail", "Latency (s)", "Status", "Error"] + + def __init__(self, args): + super().__init__(args) + # One server per decision model: the tag is the server's --decision argument. run() restarts + # the server for every further model. + self.needs_decision = (args.models or ["d3:lite"])[0] + + @staticmethod + def add_arguments(parser) -> None: + parser.add_argument("--min-agreement", type=float, default=0.85, + help="Share of golden answers whose top-1 must match the upstream BF16 model.") + parser.add_argument("--max-dp", type=float, default=0.05, + help="Largest allowed mean |probability difference| against the upstream BF16 model.") + + def select_models(self, catalog: list[dict]) -> list[str]: + return [entry["id"] for entry in catalog if entry["family"] == "d3"] or ["d3:lite"] + + # -- http ------------------------------------------------------------- + + @staticmethod + def _json_or_raw(raw: bytes) -> dict: + """The reply body as JSON; a body that is not JSON (a proxy's or crash page's 5xx) becomes + {"_raw": its start}, so the case fails instead of the whole sweep.""" + text = raw.decode("utf-8", errors="replace") + try: + body = json.loads(text or "{}") + except ValueError: + return {"_raw": text[:200]} + return body if isinstance(body, dict) else {"_raw": text[:200]} + + def post(self, body) -> tuple[int, dict, float]: + data = body if isinstance(body, bytes) else json.dumps(body).encode("utf-8") + req = urllib.request.Request(f"{self.base_url}/systemone", data=data, + headers={"Content-Type": "application/json"}, method="POST") + started = time.monotonic() + try: + with urllib.request.urlopen(req, timeout=self.args.request_timeout) as resp: + return resp.status, self._json_or_raw(resp.read()), time.monotonic() - started + except urllib.error.HTTPError as err: + return err.code, self._json_or_raw(err.read()), time.monotonic() - started + + def case(self, writer, model: str, case: str, ok: bool, detail: str, elapsed: float, error: str = "") -> None: + self.record(writer, [model, case, detail, f"{elapsed:.2f}", "ok" if ok else "error", "" if ok else error], + model=model, error=None if ok else f"{case}: {error or detail}") + print(f" {case}: {detail} -> {'ok' if ok else 'FAILED ' + error}") + + # -- cases ------------------------------------------------------------ + + def _goldens(self, writer, model: str) -> None: + agree = total = 0 + dps: list[float] = [] + bad_shape: list[str] = [] + elapsed = 0.0 + for line in open(self.asset("d3_goldens.jsonl"), encoding="utf-8"): + rec = json.loads(line) + status, body, t = self.post(rec["request"]) + elapsed += t + if status != 200: + bad_shape.append(f"{rec['id']}: status {status}") + continue + for key, want in rec["response"]["answers"].items(): + got = body["answers"].get(key, {}) + if not answer_shape_ok(got) or list(body["answers"]) != list(rec["response"]["answers"]): + bad_shape.append(f"{rec['id']}/{key}") + continue + if "probabilities" in want: + p, q = list(want["probabilities"].values()), list(got["probabilities"].values()) + dps.append(sum(abs(a - b) for a, b in zip(p, q)) / len(p)) + agree += max(range(len(p)), key=p.__getitem__) == max(range(len(q)), key=q.__getitem__) + total += 1 + elif "noul" in want: + dps.append(abs(want["noul"] - got["noul"])) + agree += (want["noul"] >= 0.5) == (got["noul"] >= 0.5) + total += 1 + share = agree / max(total, 1) + mdp = sum(dps) / max(len(dps), 1) + ok = not bad_shape and share >= self.args.min_agreement and mdp <= self.args.max_dp + self.case(writer, model, "goldens", ok, f"top-1 {agree}/{total}, mean |dp| {mdp:.4f}", elapsed, + "; ".join(bad_shape[:5]) or f"agreement {share:.3f} / mean |dp| {mdp:.4f} outside limits") + + def _types(self, writer, model: str) -> None: + questions = { + "route": {"type": "choice", "instructions": "Which team should handle this?", + "criteria": {"returns": "Refunds and replacements", "billing": "Payments", "general": None}}, + "urgent": {"type": "noul", "instructions": "Does this need a reply today?"}, + "anger": {"type": "score", "instructions": "How upset is the customer?", "criteria": ["calm", "annoyed", "angry"]}, + } + status, body, t = self.post({"state": STATE, "questions": questions}) + ok = (status == 200 and list(body.get("answers", {})) == list(questions) + and all(answer_shape_ok(a) and "error" not in a for a in body["answers"].values()) + and body["usage"]["input_tokens"] > 0 and body["usage"]["output_tokens"] == 0) + self.case(writer, model, "types", ok, f"status {status}, {len(body.get('answers', {}))} answers", t, json.dumps(body)[:200]) + + def _two_letter(self, writer, model: str) -> None: + criteria = {f"option{i:02d}": f"Option number {i}" for i in range(27)} + status, body, t = self.post({"state": STATE, "questions": {"q": {"type": "choice", "criteria": criteria}}}) + a = body.get("answers", {}).get("q", {}) + ok = status == 200 and answer_shape_ok(a) and len(a.get("probabilities", {})) == 27 + self.case(writer, model, "two-letter", ok, f"status {status}, {len(a.get('probabilities', {}))} options", t, json.dumps(body)[:200]) + + def _invalid(self, writer, model: str) -> None: + status, body, t = self.post({"questions": {"ok": {"type": "noul"}, "bad": {"type": "rank"}}}) + ok = status == 422 and body == {"detail": {"invalid_questions": { + "bad": "unsupported question type 'rank' (choice, noul or score)"}}} + self.case(writer, model, "invalid", ok, f"status {status}", t, json.dumps(body)[:200]) + status, body, t = self.post(b'{"questions": ') + ok = status == 422 and str(body.get("detail", "")).startswith("could not read the body as JSON") + self.case(writer, model, "not-json", ok, f"status {status}", t, json.dumps(body)[:200]) + + def _too_long(self, writer, model: str) -> None: + status, body, t = self.post({"state": "word " * 33000, "questions": {"q": {"type": "noul"}}}) + a = body.get("answers", {}).get("q", {}) + ok = status == 200 and a.get("error") == "max_length_exceeded" and body["usage"]["input_tokens"] == 0 + self.case(writer, model, "too-long", ok, f"status {status}, {a.get('error')}", t, json.dumps(body)[:200]) + + @staticmethod + def _decision(answer: dict): + """What an answer decides: the choice, the side of a noul, the most likely score level.""" + if answer.get("type") == "noul": + return answer.get("noul", 0) >= 0.5 + probabilities = answer.get("probabilities") or {} + return max(probabilities, key=probabilities.get) if probabilities else None + + def _docs(self, writer, model: str) -> None: + doc = (REPO / "docs/docs/models/d3.md").read_text(encoding="utf-8") + m = re.search(r"-d '(\{.*?\})'", doc, re.S) + r = re.search(r"```json\s*(\{.*?\})\s*```", doc[m.end():], re.S) if m else None + if not m or not r: + self.case(writer, model, "docs", False, "no example request and reply in docs/docs/models/d3.md", 0.0, "missing") + return + request, documented = json.loads(m.group(1)), json.loads(r.group(1)) + status, body, t = self.post(request) + answers = body.get("answers", {}) + ok = (status == 200 and list(answers) == list(request["questions"]) + and all(answer_shape_ok(a) and "error" not in a for a in answers.values())) + # the documented reply must decide what the server decides; its numbers are rounded + drift = [k for k, a in documented.get("answers", {}).items() + if self._decision(a) != self._decision(answers.get(k, {}))] + ok = ok and not drift + self.case(writer, model, "docs", ok, f"status {status}" + (f", docs decide differently on {drift}" if drift else ""), + t, json.dumps(body)[:200]) + + def run(self) -> None: + models = self.resolve_models() + if self.args.no_server and len(models) > 1: + raise SystemExit("--no-server attaches to one server, which serves one decision model; pass --models ") + handle, writer = self.open_csv() + try: + for model in self.iter_models(models): + if model != self.needs_decision: # each decision model needs its own server + self.stop_server() + self.needs_decision = model + self.start_server(append_log=True) + print(f"\n--- Decision: {model} ---") + self._goldens(writer, model) + self._types(writer, model) + self._two_letter(writer, model) + self._invalid(writer, model) + self._too_long(writer, model) + self._docs(writer, model) + finally: + handle.close() + self.write_summary(models) + + +if __name__ == "__main__": + raise SystemExit(run_task(DecisionSweep)) diff --git a/.github/workflows/model-sweep.yml b/.github/workflows/model-sweep.yml index 0c5b01e86..aeb59731c 100644 --- a/.github/workflows/model-sweep.yml +++ b/.github/workflows/model-sweep.yml @@ -212,7 +212,7 @@ jobs: # others mid-sweep. fail-fast: false matrix: - # sweep_embedding.py and sweep_audio.py are ready and tested, but stay + # sweep_embedding.py, sweep_audio.py and sweep_decision.py are ready and tested, but stay # out of the matrix until llm and vision have proven themselves against # real hardware. Add them back here to enable them; nothing else needs # to change. @@ -513,7 +513,7 @@ jobs: strategy: fail-fast: false matrix: - # sweep_embedding.py and sweep_audio.py are ready and tested, but stay + # sweep_embedding.py, sweep_audio.py and sweep_decision.py are ready and tested, but stay # out of the matrix until llm and vision have proven themselves against # real hardware. Add them back here to enable them; nothing else needs # to change.