From e078c2065200ee73e879456c1fe15049a04e3e90 Mon Sep 17 00:00:00 2001 From: vectorvoyager Date: Sat, 15 Aug 2026 13:11:33 -0500 Subject: [PATCH 1/2] Discover a UI flow with NVIDIA NIM and compile it into a reusable capability. --- .env.example | 4 + LICENSE | 2 +- README.md | 22 +- evidence/.gitkeep | 0 evidence/discovery.json | 78 ++++ evidence/discovery.png | Bin 0 -> 19910 bytes evidence/lookup-member-savings.json | 110 ++++++ evidence/replay-member-not-found.json | 23 ++ evidence/replay-success.json | 34 ++ package-lock.json | 504 ++++++++++++++++++++++++++ package.json | 3 + src/cli.ts | 164 +++++++++ src/discover/compile.ts | 192 ++++++++++ src/discover/decision.ts | 62 ++++ src/discover/loop.ts | 323 +++++++++++++++++ src/discover/nim.ts | 182 ++++++++++ src/discover/prompt.ts | 33 ++ src/discover/scripted.ts | 57 +++ src/discover/targets.ts | 32 ++ src/surface/playwright-web.ts | 60 +++ src/surface/surface.ts | 9 + tests/discover/compile.test.ts | 108 ++++++ tests/discover/loop.test.ts | 95 +++++ tests/discover/nim.test.ts | 47 +++ tests/helpers/recording-surface.ts | 6 +- 25 files changed, 2146 insertions(+), 4 deletions(-) create mode 100644 .env.example create mode 100644 evidence/.gitkeep create mode 100644 evidence/discovery.json create mode 100644 evidence/discovery.png create mode 100644 evidence/lookup-member-savings.json create mode 100644 evidence/replay-member-not-found.json create mode 100644 evidence/replay-success.json create mode 100644 src/cli.ts create mode 100644 src/discover/compile.ts create mode 100644 src/discover/decision.ts create mode 100644 src/discover/loop.ts create mode 100644 src/discover/nim.ts create mode 100644 src/discover/prompt.ts create mode 100644 src/discover/scripted.ts create mode 100644 src/discover/targets.ts create mode 100644 tests/discover/compile.test.ts create mode 100644 tests/discover/loop.test.ts create mode 100644 tests/discover/nim.test.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f0a3b46 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +NVIDIA_API_KEY= +NVIDIA_MODEL=nvidia/nemotron-3.5-lightning-30b-a3b +NVIDIA_BASE_URL=https://integrate.api.nvidia.com/v1 + diff --git a/LICENSE b/LICENSE index 61b5237..0ce727f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 Jeesh +Copyright (c) 2026 Jeethesh Reddy Gattupalli singalreddy Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 791f6a2..79fc98b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ LLM-driven computer-use system that discovers UI workflows, records them as reus ## Status -The capability schema, local core-servicing app, deterministic replay, policy, and human handoff are in place. Discovery is not built yet. +Discovery, deterministic replay, policy, and human handoff are in place. A live model call is required only for discovery. ## Setup @@ -13,9 +13,13 @@ Requires Node 22.12+. ```bash npm install npx playwright install chromium +cp .env.example .env npm run check ``` +Add NVIDIA API key in `.env` as `NVIDIA_API_KEY` to run discovery. Create a key at [https://build.nvidia.com/models](https://build.nvidia.com/models). +Replay and `npm run check` do not need a key. + Start the local target app (no API keys): ```bash @@ -26,4 +30,18 @@ It listens on `http://127.0.0.1:4173/`. Member `10001` has a savings balance; an ## Demo -Not available yet. This section will have the commands to discover a goal and replay the resulting capability. +Discovery talks to NVIDIA NIM (`nvidia/nemotron-3.5-lightning-30b-a3b`) and writes a capability under `evidence/`. Replay does not call a model. Override the model with `NVIDIA_MODEL` in `.env` (must be a chat NIM with tool calling). + +```bash +npm run discover -- --goal "Look up the member savings balance" --param memberId=10001 --out evidence/lookup-member-savings.json +npm run replay -- --capability evidence/lookup-member-savings.json --param memberId=10001 --out evidence/replay-success.json +npm run replay -- --capability evidence/lookup-member-savings.json --param memberId=99999 --out evidence/replay-member-not-found.json +``` + +Without a key, `npm run check` still exercises discovery against the live app using a scripted model. To dry-run the CLI itself: + +```bash +npm run discover -- --model scripted --param memberId=10001 --out evidence/lookup-member-savings.json +``` + +`--model scripted` is a test double. Evidence meant to show a real discovery run must use the default NIM path. diff --git a/evidence/.gitkeep b/evidence/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/evidence/discovery.json b/evidence/discovery.json new file mode 100644 index 0000000..58a3a0f --- /dev/null +++ b/evidence/discovery.json @@ -0,0 +1,78 @@ +{ + "goal": "Look up the member savings balance", + "model": "nvidia/nemotron-3.5-lightning-30b-a3b", + "events": [ + { + "type": "acted", + "detail": "navigate entry" + }, + { + "type": "decided", + "detail": "fill Member ID" + }, + { + "type": "acted", + "detail": "fill Member ID" + }, + { + "type": "decided", + "detail": "click Search" + }, + { + "type": "acted", + "detail": "click Search" + }, + { + "type": "decided", + "detail": "extract string" + }, + { + "type": "acted", + "detail": "extract string" + }, + { + "type": "decided", + "detail": "extract 1240.50" + }, + { + "type": "acted", + "detail": "extract 1240.50" + }, + { + "type": "decided", + "detail": "extract 1240.50" + }, + { + "type": "acted", + "detail": "extract 1240.50" + }, + { + "type": "decided", + "detail": "extract 1240.50" + }, + { + "type": "acted", + "detail": "extract 1240.50" + }, + { + "type": "decided", + "detail": "extract 1240.50" + }, + { + "type": "acted", + "detail": "extract 1240.50" + }, + { + "type": "decided", + "detail": "extract 1240.50" + }, + { + "type": "acted", + "detail": "extract 1240.50" + }, + { + "type": "decided", + "detail": "done: Goal accomplished - savings balance extracted from member detail page" + } + ] +} diff --git a/evidence/discovery.png b/evidence/discovery.png new file mode 100644 index 0000000000000000000000000000000000000000..ce1f94b9923e345791f3256d6afd74307f1696f9 GIT binary patch literal 19910 zcmeIacUY5YyDy4kL1mP&(A$g*qS6GVNgGFzks_k>E)kI4lulw9M5#846d6GT>4*>@ zK#&roOYZ~-y(B;=34~X*Q^L0_3L3;&~p zt*hbj@ynMRq6av*dw$(Kjx$9qai!gpl(oCaZ;)=Mn0Cy-j;E;MxPgJeZzqpAb-l6P z3K_WG?m@Jlk2r?k3h_#DX-`)6ad?VX-$}kD){_4XK5u+B|JDI`J@iuz{QuX>Y$G7Sg@h zM(>oVYqg^X=gmvK>-l7N@+`1{AFPazE%fHB z(nw+XrRLKKyXf7Gt+vEovyhE*NgiXK<$L{mvd$`wtQbpHQroc{8r$_Jo~-pZqb*;C z;+9L95(U8zNKrvNQr{So_+623yj}3l@*6^w%d9-JNu2aKypXoY$;P&VFZr{B%_h(( ztP)KrpM6sCfMr@RCK98%_WVxWX+mOzw6n&8g^Y#^O!il=Qt{*&BdnLf6ztoh7LP7x+< zz;_<&v+U?mbT)$fHJiHM^tM=9R$pN>V5 zZq<(~V%)EKbblPB`klqIm|4rEI~Efxa!G$Qi@6hl!tQPh2{Z~W;{1pz)@P2|E)7so zPwv#A1tdrcaUIHzIMe9jXVU|c6&0SlEeuS^YQy7Y5=E#eB*%Dz$|_KBz z&r^LVs!*9el}VL{4Qk5Ajrk4S$t%FQ$0W+kTX6)rvbRYRRx zSf|0y?&;S`#sX$PxMAJod}c=8UlAt|6i0iC*4uYBcFx)O&%6~4`%!sO1~WL@XF-hG zB?dTVp13iP<~^;wmpwY~Rz{bBmRo6?f?l6lro)o*$4{B5)f!Kw&d|ng5k-VrhCPV6R=8^~or^{?bnrtZ1y;1SF4>w^ zt@wy-zn$eBDaJWUHEw*jrt^_kQmV(ZDeKmT=X-N3%ZJOnIMgS1rsH3Shvz5D+51X1 z{)8(W4%rMSbRS)$vf_!1kIQz_L+kTC(s)KJO~fn_i`kmLQ0Cn0zWT;gef@hMkJT5H z@!BsFC4CjuL0cI67HJl%S;J3gmKU{hBgNc-gSjC4D*!a@nAF6D8UwqRA7#Hw6sw;O z@>03`?a#81<&pwwtlKbiRwz%V^88Yq#hCI;f&zWCi*hRZ?eYozVB<1en}VNac}n1d z@v+pL0&B64Wuq0#iTK-!AJbQ5h>{;uSlj*Ia_@47kjpz}CA4DR@plg;=o6M?$<{t? ziy1NJ3sx}5;jIpWi8$BrZuRsR*5{|2u&_u5+N@K{@OF@e8#^5H~K4SlS0&xxE=9BY-|r>9S`kc5ClXo-8WUNSOm(W;fPWR>*R{X zz}a`WkozWI%$^+J7`u_IFIV^d{Rtfn#SW6|*W9)D9U3Ac91j+aWp-QCWYb(n15?^l zQlrn_E46L)ce>jDR3Kl?kIe9I2qM$RvvaIlv}fvGmRY!VX0;YI6ib)chE;72osUEP z#vf+BE&3S`as*GiOEdv8Girp1Hwt_|}kt_Q-DCr_Q5Q)@RP4zC41Zd{^5Xu$_q5+#yn4NyXQ; z91&1B%N%&gszOa1S=)LolL7Cn5^x{zp|%MSyua-Jb(v_7U}73zV!U)g@`FAF**)rj zKru>o-S--Ji6PWB&SU0y)^5Ao&_(~v!So@ljbf>wqWchSwOdDHt7_|-W;|i!vmd3- zCgg#?o6e`-jx-cuEczvgm6nz6=hUV%ixIknoj2G6K%LI2i zw4GzAb*yGS`ue1!nMe^bnc3)%p?seh^Tr0RHFXSOJ^Cu?{OQ+H;xQ$r(m$x*+WaP} zI1GA8_Hlb$tqpUneWRc9l4Ul*+|g);HA7qK(0=qh>5-&>oh3E%ZfWik{5Mlsf`L#7 z>MTZ~>0l8m;u)6rAO%OcV*gIyO6{mG&SQP<6UdjbAj%OGufQs`*W~qKfpv<8>Z$W~ zu5e6UyFU0bMK5uxp7+G>3Znt#M3+xh+3{KwJM?y_3*c9Z^b*dKB4n5sirX6BWi;GS zr`hTnuT4z(F7t!X6SsJ!G^i9Ii~2}d_ouBi(HZir?5Xfqlnd_h{$6qS(xdvKB~IWf zRtea&;rO+*>B*AG#Mw|0OyIKv);)1A`@B+$U!^{YZn$R&PLY7p+9ON@UmJY!H=PF~K55LupqFUJDO54-iRx5$Wd{r<(W$gDLS zB}wnpLh+y_DEuylF1^{z%v7|=%;j6g`jnypX+`&rM@_NH^x8Va`10rZ82dC1Q zqD_96T;^C15h0K|PU1z2mHq90lG8r%9g{Ncc310}lRO(wkEYzL(2ZJ*O$lon~BQe6ds;f-5eI@ zs=j)zlDYjX#kWE+z5YS}vcaW?@kWO0L6nqaDcV>;wOSxU1wmLZqoy;{462RN*MrI*eM)^X**U)_Zy?5LCCM9U)TdLKOc&f zgJt{UA=}@7_<`x~q00Ta7x0^Fvj4jL5R_P_8QH^p@7mIjN1*5?c? zEt83AbALTaRP=baTOTQ+!F(Skd~O8L4%Khz)3$_}D4%^Npz7~gZ3&BULSI#u`zdJG zp4skV3fkh-U=(`# zb?Yfst<-W9i;T9uy--j-lUO_2rZBuD&RatwMBz4vTsZu;hCT51N5(1IQ!Vl?eU@aO zC96|5()(5l@6>6Fj2#kBMpogW(YG~r&p-P(WGZh4>KRV&=drvQ*p_9Go))fC@K!w} zkf80;DC9R1xR^&@n@REzelh|c@8NMSI^TF!fm}!s7&H6a$8g=orunxsZ@{rBT*&#x z_U_UH=K||0ZDQ398F#_>vLZ2He>*X>0cR^WQ4mPx*)a-8VX^hfcu zH_uOqtM9INUkN+9%b<3kcH*QgO7C?}jz8ga@5wT_lZ0VXQ%Q|69q}N-iuDvnzv!o` z1NY;r=`%^orNi!&sx5bbF5@-kLcDe0e6NX(kgVO+#lE~4&=do`Y1WEeNK*EhSzjLB>ekl~f*o=tRt8X`6vzByVE61Iqz(^X6+y2xiPODz zw>MB47aZGzNl}W`+Y32uaA&}K)~)4uxP}7Y&K(L~#3WZQNx{o`aLqbPX{%8LQ+Yo( zT2Oh7xkKrQ7f^H;3=;}lC*vsS%|S--lO-2JE+a*EI)xsM zi7frsECAP8H2W@yT$_R%xtxOP?Ug#t4hpyx>%HtpTjNEiPCeDZ6dC)|r{i->3ea8G z#gFh`%v#?*Lr}epp-qI{>#yD3dsJyW*2MZOwSxYVhm2;dHo^+}z7?9eMS3QU?!DyPp|&N7MHj!)SBc*tla&g2AM!lyjqn(M5 z{vCTkM%SP2<;aO*bnM%Y14B3E$Gk?Ci(C60PDDDCRt9G4%#FNmj2nJ;GjzkNj|cJ) zVElLu#ssGaPl>Sy{Z^sQ zQU28vPX*Sdp+MB$y^?AC?ZMhqlRcUPYlZVY<{{u(c)3+mhxNg;F3!XYP3Pu6BnZ$`C=I#^3iX0DhU9 ziiBYsg@fJ<`rLKwaeaMS4-MxkN@3`Xg#JW&c{vUB^)kEAo7l({zhO75Yq_-HbaO1K zrOU(0y)#WS_iR;;Qi)4``Dtl`J8b}3p77=^3Q+@Z;h(Auw9%m-y5IAIEubEiEVVl` zn;w1X+z zAg_&k5zf4fqzB+nz^-(&w@p^4+vld!^f-^o)oZ%*S~0>xtlJsH8KKU1h_}==GG%Mr z_t|H*9mFa{r-0F0fbIo6_QuNWdUanGYaW$6>}EiWs>~v0TG_G3y_~)8?)?0hD^K58 z`vBs{?$n=Yl5>GtFRI@@@`RmJT2iY+22J9^-`tNPL@H0exjh!LH3|Uzvl4DDF%jiL#;nU2&{srarFYw~@jC|L{R@lbwc z@4LG5y)uyJ<%Tna#!LFl7Y9%=Rg~(THJsbf7qcEt>eY1jvWce%B~gTZ2kywKN&B_b z2F1o(8yIxTxv@t5h+J_Xq_}R6a7&q=ROc3GEYHIJ)}LpgXdxZC748_}ALe^`zjao; z4Z8fkj^BLO^j}YoI(L0&5@D`{;Th91e8|pPX&&%=#wH#7dSxCGhy*DMth?1*MTh#1 zq?`=PZ1gd&6G&KU1~Ii#uKwiKbMbd)M%wUU<@POcq4HiY9xU`Y4SjKXD_%`)S1EDn zZ;xvM5;HkC&SUNW9)g01=bO{jH|@K&0l_6DDS#D@$zIohonE{;%oG|w_)VYbvPxVw z+-CPba8%GAz@L-<;u%)kSkr6qE%Im(;D_qOpC{~;Ee=&Pw-?Xe{usR4z=Pji_TUKM z16)rn8}%J_)A11W={Et_K^`eDQg!Le?Y9n+I=WP|my-dc_Idgfs>bg3{_ z>QL+3eHI?mel}-b##SB*Fwj`AcqDz+z`?!;rCV71>d*>8>$I=$@AW7R9ab$RGG-Jr zFzmGI;}Jvt@HualJ<$VZ*EKy#2OWUlS}24QibMC(XWEiv7@BwI~FKgY}FhUKmWVt9CA>m$>OY0 zQdbXhrrv>#RCMWE0)jy}0P2_rZT#m*M!a=JpSeryAU&;m zs=QFIoZZ;LKJ!M3-DC5zVW4g1?hfU1i*%?wRACh$kKMZY_Hy%nSqXJ0g0%vgc_PYU z0MIRD=B;C#buoai#!Agz4v~OTNuF%;5@$qQ)y`X21#W!QO9B>pZNKRW+#1Ks@-6Q# z^t@%So>iH^@&c)o`qv+c63rohnH2PS4Zpt5qJJf$MVzXFxVfG3x<6jHQ5@zj5nQ@+G%A zQ?-%Az2>laFk29h1Z>g2%iI6cApU>*4`@yMS7uW&Odv%^E@$W?*!M+(Y#~JQW%Rcu z%DccC>WoOXrl?@c%9P-aY-4Zj*`s>*+l!}11Q2Kmm50-Y$_$kz|LUd3-Zw{hVVFRcjSF7ousxQJiYPxZM)K3*o zYPjqGi46jX7$NLS3IAs>Jp^>qQV(|w+P4l?X+DeTDF?iR5l zRSuy3wZ6Ktu-f>;y-4Y4F8}&!BU~b1X)(8$+s6>tdbd%7X%MKFsva_V)1@ctSl!qW z#gRfAJo+VhW1)9(pa_+#I{8eL$=t%nxB+gP;jSidO*5+Z(L({ zUY}kJ7&!o?l;mFFRQu;4xS`=RUvB_J{ltq-kn>x#8uUV=14cZ&?5;t=5+ zi6JGR@{upVK8Jmv7<=6G2hC7>AHu4UVwZ>2I*QS_qVEwt4}6z+YuooM@eKkT{qYQP0{)1hQmzOwW5 z^#EnMi@?nw^gOW)F0t84n)lNEd!Xjmw$95qi?{sRTnRAta+*F09CGSFi$MeYVmj1Y z@vATY*QIMRk9q>-pL(dn>3}j9z#^prsuDKL7Ytk!udLnkowrI}c9FrNDk7lIKt=e` zeP1Pky8v@r{OYW`WMl0kh_Ak1;e@lc3;rQubxj(#bbM4D&7c7QOB)le&?(ANm1f3k z$0mb4e6*21M(!mjv=*T;<4?E5`U;BMF9FBh`!XG8;a(2<6Pgq$Md<50EeDS9_oQh? zfX|ebqs^tqSXZY^MT7H7@f%a-9x8G1188;-(}+BY8?}9MpaW$4{)o$_5yu{FwFYjTPt0hE2eDJqIz-uYx7pp)q7j!+M@bq@j?jJwmsIYYzy z_T@3gQ;rXL(qZ9SzJ#P4mneJPaAtS-k8(IP@t=s^iU^W%8NEzJGL?s^lK)v>^uL1R zR~+MqPMlt!?-tKd>dw?pRrIh%Dnj34|Gt$Mzg*#0a4j2l$F-$uXpr0AUJxgvne^E~ z{VSkwN&X1&GRP|vb^4$ST?f^`cpPE5fOpsd;bAvD)%03%!1f6+q&!^gcxK)YKfq!r zxdR@qkN9+Ok(u{Z!HJhR_XdLDE9N>e+Y14LThJz{zOaB=k?p!CIw@l<$*z=b?^yzdKWzRIg|Y0PE8)z;Du+R(a;QY zTx*-;+@mEDa$mA?y-P=yH|Dw5l5_pDWAypPbch1C0Y#v>sz`7mq%RL+W27#?tr~wk z%rTBcIQ%jqKg?zu=kc4-Af_Qrgo&W3ZHgj+bMH;*@CQf_cd#SgIuIhK1*IQ?rt7T_aDcf?ELS0OwUl`$_^cNM77%@~BFo8u zg1eBMYy)pZM#g-O`W0axKz;+VJNu8k-yyfS*UyMdd#yO)S;yEHK7K~)U$cNK$sR3n zQtX_;IHzpxXwSRhdY%pip_o8^Xc|1 z7QgG(gKAZ!%6Eo^3Hh2%5#jI{LC90q*5A0#zqm~yNiE119AG}gQKrR!34}itx)hz2 z!!OR%6k**;2TxzCm!Od+6PgL@C7=NT6~onLxe{@(Mx2-C{(^6{CCW=g)0eTtXxR-U zS^4%$Sp6dTOZSb{dbuD01y|D_D;gvDsGxzf!|f&SE(&ox%Xp`2!^o=AghKS23#v!{{H?7Q+gT$8^=L8|Fj91F#1Cboc__$ z^(&xK0)Bi1H!|qln+^4LPT^S`gzA2PuAhUS%+b#*2aD4adF2{&{Q;}hWx$4 zxA(J-dj9xGTsPr`Ok)i->^Q8B*O$5DsR9kWK0Q#$@})7x!R1#=f7sq!#=%adY}qsl z`sl*hIvhg)DYAaE@@l2=7q6*6Nd!p5Q|{lIy4UeejiYWt&b-t$0Em8fS_X5L8dS?J z>_3j9R0}u`LBW^dK=>mu*6MMn{;2vYPy#w0szIAeWkQPXBXG|kP2mzB#%BNUgVNK~ z&}kr>RG)Tb`Wi$H=lxg4^%Mqwy_ODgNU{l}bi1EAZ25;z~A7e7HBAe&G{n~WF+PJ=Zh9LT`$ zuUL1wHu=trz$beo`h;XIqHE#ol>w_Yd{HJ7)Dd)?5b_y6xU*;Pfj8=vV9Q`I;bl?S zw|Q@(`w^J9X$a54rH!x$ZaAmJ2A@{2*mJ?5l^0?uFjG31Nx$cy-j3hy{`QUQ(0Im4 z)WRyM5v(|NJFax}Dv+)3fP9>H4E)OA` zCRw@GEP>w{rTjynONc#$NhLbS@TnFXm<6bLw3j*!KvN1C5`ZO`&vtM)>mc_!V8#Z9 zG@M2&9{gxRDus9$bT$q=gGH&Undu3A287RklCpPd|185nWfRuMzdnJQ-!&b8GKDiX z1pSw2Aro-pHo-G2~ zgbrLXRSDRA3 zo6jDb%1Awab`0b*LTOJ4UZ=dcy*`R+MF$sOt=7vOyx*-y3hubHm@C)+OPs#a#GgkL z27VR}7gpcsxt&fWd9Wjx*kYfd`;fY&(b%&dk7w9RPK^%WF|0~7(nN547jXLH*Ce-6 zGwYDfu+COEP@0j~Jj$ozOnrbCn7b5A2YWdz!p@{kK5QI7TU)y9fnx9*=y+)>yK-os zAo(MtRK;8hwlxoRWUB6ulz9~iKH?`0^4haNK3oD%7!84S?%nS~4HnyHk?aZ4IY5=y zesvTpCi1$^ayB%Zs)Nz{{A$av@sEIsMNjZdQ~jO@Rk`Dq?UXex!=1$GH|Rg)pkSg; zM;>xBz7+}9oPqRil&^i2<%G)gRrPMIM+EjXBpr@J5|3;@lD1iGfuRc+u$bco=rF-eYYm+$&i z7eY*Alx_G)GlIKF@C2n6>&%sZMRE!wSyp|OIj2IY^day$A_XiNAD+s#wb<{`{N`-Y z3UuAL4j1}S#tbTpeiBoKqhp!?;`FL5a}G3LqD&+tHXWoQR3l*-E`tYK^buHDjEBRaF+}k}ZgvSmYbbE(O;50p$TD`O%AVYX zq#2^JJTc)bP;?}P&EV-+6To?o2KLMjNbg}3YXd-J)(>XP!jwLMfNHqjAgF?Ix$gvB zqrmDF5VxZ3E~_dYkkZ!n|3YZ-+Jgvv-Vgm*1|+Br@C<6Kl`xHetn}e`Y#nbeWSSIM z7yTqOTzZvxY!Fgx-GuWKNXbPb{WnD3JZnk_R$u!j`m*`hi(^4xtUV*Rr7G^vdr!U8 zB5WcJgNfD5if@Ux@4AL;9D`;wJhAyMf0_@2mHQ={>K{I_UAtN3qD)yq_#MCdd^DuZMbUGo;^$xaMS+j&m^ ziN*84#il1zo~K#&n7!e4JZX=#41WCtox#D!t5eYUsFdH{sJV5%Ukh5CcEn1_tuFD#cUq%X<+;JZXz93Td&Kc;mNWK7=@ zp2eKzWWNHZ#D19EtK?7$2#Z9L=FkoS zy8tF)6S-xdHhI!}E#ZI<_Rqn~UL=-z5UbsWOQp`88-hlD8zyv5_9;hein`|C? z&S%W~=2~jS*IZQ8bZHd{6r1`1qcT0Jhhk_nojZ%3e<; z6#)L}#=r^%(=Iyb|L(_PWmz6_*6_tgs_UVQgtqk4LYr!RPUb zS%Q0U4b>dH_R$>=?2F^8CcMo)-(GyZgpX}b>y@A^wb@}9- z4PkfyNheH2c2z-~^L{~ExMofDG}uw3w})H`(2n#>hUi0RTZX`)*n(gm^h=v@4bq`N zl=z((5e@YBzb+$5J7hel5HQCP5fLnU2ZrDc;t3@UGyI%dzck)dG=}{yEn?Pk$VAdT zt5X5-udy7(F#DfPn<`DcxH4I=7s;zsC1wZ_8lrGF=d>HxJ$2;~^8qfg;Y%ry9}EP`xpcl+N@&k{~vN>nCx>|qxyod1~QAV!_4Y_fr#(7?|h2N&a?bs+;Bru2WE zrOA09&u&se+^y@^_}2-JfoZ+J(X6^a~~T@yadvRu`HU*S9n(FVd~HS$fM&}Qr{uNXg3bgh$)75*glWb zco`@Sh$lOkN$&J_;>Sj60#4qz|8^;XAse2l>uPgTF~0aB#Ylz zKxS(*>?Su4PdsinlJCN`NZyxHp)fnsNEp18q=8|6dnrmfj`(uP_}Jq$qc}g0!O!+^ zkevH+%b~(`->C*Tz1skB3C>{=oO1_S?&(pXA@E9FE~h$4m4RF2sucnf7sDmM;2kp7 zg3Q=th#;{w^h;oLB|i3p2SR6n_yWin%ems$fcXVkZy?AzWn2xB2F#Z@z@CGX$orQ- z+;-3?m~Csmdln{@V0_Ht!xE6{NqHrJ%!~lLoy}#K-@s_?b=24+>$N;s?09h*(Opon z$W&g@8syB;buj2p6;u>S+#c$fexcsBL*3A>}1M;b!GNG4wEfG7tD4|$7M;S143 z<(V-KigP*(f|gDgXgVe)C~gY%T_yxmD0h7kUXdT(_yOV2) zijvQVtYoeZu18f)?zn6qEgwhT-YCGN#OfN8-h({H!!~;kQCJMR?JAa#6P1i zPp0m`Z8f+-v;vCuWC{j42@sr}#oU8~@8#78qMqU2vjbj_d+ds36|_E7ZUo0$`8Kh) z1M5T$12#4;C&;2EWjOw}erX`+v^QXI5gC{gbrI!3Tz*12VZe83pa_{UsJF<4bVVlS z(n1!Ly8;~&;^5ua!AZPDeOU&g zU4al^hwnOaEr5kY^c_+ibM%C-Fjr}JHdP69^O-ST%SI=pee&{aQBK0%#w)Vd+Fg8s z>^l^rLxG6juXm>hA!=Bu?H~niaRant8}X!-zc%f9h%P@Mhf&e3>V1pAL@dz#P@sXq z&*5MqZ4MajXF%g3In(y)D0FO*1Qv1$9L#OXcDN3gF0=ehW0^j-u!zr_JU@^Q2)O=a zW*V3@;DeyNIMNT;6($`%HUVT|1Fmrl={~ill4$WWW=OM#?%U%Jhw<|(gJr}3qQ;bH zWdg6Kz73fsKr^B`q&1(kPlxi0HV()Hv{w^ix@os-k=hEi**;r z#$1{eM@mSy_8qhbAb8N~olZs?fq#$ZLXrdF`pQc!VY60of9mlumLzOunZ^T{SigntU+=PjG$q{ubm%^lp(d zYs!3#biP44zzC8mc$_Owe{904!l{LMx3+|r!qBOUAUf{z1j0-A4%%3Fu<$8!25FJ5 zf^Bz&ksmM(ca6JJ%^IFtVjyDxBebIw0k#tvs5NVXTfOc&GGnu*ieWMmw%~yp`65Bb zV~#?Zs^Ekt>cU{Gy=X*D1co&*JHtdfV@8*bjAn1+3uR-%4S1;Oqv-wC+l({VgQXB{ssB&_ zw+YxH5gg$S^(QI`daehC=Nc#%4rBwYnEp8psu&BArK4Et^QcRp@!WJ@Au*Y0DrcO8 zwyqgz9QA|X+bS>^kO?bG&_hVjacQ`7wPKdy4Czu#`UF)qn(n*L4FqDJ%&|jOs_=f$iT!VkPrxb1=fA9rMv5Mokc4k%MMmaYmU~w_fJ!ZrOH+I$Z{aKJ z6(t!NiiBQpPtW+AmXwVlmMsNT)nPcycx(1aGbOM4E>zV{0avXAidar z9a?AkFtG;)!jy(UAS4v0nZY-=x8m$*@Qlkhn0lV_$hUr@lukQw1y0qA4~!G(nwlnY zJNFte9QWO$j%W1e8lAGRhUS#j#XCRVuz}A2$Q7e3{4vScdSv`ctlb<*xS>fRZsGg# zb41{oY$XwycJnO}ch=oq?1BJvPc)xA$ zq||r#&i~3b!^Es$O)AD3LUm09q$T~>IQee%6iztfmW?04W2kH#6YxWrFI8v7e2j%+>$9#vgVTcwa5RWTN z5NQ^L4e0M|8rcNp-Kw57jh44?H`@RUQrK4Wh+~u5ETLI6izP~jX9|YgkL8xiCV#md zJ66S`YTuf$wBF5=tGu(+ObnS_M%HZJ{5_lcYCT=_+FkljN{ap~kM~v0r3C;&E?tZI z%psjMQg~;zj?*N(Pe|^=nFXwV>i&uCv7p#&6B+4qTpvQV4MAgR!_3AO88=HHK%x^S zY@QLaGr2`f6)NzqJrtX%+vMCYT|NFdV0XK%d@>xKhr2SJMRe-^>_?PZ=*np3wrDLd z^Z9yvrnQNT$BZe@w#M*AkD_UJGPtF}LDP~xt*>`iuRCf@I^#EzZYo(OY|QJc|EHvO{EM@4aVcC6l1U2KtwfOgoeJo5$^Hb{MlUTsjG zP1W!iDRb?~N)zx%#?>9AV(2aOqSL|)176JX+4fOjw zTDX5QO}1tx@c9kIi%w=HqLrK<$0#&#BTRn9>jjZD<#)YG<>u@RHcGq!wa z&`U#?cO6wRgbD4X&jGhH9gbCHY@LV#Wt?$aKMyQIgWIt5LGmQ4I<(?i)PAg}7I$xs z$;cNcdLt0aY9t?$>A_<0*T@#D?JUvzt#?xO>3sz@BWap_nH?it!%Gz|s~Xl#R!~Ss zbdY~9fMquSTK{ui={fbezvyZ~#^DA4fGpy^vHWNJck|mY|9DH zeK|(SGd+?nbmYQ>7ZtsQ+dmDUZ|~UuV9|lp$fNfzIj`J7xzX%bG!#ac@g9+gC^Pb( zhgfD{ZRsLt)A{#d=NcHktqHKAmOl|>in})jQ`x zm;s%qlTNZ0OgKK`@+@Rb;X%n*t*GXYdJ^GUs8Uroa|Nfa$4#C;CWjr7u=rxo?-kUo z7b||la)qOYQ)B4_9u7Jqbvm~`?beCjp^{>sc6vwHL%-p?GME1H*%a(}M7$!ga^c|~ zj!hV2;Se4TqU?VJ&!2l$1vaRVO*j2sf7CtOj8_T7?`B)0>6LwXvoO9na&S}%Yw1>Z zBxfwZar&H5IPU&iOruaNs$&#ql5do=B+RcM3CU7<=UAS3UMY}-xbAuCe#kCZSH=D@ zg(SsW3(s`?V&O}ju?LH6n10PG&qmzLk%JHXzklTH_gE8~Q6vz0n5uHZ7JkwHhiBCOos<2yari%f0P5i1cL@HuB>!BJ z{~wQo{`2_!A9s9U#^lyNx8?u1ZTX*#C;fLG!2ajf{&Q>p3E2NzRN)E>^e2n;I=b&K g_~HO!tM;=14.21.3" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -985,6 +1428,48 @@ "dev": true, "license": "MIT" }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -1550,6 +2035,25 @@ "node": ">=14.0.0" } }, + "node_modules/tsx": { + "version": "4.23.12", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", + "integrity": "sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/typescript": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", diff --git a/package.json b/package.json index f36075b..df6248c 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,8 @@ }, "scripts": { "app": "node apps/core-servicing/server.ts", + "discover": "tsx src/cli.ts discover", + "replay": "tsx src/cli.ts replay", "lint": "biome check .", "typecheck": "tsc --noEmit", "test": "vitest run", @@ -21,6 +23,7 @@ "devDependencies": { "@biomejs/biome": "2.5.8", "@types/node": "26.2.0", + "tsx": "4.23.12", "typescript": "7.0.2", "vitest": "4.1.10" } diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..33c07e4 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,164 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { createServer } from "../apps/core-servicing/server"; +import { discover } from "./discover/loop"; +import { NvidiaNimModel } from "./discover/nim"; +import { lookupMemberScript, ScriptedModel } from "./discover/scripted"; +import { replay } from "./replay/engine"; +import { parseCapability } from "./schema/capability"; +import { PlaywrightWebSurface } from "./surface/playwright-web"; + +const CANONICAL = "http://127.0.0.1:4173/"; + +await loadEnv(); + +const command = process.argv[2]; +if (command === "discover") { + await runDiscover(); +} else if (command === "replay") { + await runReplay(); +} else { + process.stderr.write("usage: node src/cli.ts discover|replay\n"); + process.exitCode = 1; +} + +async function runDiscover() { + const goal = + flag("goal") ?? "Look up the member savings balance and return it."; + const out = flag("out") ?? "evidence/lookup-member-savings.json"; + const params = parseParams(); + const modelName = flag("model") ?? "nim"; + const model = + modelName === "scripted" + ? new ScriptedModel(lookupMemberScript(params.memberId ?? "10001")) + : nimModel(); + + const { baseUrl, close } = await startApp(); + const surface = await PlaywrightWebSurface.launch(); + try { + const result = await discover({ + goal, + entryPoint: `${baseUrl}/`, + artifactEntryPoint: CANONICAL, + surface, + model, + params, + evidenceDir: dirname(out), + }); + await mkdir(dirname(out), { recursive: true }); + await writeFile( + join(dirname(out), "discovery.json"), + `${JSON.stringify(result.log, null, 2)}\n`, + ); + if (result.status !== "success") { + process.stderr.write(`${result.status}: ${result.reason}\n`); + process.exitCode = 1; + return; + } + await writeFile(out, `${JSON.stringify(result.capability, null, 2)}\n`); + process.stdout.write(`wrote ${out}\n`); + } finally { + await surface.close(); + await close(); + } +} + +async function runReplay() { + const path = flag("capability") ?? "evidence/lookup-member-savings.json"; + const out = flag("out"); + const params = parseParams(); + const capability = parseCapability(JSON.parse(await readFile(path, "utf8"))); + const { baseUrl, close } = await startApp(); + const surface = await PlaywrightWebSurface.launch(); + try { + const result = await replay(capability, { + surface, + entryPoint: `${baseUrl}/`, + params, + evidenceDir: out ? dirname(out) : undefined, + }); + const json = `${JSON.stringify(result, null, 2)}\n`; + if (out) { + await mkdir(dirname(out), { recursive: true }); + await writeFile(out, json); + process.stdout.write(`wrote ${out}\n`); + } else { + process.stdout.write(json); + } + if (result.status === "failed") process.exitCode = 1; + } finally { + await surface.close(); + await close(); + } +} + +function nimModel() { + const key = process.env.NVIDIA_API_KEY; + if (!key) { + throw new Error( + "NVIDIA_API_KEY is required for discover (or pass --model scripted)", + ); + } + return new NvidiaNimModel(key); +} + +function flag(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + if (index < 0) return undefined; + return process.argv[index + 1]; +} + +function parseParams(): Record { + const params: Record = {}; + for (let i = 0; i < process.argv.length; i += 1) { + if (process.argv[i] !== "--param") continue; + const raw = process.argv[i + 1]; + if (!raw) continue; + const eq = raw.indexOf("="); + if (eq < 0) continue; + params[raw.slice(0, eq)] = raw.slice(eq + 1); + } + if (!params.memberId) params.memberId = "10001"; + return params; +} + +async function startApp() { + const server = createServer(); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("expected a TCP address"); + } + return { + baseUrl: `http://127.0.0.1:${address.port}`, + close: () => + new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }), + }; +} + +async function loadEnv() { + try { + const text = await readFile(".env", "utf8"); + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eq = trimmed.indexOf("="); + if (eq < 0) continue; + const key = trimmed.slice(0, eq); + let value = trimmed.slice(eq + 1); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + if (process.env[key] === undefined) process.env[key] = value; + } + } catch { + // no .env is fine + } +} diff --git a/src/discover/compile.ts b/src/discover/compile.ts new file mode 100644 index 0000000..3ef4f03 --- /dev/null +++ b/src/discover/compile.ts @@ -0,0 +1,192 @@ +import { + type Capability, + parseCapability, + SCHEMA_VERSION, + type Step, +} from "../schema/capability"; +import type { Decision } from "./decision"; +import { clickRisk, roleTarget, slug, tableTarget } from "./targets"; + +export type RecordedAct = + | { action: "navigate" } + | { + action: "fill"; + role: string; + name: string; + value: string; + param?: string; + } + | { action: "click"; role: string; name: string } + | { + action: "extract"; + rowText: string; + columnHeader: string; + output: string; + outputType?: "string" | "number" | "boolean" | "money"; + }; + +export function compileCapability(input: { + entryPoint: string; + appFamily: string; + params: Record; + acts: RecordedAct[]; + done: Extract; +}): Capability { + const paramNames = new Set(); + const outputs = new Map< + string, + { type: "string" | "number" | "boolean" | "money"; description: string } + >(); + const steps: Step[] = []; + const lastClickIndexByName = new Map(); + const seenCells = new Set(); + + for (const act of input.acts) { + if (act.action === "navigate") { + steps.push({ + id: "open-app", + action: "navigate", + url: { kind: "entry" }, + risk: "read", + }); + continue; + } + if (act.action === "fill") { + const param = + act.param ?? + Object.entries(input.params).find( + ([, value]) => value === act.value, + )?.[0]; + if (param) paramNames.add(param); + steps.push({ + id: `fill-${slug(act.name)}`, + action: "fill", + target: roleTarget(act.role, act.name), + value: param + ? { kind: "param", name: param } + : { kind: "literal", value: act.value }, + risk: "reversible", + }); + continue; + } + if (act.action === "click") { + lastClickIndexByName.set(act.name, steps.length); + steps.push({ + id: `click-${slug(act.name)}`, + action: "click", + target: roleTarget(act.role, act.name), + risk: clickRisk(act.name), + }); + continue; + } + const cell = `${act.rowText}\0${act.columnHeader}`; + if (seenCells.has(cell)) continue; + seenCells.add(cell); + const output = outputName(act); + outputs.set(output, { + type: act.outputType ?? (act.rowText === "Savings" ? "money" : "string"), + description: `Value of ${act.rowText} / ${act.columnHeader}`, + }); + steps.push({ + id: `extract-${slug(output)}`, + action: "extract", + target: tableTarget(act.rowText, act.columnHeader), + output, + risk: "read", + }); + } + + const outcomes = + input.done.businessOutcomes && input.done.businessOutcomes.length > 0 + ? input.done.businessOutcomes + : lastClickIndexByName.has("Search") + ? [{ matchText: "Member not found", code: "member_not_found" }] + : []; + for (const outcome of outcomes) { + const searchIndex = + lastClickIndexByName.get("Search") ?? + [...lastClickIndexByName.values()].at(-1); + if (searchIndex === undefined) continue; + const step = steps[searchIndex]; + if (step?.action !== "click") continue; + const existing = step.on ?? []; + steps[searchIndex] = { + ...step, + on: [ + ...existing, + { + match: { kind: "text", value: outcome.matchText }, + // biome-ignore lint/suspicious/noThenProperty: handler verb in the artifact, not a thenable + then: { type: "business_outcome", code: outcome.code }, + }, + ], + }; + } + + const parameters = [...paramNames].map((name) => ({ + name, + type: "string" as const, + sensitivity: /id|ssn|account/i.test(name) + ? ("identifier" as const) + : ("none" as const), + description: `Invocation parameter ${name}`, + })); + + const name = scrubParams(input.done.name, input.params); + const description = scrubParams(input.done.description, input.params); + const checkpointValue = scrubParams(input.done.checkpointValue, input.params); + const moneyCheckpoint = /^\$[\d,.]+$/.test(checkpointValue); + const checkpoint = moneyCheckpoint + ? { kind: "heading" as const, value: "Member detail" } + : { + kind: input.done.checkpointKind, + value: checkpointValue, + }; + + return parseCapability({ + schemaVersion: SCHEMA_VERSION, + id: slug(name), + name, + description, + revision: 1, + app: { + family: input.appFamily, + surface: "web", + entryPoint: input.entryPoint, + }, + parameters, + outputs: [...outputs.entries()].map(([name, field]) => ({ + name, + type: field.type, + sensitivity: field.type === "money" ? "financial" : "none", + description: field.description, + })), + steps, + success: { + checkpoint, + outputs: [...outputs.keys()], + }, + }); +} + +function outputName(act: Extract): string { + if ( + /^[A-Za-z_][A-Za-z0-9_]*$/.test(act.output) && + act.output !== "string" && + act.output !== "number" && + act.output !== "boolean" + ) { + return act.output; + } + if (act.rowText.toLowerCase() === "savings") return "savingsBalance"; + return slug(`${act.rowText}-${act.columnHeader}`).replaceAll("-", "_"); +} + +function scrubParams(text: string, params: Record): string { + let out = text; + for (const [name, value] of Object.entries(params)) { + if (!value) continue; + out = out.split(value).join(`[${name}]`); + } + return out; +} diff --git a/src/discover/decision.ts b/src/discover/decision.ts new file mode 100644 index 0000000..41331cf --- /dev/null +++ b/src/discover/decision.ts @@ -0,0 +1,62 @@ +import { z } from "zod"; +import type { Control, Observation } from "../surface/surface"; + +export const DecisionSchema = z.discriminatedUnion("type", [ + z.strictObject({ + type: z.literal("fill"), + role: z.string().min(1), + name: z.string().min(1), + value: z.string(), + param: z.string().min(1).optional(), + reason: z.string().min(1), + }), + z.strictObject({ + type: z.literal("click"), + role: z.string().min(1), + name: z.string().min(1), + reason: z.string().min(1), + }), + z.strictObject({ + type: z.literal("extract"), + rowText: z.string().min(1), + columnHeader: z.string().min(1), + output: z.string().min(1), + outputType: z.enum(["string", "number", "boolean", "money"]).optional(), + reason: z.string().min(1), + }), + z.strictObject({ + type: z.literal("done"), + name: z.string().min(1), + description: z.string().min(1), + checkpointKind: z.enum(["text", "url", "heading", "role_name"]), + checkpointValue: z.string().min(1), + businessOutcomes: z + .array( + z.strictObject({ + matchText: z.string().min(1), + code: z.string().min(1), + }), + ) + .optional(), + reason: z.string().min(1), + }), + z.strictObject({ + type: z.literal("give_up"), + reason: z.string().min(1), + }), +]); +export type Decision = z.infer; + +export type ModelInput = { + goal: string; + observation: Observation; + inventory: Control[]; + history: string[]; + paramNames: string[]; + params: Record; +}; + +export type Model = { + readonly name: string; + decide(input: ModelInput): Promise; +}; diff --git a/src/discover/loop.ts b/src/discover/loop.ts new file mode 100644 index 0000000..bbb357d --- /dev/null +++ b/src/discover/loop.ts @@ -0,0 +1,323 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import type { Handoff } from "../handoff/types"; +import { authorize } from "../policy/authorize"; +import { redactText } from "../policy/redact"; +import { DEFAULT_POLICY, type Policy } from "../policy/schema"; +import type { Capability, Parameter, Step } from "../schema/capability"; +import { RunSession } from "../session/session"; +import { LocatorError, type Surface } from "../surface/surface"; +import { compileCapability, type RecordedAct } from "./compile"; +import type { Decision, Model } from "./decision"; +import { clickRisk, roleTarget, tableTarget } from "./targets"; + +export type DiscoveryLogEvent = { + type: "acted" | "decided" | "escalated" | "human"; + detail: string; +}; + +export type DiscoveryResult = + | { + status: "success"; + capability: Capability; + events: DiscoveryLogEvent[]; + log: DiscoveryLog; + } + | { + status: "escalated" | "failed"; + reason: string; + events: DiscoveryLogEvent[]; + log: DiscoveryLog; + }; + +export type DiscoveryLog = { + goal: string; + model: string; + events: DiscoveryLogEvent[]; +}; + +export type DiscoverOptions = { + goal: string; + entryPoint: string; + surface: Surface; + model: Model; + params?: Record; + policy?: Policy; + maxSteps?: number; + appFamily?: string; + evidenceDir?: string; + artifactEntryPoint?: string; + handoff?: Handoff; + session?: RunSession; +}; + +export async function discover( + options: DiscoverOptions, +): Promise { + const params = options.params ?? {}; + const policy = options.policy ?? DEFAULT_POLICY; + const maxSteps = options.maxSteps ?? 12; + const appFamily = options.appFamily ?? "core-servicing"; + const session = options.session ?? new RunSession(options.surface); + const surface = session.surface; + const events: DiscoveryLogEvent[] = []; + const acts: RecordedAct[] = []; + const history: string[] = []; + const paramFields: Parameter[] = Object.keys(params).map((name) => ({ + name, + type: "string", + sensitivity: "identifier", + description: name, + })); + const scrub = (text: string) => redactText(text, params, paramFields); + + const log = (): DiscoveryLog => ({ + goal: options.goal, + model: options.model.name, + events, + }); + + const navStep: Step = { + id: "open-app", + action: "navigate", + url: { kind: "entry" }, + risk: "read", + }; + const denied = authorize(navStep, options.entryPoint, policy); + if (denied) { + return { + status: denied.kind === "escalate" ? "escalated" : "failed", + reason: denied.reason, + events, + log: log(), + }; + } + await surface.goto(options.entryPoint); + acts.push({ action: "navigate" }); + history.push("navigate entry"); + events.push({ type: "acted", detail: "navigate entry" }); + + const intervene = async (reason: string, step: Step): Promise => { + events.push({ type: "escalated", detail: reason }); + if (!options.handoff) return false; + session.cede(); + let screenshotPath: string | undefined; + if (options.evidenceDir) { + await mkdir(options.evidenceDir, { recursive: true }); + screenshotPath = join(options.evidenceDir, "discovery-hitl.png"); + await surface.screenshot(screenshotPath); + } + const outcome = await options.handoff.intervene(session, { + capabilityId: "discovery", + step, + reason, + observation: await surface.observe(), + screenshotPath, + }); + for (const action of outcome.actions) { + events.push({ type: "human", detail: action.detail }); + } + session.resume(); + return outcome.resume !== "abort"; + }; + + for (let n = 0; n < maxSteps; n += 1) { + const observation = await surface.observe(); + const inventory = await surface.inventory(); + const decision = await options.model.decide({ + goal: options.goal, + observation: { + ...observation, + text: scrub(observation.text), + }, + inventory: inventory.map((control) => + control.value ? { ...control, value: scrub(control.value) } : control, + ), + history, + paramNames: Object.keys(params), + params, + }); + events.push({ + type: "decided", + detail: scrub(describeDecision(decision)), + }); + + if ( + decision.type === "fill" && + acts.some((act) => act.action === "fill" && act.name === decision.name) + ) { + history.push( + `ignored repeat fill of ${decision.name}; pick a different action`, + ); + continue; + } + + if (decision.type === "done") { + const capability = compileCapability({ + entryPoint: options.artifactEntryPoint ?? options.entryPoint, + appFamily, + params, + acts, + done: decision, + }); + if (options.evidenceDir) { + await mkdir(options.evidenceDir, { recursive: true }); + await surface.screenshot(join(options.evidenceDir, "discovery.png")); + } + return { status: "success", capability, events, log: log() }; + } + + if (decision.type === "give_up") { + const continued = await intervene(decision.reason, navStep); + if (!continued) { + return { + status: "escalated", + reason: decision.reason, + events, + log: log(), + }; + } + continue; + } + + try { + await act(decision, { surface, policy, params }); + } catch (error) { + if (error instanceof LocatorError) { + const continued = await intervene( + `locator miss: ${error.expected}`, + navStep, + ); + if (!continued) { + return { + status: "failed", + reason: scrub(`${error.expected}; ${error.observed}`), + events, + log: log(), + }; + } + continue; + } + if (error instanceof PolicyError) { + return { + status: error.kind, + reason: error.message, + events, + log: log(), + }; + } + throw error; + } + + acts.push(toAct(decision)); + history.push(scrub(describeDecision(decision))); + events.push({ + type: "acted", + detail: scrub(describeDecision(decision)), + }); + } + + return { + status: "escalated", + reason: `stopped after ${maxSteps} steps`, + events, + log: log(), + }; +} + +class PolicyError extends Error { + constructor( + readonly kind: "escalated" | "failed", + message: string, + ) { + super(message); + this.name = "PolicyError"; + } +} + +async function act( + decision: Exclude, + ctx: { surface: Surface; policy: Policy; params: Record }, +) { + if (decision.type === "fill") { + const step: Step = { + id: `fill-${decision.name}`, + action: "fill", + target: roleTarget(decision.role, decision.name), + value: { kind: "literal", value: decision.value }, + risk: "reversible", + }; + denyOrThrow(authorize(step, undefined, ctx.policy)); + await ctx.surface.fill(step.target, decision.value); + return; + } + if (decision.type === "click") { + const step: Step = { + id: `click-${decision.name}`, + action: "click", + target: roleTarget(decision.role, decision.name), + risk: clickRisk(decision.name), + }; + denyOrThrow(authorize(step, undefined, ctx.policy)); + await ctx.surface.click(step.target); + return; + } + const step: Step = { + id: `extract-${decision.output}`, + action: "extract", + target: tableTarget(decision.rowText, decision.columnHeader), + output: decision.output, + risk: "read", + }; + denyOrThrow(authorize(step, undefined, ctx.policy)); + await ctx.surface.extract(step.target); +} + +function denyOrThrow(denial: ReturnType): void { + if (!denial) return; + throw new PolicyError( + denial.kind === "escalate" ? "escalated" : "failed", + denial.reason, + ); +} + +function toAct( + decision: Exclude, +): RecordedAct { + if (decision.type === "fill") { + return { + action: "fill", + role: decision.role, + name: decision.name, + value: decision.value, + param: decision.param, + }; + } + if (decision.type === "click") { + return { action: "click", role: decision.role, name: decision.name }; + } + return { + action: "extract", + rowText: decision.rowText, + columnHeader: decision.columnHeader, + output: decision.output, + outputType: decision.outputType, + }; +} + +function describeDecision(decision: Decision): string { + switch (decision.type) { + case "fill": + return decision.param + ? `fill ${decision.name} from param.${decision.param}` + : `fill ${decision.name}`; + case "click": + return `click ${decision.name}`; + case "extract": + return `extract ${decision.output}`; + case "done": + return `done: ${decision.reason}`; + case "give_up": + return `give_up: ${decision.reason}`; + } +} diff --git a/src/discover/nim.ts b/src/discover/nim.ts new file mode 100644 index 0000000..842d12c --- /dev/null +++ b/src/discover/nim.ts @@ -0,0 +1,182 @@ +import { + type Decision, + DecisionSchema, + type Model, + type ModelInput, +} from "./decision"; +import { SYSTEM_PROMPT, userPrompt } from "./prompt"; + +export const DEFAULT_NIM_BASE = "https://integrate.api.nvidia.com/v1"; +export const DEFAULT_NIM_MODEL = "nvidia/nemotron-3.5-lightning-30b-a3b"; + +const TOOLS = [ + tool( + "fill", + "Type into a labeled control.", + { + role: { type: "string" }, + name: { type: "string" }, + value: { type: "string" }, + param: { type: "string" }, + reason: { type: "string" }, + }, + ["role", "name", "value", "reason"], + ), + tool( + "click", + "Activate a control by role and accessible name.", + { + role: { type: "string" }, + name: { type: "string" }, + reason: { type: "string" }, + }, + ["role", "name", "reason"], + ), + tool( + "extract", + "Read a table cell by row text and column header.", + { + rowText: { type: "string" }, + columnHeader: { type: "string" }, + output: { type: "string" }, + outputType: { + type: "string", + enum: ["string", "number", "boolean", "money"], + }, + reason: { type: "string" }, + }, + ["rowText", "columnHeader", "output", "reason"], + ), + tool( + "done", + "Goal is met. Emit the capability title and success checkpoint.", + { + name: { type: "string" }, + description: { type: "string" }, + checkpointKind: { + type: "string", + enum: ["text", "url", "heading", "role_name"], + }, + checkpointValue: { type: "string" }, + businessOutcomes: { + type: "array", + items: { + type: "object", + properties: { + matchText: { type: "string" }, + code: { type: "string" }, + }, + required: ["matchText", "code"], + }, + }, + reason: { type: "string" }, + }, + ["name", "description", "checkpointKind", "checkpointValue", "reason"], + ), + tool( + "give_up", + "No safe next action.", + { + reason: { type: "string" }, + }, + ["reason"], + ), +]; + +export class NvidiaNimModel implements Model { + readonly name: string; + + constructor( + private readonly apiKey: string, + model = process.env.NVIDIA_MODEL ?? DEFAULT_NIM_MODEL, + private readonly baseUrl = process.env.NVIDIA_BASE_URL ?? DEFAULT_NIM_BASE, + ) { + this.name = model; + } + + async decide(input: ModelInput): Promise { + const response = await fetch( + `${this.baseUrl.replace(/\/$/, "")}/chat/completions`, + { + method: "POST", + headers: { + authorization: `Bearer ${this.apiKey}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + model: this.name, + temperature: 0, + max_tokens: 1024, + messages: [ + { role: "system", content: SYSTEM_PROMPT }, + { role: "user", content: userPrompt(input) }, + ], + tools: TOOLS, + chat_template_kwargs: { enable_thinking: false }, + }), + signal: AbortSignal.timeout(90_000), + }, + ); + if (!response.ok) { + const body = await response.text(); + throw new Error(`nim ${response.status}: ${body.slice(0, 240)}`); + } + return parseChatDecision(await response.json()); + } +} + +export function parseChatDecision(data: unknown): Decision { + const message = ( + data as { + choices?: Array<{ + message?: { + content?: string | null; + tool_calls?: Array<{ + function?: { name?: string; arguments?: unknown }; + }>; + }; + }>; + } + ).choices?.[0]?.message; + const call = message?.tool_calls?.[0]?.function; + if (call?.name) { + return DecisionSchema.parse({ + type: call.name, + ...asObject(call.arguments), + }); + } + const raw = message?.content; + if (!raw) throw new Error("nim returned an empty decision"); + return DecisionSchema.parse(JSON.parse(stripFence(raw))); +} + +function asObject(raw: unknown): Record { + if (raw && typeof raw === "object" && !Array.isArray(raw)) { + return raw as Record; + } + if (typeof raw === "string") + return JSON.parse(raw) as Record; + return {}; +} + +function stripFence(raw: string): string { + const trimmed = raw.trim(); + const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/); + return fenced?.[1] ?? trimmed; +} + +function tool( + name: string, + description: string, + properties: Record, + required: string[], +) { + return { + type: "function" as const, + function: { + name, + description, + parameters: { type: "object", properties, required }, + }, + }; +} diff --git a/src/discover/prompt.ts b/src/discover/prompt.ts new file mode 100644 index 0000000..bf451c6 --- /dev/null +++ b/src/discover/prompt.ts @@ -0,0 +1,33 @@ +import type { ModelInput } from "./decision"; + +export const SYSTEM_PROMPT = `You drive a legacy UI one action at a time to accomplish a goal. +You see an accessibility inventory (role + accessible name) and page text. Prefer those names. Never invent CSS or test IDs. +Call exactly one tool: +- fill — type into a control that has no value yet. Set param to the invocation parameter name when the value must not be hardcoded. Never fill the same control twice. +- click — activate a control. After Member ID is filled, click Search. Search is allowed. Do not transfer, delete, or confirm. +- extract — required before done when the goal asks for a displayed value. Read a table cell by row text and column header (Savings / Balance). +- done — only after extracts succeed. Use checkpointKind heading and the page h1 (e.g. Member detail). Never use a dollar amount as the checkpoint. If the UI can show "Member not found", include businessOutcomes [{ matchText: "Member not found", code: "member_not_found" }]. +- give_up — no safe next action. +Do not return prose. Do not invent locators that are not in the inventory (table extract is the exception). +Do not put invocation param values (member numbers, balances) into done.name or done.description. +Page text often omits input values. Trust the inventory value= field and Already done. Do not repeat those actions.`; + +export function userPrompt(input: ModelInput): string { + return [ + `Goal: ${input.goal}`, + `Invocation params: ${JSON.stringify(input.params)}`, + `Param names (parameterize these in the artifact): ${input.paramNames.join(", ") || "(none)"}`, + `URL: ${input.observation.url}`, + input.observation.dialog ? `Dialog: ${input.observation.dialog}` : "", + `Controls:\n${input.inventory.map(formatControl).join("\n") || "(none)"}`, + `Page text:\n${input.observation.text.slice(0, 4000)}`, + `Already done:\n${input.history.join("\n") || "(none)"}`, + ] + .filter(Boolean) + .join("\n\n"); +} + +function formatControl(control: ModelInput["inventory"][number]): string { + const value = control.value ? ` value=${JSON.stringify(control.value)}` : ""; + return `- ${control.role} "${control.name}"${value}`; +} diff --git a/src/discover/scripted.ts b/src/discover/scripted.ts new file mode 100644 index 0000000..250dba5 --- /dev/null +++ b/src/discover/scripted.ts @@ -0,0 +1,57 @@ +import type { Decision, Model, ModelInput } from "./decision"; + +/** Test double: returns a fixed decision sequence. Not used for evidence. */ +export class ScriptedModel implements Model { + readonly name = "scripted"; + private index = 0; + + constructor(private readonly decisions: Decision[]) {} + + async decide(_input: ModelInput): Promise { + const next = this.decisions[this.index]; + if (!next) { + return { type: "give_up", reason: "scripted model exhausted" }; + } + this.index += 1; + return next; + } +} + +export function lookupMemberScript(memberId: string): Decision[] { + return [ + { + type: "fill", + role: "textbox", + name: "Member ID", + value: memberId, + param: "memberId", + reason: "enter the member number from params", + }, + { + type: "click", + role: "button", + name: "Search", + reason: "submit the inquiry", + }, + { + type: "extract", + rowText: "Savings", + columnHeader: "Balance", + output: "savingsBalance", + outputType: "money", + reason: "read savings from the account table", + }, + { + type: "done", + name: "Look up member savings balance", + description: + "Search a member by ID and read the savings balance from the member detail table.", + checkpointKind: "heading", + checkpointValue: "Member detail", + businessOutcomes: [ + { matchText: "Member not found", code: "member_not_found" }, + ], + reason: "goal met", + }, + ]; +} diff --git a/src/discover/targets.ts b/src/discover/targets.ts new file mode 100644 index 0000000..96d84c0 --- /dev/null +++ b/src/discover/targets.ts @@ -0,0 +1,32 @@ +import type { LocatorSet, Risk } from "../schema/capability"; + +export function roleTarget(role: string, name: string): LocatorSet { + const candidates: LocatorSet["candidates"] = [ + { strategy: "role_name", role, name }, + ]; + if (role === "textbox") { + candidates.push({ strategy: "label", label: name }); + } + return { candidates }; +} + +export function tableTarget(rowText: string, columnHeader: string): LocatorSet { + return { + candidates: [{ strategy: "table_cell", rowText, columnHeader }], + }; +} + +export function clickRisk(name: string): Risk { + return /transfer|delete|confirm|wire|post/i.test(name) + ? "irreversible" + : "read"; +} + +export function slug(value: string): string { + return ( + value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, "") || "step" + ); +} diff --git a/src/surface/playwright-web.ts b/src/surface/playwright-web.ts index 2708b8f..b3efc60 100644 --- a/src/surface/playwright-web.ts +++ b/src/surface/playwright-web.ts @@ -6,6 +6,7 @@ import { } from "playwright"; import type { Checkpoint, Locator, LocatorSet } from "../schema/capability"; import { + type Control, describeTarget, LocatorError, type Observation, @@ -68,6 +69,36 @@ export class PlaywrightWebSurface implements Surface { }; } + async inventory(): Promise { + const roles = [ + "textbox", + "button", + "link", + "heading", + "combobox", + "checkbox", + ] as const; + const controls: Control[] = []; + for (const role of roles) { + const loc = this.page.getByRole(role); + const count = await loc.count(); + for (let i = 0; i < count; i += 1) { + const name = await accessibleName(loc.nth(i)); + if (!name) continue; + const control: Control = { role, name }; + if (role === "textbox" || role === "combobox") { + const value = await loc + .nth(i) + .inputValue() + .catch(() => ""); + if (value) control.value = value; + } + controls.push(control); + } + } + return controls; + } + async checkpointMet(checkpoint: Checkpoint) { switch (checkpoint.kind) { case "text": @@ -167,3 +198,32 @@ export class PlaywrightWebSurface implements Surface { function asRole(role: string): Parameters[0] { return role as Parameters[0]; } + +async function accessibleName(locator: PwLocator): Promise { + return locator.evaluate((el) => { + const node = el as { + getAttribute: (name: string) => string | null; + id: string; + type?: string; + value?: string; + textContent: string | null; + ownerDocument: { + querySelector: ( + selector: string, + ) => { textContent: string | null } | null; + }; + }; + const labelled = node.getAttribute("aria-label"); + if (labelled?.trim()) return labelled.trim(); + if (node.type === "submit" || node.type === "button") { + return (node.value ?? "").trim(); + } + if (node.id) { + const label = node.ownerDocument.querySelector( + `label[for="${node.id.replaceAll('"', '\\"')}"]`, + ); + if (label?.textContent?.trim()) return label.textContent.trim(); + } + return (node.textContent ?? "").replace(/\s+/g, " ").trim(); + }); +} diff --git a/src/surface/surface.ts b/src/surface/surface.ts index fb6462e..4fd44b5 100644 --- a/src/surface/surface.ts +++ b/src/surface/surface.ts @@ -6,6 +6,14 @@ export type Observation = { dialog?: string; }; +/** Accessible control the model can name. Prefer this over CSS. */ +export type Control = { + role: string; + name: string; + /** Current value for textboxes; omitted when empty. Page text often hides this. */ + value?: string; +}; + export type Surface = { goto(url: string): Promise; click(target: LocatorSet): Promise; @@ -14,6 +22,7 @@ export type Surface = { extract(target: LocatorSet): Promise; dismiss(target: LocatorSet): Promise; observe(): Promise; + inventory(): Promise; checkpointMet(checkpoint: Checkpoint): Promise; screenshot(path: string): Promise; close(): Promise; diff --git a/tests/discover/compile.test.ts b/tests/discover/compile.test.ts new file mode 100644 index 0000000..adffcc8 --- /dev/null +++ b/tests/discover/compile.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; +import { compileCapability } from "../../src/discover/compile"; + +describe("compileCapability", () => { + it("parameterizes filled values and omits them from the artifact", () => { + const capability = compileCapability({ + entryPoint: "http://127.0.0.1:4173/", + appFamily: "core-servicing", + params: { memberId: "10001" }, + acts: [ + { action: "navigate" }, + { + action: "fill", + role: "textbox", + name: "Member ID", + value: "10001", + }, + { action: "click", role: "button", name: "Search" }, + { + action: "extract", + rowText: "Savings", + columnHeader: "Balance", + output: "savingsBalance", + outputType: "money", + }, + ], + done: { + type: "done", + name: "Look up member savings balance", + description: "Retrieved savings balance for member 10001", + checkpointKind: "heading", + checkpointValue: "Member detail", + businessOutcomes: [ + { matchText: "Member not found", code: "member_not_found" }, + ], + reason: "goal met", + }, + }); + + const json = JSON.stringify(capability); + expect(json).not.toContain("10001"); + expect(capability.description).toContain("[memberId]"); + expect(capability.steps.some((step) => step.action === "fill")).toBe(true); + const fill = capability.steps.find((step) => step.action === "fill"); + expect(fill && "value" in fill && fill.value).toEqual({ + kind: "param", + name: "memberId", + }); + const click = capability.steps.find((step) => step.action === "click"); + expect(click?.on?.[0]?.then).toEqual({ + type: "business_outcome", + code: "member_not_found", + }); + }); + + it("normalizes bad extract names, drops duplicate cells, and rejects a money checkpoint", () => { + const capability = compileCapability({ + entryPoint: "http://127.0.0.1:4173/", + appFamily: "core-servicing", + params: { memberId: "10001" }, + acts: [ + { action: "navigate" }, + { + action: "fill", + role: "textbox", + name: "Member ID", + value: "10001", + }, + { action: "click", role: "button", name: "Search" }, + { + action: "extract", + rowText: "Savings", + columnHeader: "Balance", + output: "string", + }, + { + action: "extract", + rowText: "Savings", + columnHeader: "Balance", + output: "1240.50", + outputType: "money", + }, + ], + done: { + type: "done", + name: "Member savings balance lookup", + description: "done", + checkpointKind: "text", + checkpointValue: "$1,240.50", + reason: "goal met", + }, + }); + + expect(capability.outputs).toEqual([ + expect.objectContaining({ name: "savingsBalance", type: "money" }), + ]); + expect( + capability.steps.filter((step) => step.action === "extract"), + ).toHaveLength(1); + expect(capability.success.checkpoint).toEqual({ + kind: "heading", + value: "Member detail", + }); + expect( + capability.steps.find((step) => step.action === "click")?.on?.[0]?.then, + ).toEqual({ type: "business_outcome", code: "member_not_found" }); + }); +}); diff --git a/tests/discover/loop.test.ts b/tests/discover/loop.test.ts new file mode 100644 index 0000000..d5896da --- /dev/null +++ b/tests/discover/loop.test.ts @@ -0,0 +1,95 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { discover } from "../../src/discover/loop"; +import { lookupMemberScript, ScriptedModel } from "../../src/discover/scripted"; +import { replay } from "../../src/replay/engine"; +import { PlaywrightWebSurface } from "../../src/surface/playwright-web"; +import { startApp } from "../helpers/http-app"; + +const apps: Array<{ close: () => Promise }> = []; +const surfaces: PlaywrightWebSurface[] = []; + +afterEach(async () => { + await Promise.all(surfaces.splice(0).map((surface) => surface.close())); + await Promise.all(apps.splice(0).map((app) => app.close())); +}); + +describe("discover lookup-member-savings", () => { + it("compiles a live run into a capability that replays without a model", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + + const discovered = await discover({ + goal: "Look up the member savings balance and return it.", + entryPoint: `${app.baseUrl}/`, + artifactEntryPoint: "http://127.0.0.1:4173/", + surface, + model: new ScriptedModel(lookupMemberScript("10001")), + params: { memberId: "10001" }, + }); + + expect(discovered.status).toBe("success"); + if (discovered.status !== "success") return; + expect(JSON.stringify(discovered.capability)).not.toContain("10001"); + expect(discovered.log.model).toBe("scripted"); + + await surface.goto("about:blank"); + const replayed = await replay(discovered.capability, { + surface, + entryPoint: `${app.baseUrl}/`, + params: { memberId: "10001" }, + }); + + expect(replayed.status).toBe("success"); + if (replayed.status === "success") { + expect(replayed.outputs.savingsBalance).toBe("$1,240.50"); + } + }, 30_000); + + it("escalates when the model gives up and there is no handoff", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + + const result = await discover({ + goal: "Do something impossible", + entryPoint: `${app.baseUrl}/`, + surface, + model: new ScriptedModel([ + { type: "give_up", reason: "cannot find a safe path" }, + ]), + }); + + expect(result.status).toBe("escalated"); + if (result.status === "escalated") { + expect(result.reason).toContain("cannot find a safe path"); + } + }, 30_000); + + it("lists Member ID and Search from the accessibility tree", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + await surface.goto(`${app.baseUrl}/`); + const inventory = await surface.inventory(); + expect(inventory).toContainEqual({ role: "textbox", name: "Member ID" }); + expect(inventory).toContainEqual({ role: "button", name: "Search" }); + + await surface.fill( + { + candidates: [ + { strategy: "role_name", role: "textbox", name: "Member ID" }, + ], + }, + "10001", + ); + expect(await surface.inventory()).toContainEqual({ + role: "textbox", + name: "Member ID", + value: "10001", + }); + }, 30_000); +}); diff --git a/tests/discover/nim.test.ts b/tests/discover/nim.test.ts new file mode 100644 index 0000000..28399e6 --- /dev/null +++ b/tests/discover/nim.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { parseChatDecision } from "../../src/discover/nim"; + +describe("parseChatDecision", () => { + it("reads a tool call as the decision", () => { + const decision = parseChatDecision({ + choices: [ + { + message: { + tool_calls: [ + { + function: { + name: "click", + arguments: JSON.stringify({ + role: "button", + name: "Search", + reason: "submit the inquiry", + }), + }, + }, + ], + }, + }, + ], + }); + expect(decision).toEqual({ + type: "click", + role: "button", + name: "Search", + reason: "submit the inquiry", + }); + }); + + it("falls back to JSON content when no tool call is present", () => { + const decision = parseChatDecision({ + choices: [ + { + message: { + content: + '```json\n{"type":"give_up","reason":"no safe control"}\n```', + }, + }, + ], + }); + expect(decision.type).toBe("give_up"); + }); +}); diff --git a/tests/helpers/recording-surface.ts b/tests/helpers/recording-surface.ts index 78ae302..5b006c2 100644 --- a/tests/helpers/recording-surface.ts +++ b/tests/helpers/recording-surface.ts @@ -1,5 +1,5 @@ import type { Checkpoint, LocatorSet } from "../../src/schema/capability"; -import type { Observation, Surface } from "../../src/surface/surface"; +import type { Control, Observation, Surface } from "../../src/surface/surface"; export class RecordingSurface implements Surface { readonly gotos: string[] = []; @@ -27,6 +27,10 @@ export class RecordingSurface implements Surface { return { url: this.gotos.at(-1) ?? "", text: "" }; } + async inventory(): Promise { + return []; + } + async checkpointMet(_checkpoint: Checkpoint) { return true; } From 3cdfaad21ccbeb4efaf8f2783273e5af69e27fef Mon Sep 17 00:00:00 2001 From: vectorvoyager Date: Sat, 15 Aug 2026 13:50:37 -0500 Subject: [PATCH 2/2] Recover from session warnings on replay, hand off locator misses, and add report --- README.md | 18 ++- REPORT.md | 88 +++++++++++++ apps/core-servicing/server.ts | 17 ++- evidence/lookup-member-savings.json | 23 +++- evidence/replay-success.json | 5 + src/discover/compile.ts | 52 +++++--- src/discover/scripted.ts | 6 + src/replay/engine.ts | 119 ++++++++++++------ src/schema/capability.ts | 2 + tests/app/core-servicing.test.ts | 12 ++ tests/discover/compile.test.ts | 12 +- tests/discover/loop.test.ts | 3 + tests/fixtures/lookup-member-savings.json | 19 +++ tests/handoff/hitl.test.ts | 85 +++++++++++++ tests/replay/lookup-member.test.ts | 84 +++++++++++++ tests/replay/recover.test.ts | 144 ++++++++++++++++++++++ 16 files changed, 622 insertions(+), 67 deletions(-) create mode 100644 REPORT.md create mode 100644 tests/replay/recover.test.ts diff --git a/README.md b/README.md index 79fc98b..399eb4e 100644 --- a/README.md +++ b/README.md @@ -20,28 +20,38 @@ npm run check Add NVIDIA API key in `.env` as `NVIDIA_API_KEY` to run discovery. Create a key at [https://build.nvidia.com/models](https://build.nvidia.com/models). Replay and `npm run check` do not need a key. -Start the local target app (no API keys): +`npm run app` is only if you want to click the UI in a browser. `discover` and `replay` start their own copy of the app; you do not need this running for the demo commands. ```bash npm run app ``` -It listens on `http://127.0.0.1:4173/`. Member `10001` has a savings balance; any other id returns "Member not found". +It listens on `http://127.0.0.1:4173/`. Members `10001` (`$1,240.50`) and `10002` (`$50.00`) have savings balances; an unknown id returns "Member not found". A successful lookup shows a session warning; replay dismisses it and logs a `recovered` event. ## Demo Discovery talks to NVIDIA NIM (`nvidia/nemotron-3.5-lightning-30b-a3b`) and writes a capability under `evidence/`. Replay does not call a model. Override the model with `NVIDIA_MODEL` in `.env` (must be a chat NIM with tool calling). +Recorded run (do not re-run `discover` into these paths unless you intend to replace it): + +- `evidence/lookup-member-savings.json` — compiled capability (the contract) +- `evidence/discovery.json` — raw model log from that run +- `evidence/discovery.png` — screenshot at the end of discovery +- `evidence/replay-success.json` — replay for `10001` (`$1,240.50`, plus `recovered`) +- `evidence/replay-member-not-found.json` — replay for `99999` + ```bash npm run discover -- --goal "Look up the member savings balance" --param memberId=10001 --out evidence/lookup-member-savings.json npm run replay -- --capability evidence/lookup-member-savings.json --param memberId=10001 --out evidence/replay-success.json npm run replay -- --capability evidence/lookup-member-savings.json --param memberId=99999 --out evidence/replay-member-not-found.json ``` -Without a key, `npm run check` still exercises discovery against the live app using a scripted model. To dry-run the CLI itself: +The same capability with `--param memberId=10002` returns `$50.00` (no second discovery). Human handoff is exercised in tests; the operator UI is mocked. + +Without a key, `npm run check` still exercises discovery against the live app using a scripted model. To dry-run the CLI itself, write somewhere other than `evidence/`: ```bash -npm run discover -- --model scripted --param memberId=10001 --out evidence/lookup-member-savings.json +npm run discover -- --model scripted --param memberId=10001 --out /tmp/lookup-member-savings.json ``` `--model scripted` is a test double. Evidence meant to show a real discovery run must use the default NIM path. diff --git a/REPORT.md b/REPORT.md new file mode 100644 index 0000000..f33ad9b --- /dev/null +++ b/REPORT.md @@ -0,0 +1,88 @@ +# REPORT + +## Architecture + +Single process, TypeScript, CLI. Discovery and replay share one `Surface` (today `PlaywrightWebSurface`) and the same policy gate. The LLM is a recording engineer, not the production executor. + +``` +goal + entry URL + │ + ▼ + discover() ──observe a11y inventory──► NVIDIA NIM (one tool call / turn) + │ fill | click | extract | done + ▼ + compile() ──parameterize, scrub, normalize──► capability JSON + │ + ▼ + replay() ──no model──► success | business_outcome | escalated | failed + │ + └── stuck / irreversible ──► HITL on the same RunSession +``` + +Zod validates the capability at the boundary. Playwright maps intents onto a live page. Policy runs **before** every act. `npm run check` (typecheck, lint, test) does not need an API key; CI uses a `ScriptedModel`. A genuine discovery run used `nvidia/nemotron-3.5-lightning-30b-a3b` via OpenAI-compatible `/chat/completions` (`fetch`, no extra SDK). + +Rejected: queues, a capability catalog service, Python/Pydantic (one language for schema, replay, and the browser adapter). Rejected saving the model transcript as the artifact. + +The target is a local server-rendered core-servicing mock (`apps/core-servicing`, port 4173): nested layout tables, ids `f1`/`cmd1`, no test IDs. Search → member detail. Known members: `10001` savings `$1,240.50`, `10002` savings `$50.00`. An unknown id (`99999` in evidence) returns “Member not found”. A successful lookup also shows a session-expiring dialog; the balance table stays hidden until Continue. Rejected a public cart demo (ToS, no controlled not-found). Rejected framesets in v1 (replay would need frame targeting before the first replay worked). + +## Artifact schema + +The file a calling agent invokes is `schemaVersion` 1.0.0 JSON (`src/schema/capability.ts`). `schemaVersion` is the format; `revision` is this flow. `strictObject` so a reviewed file cannot silently drop fields. + +A capability is a **contract**: typed `parameters` and `outputs` with sensitivity; ordered `steps`; a success `checkpoint`. Values are refs (`param` / `literal` / `entry`), never a discovery-time member number. Fill is `{ kind: "param", name: "memberId" }`, not `"10001"`. + +Steps are a discriminated union on `action`. Invalid combos (fill without a value, extract without an output) fail at parse time. Each step has `risk`: `read` | `reversible` | `irreversible`. + +Locators are an ordered candidate list, biased to a surface with no clean DOM: `role_name` → `label` → `nearby_text` → `table_cell` → `structural` → `css`. CSS is last-resort. Test IDs are not a strategy. `table_cell` is row text plus column header, not a CSS nth-child. + +`on` clauses on steps declare what replay should do when the page matches: `business_outcome`, `recover`, `escalate`, or `fail`. Replay does not invent that taxonomy at runtime. + +The compiler (`src/discover/compile.ts`) is the seam between a noisy model run and that contract. A live Nemotron session extracted the same Savings cell under names `string` and `1240.50`, used `$1,240.50` as a checkpoint, and put a member id in the description. Compile scrubs param values, dedupes table cells, renames invalid outputs to `savingsBalance`, defaults `member_not_found` on Search, and replaces a money-amount checkpoint with heading `Member detail`. `evidence/discovery.json` is the raw log; `evidence/lookup-member-savings.json` is what production keeps. + +## Determinism & error handling + +Replay walks the artifact with no LLM. For each step it tries locators in rank order, then evaluates `on` against the observed page. Fill events log `param.memberId`, not the value. + +Nested layout tables made a naive `table tr` extract return the whole page; the adapter targets **direct rows only** (`:scope > tbody > tr`). That is a replay bug, not “the UI drifted.” + +Terminal statuses: `success` | `business_outcome` | `escalated` | `failed`. + +- **`business_outcome`** is a legitimate caller result (`member_not_found`), not a crash. Evidence: `evidence/replay-member-not-found.json`. +- **`recovered`** is an **event**, not a status. After Search, the mock shows a session-expiring dialog and hides the balance table. Replay matches `on` → `recover` → dismisses Continue, logs `recovered`, then extracts. If dismiss works, the caller still gets success (or a business outcome). Evidence: `evidence/replay-success.json` includes that event. +- **`failed`** always includes `stepId`, `expected`, `observed`, and optionally a screenshot. Unknown UI (locator miss with no handoff, failed checkpoint) stops here. Replay does not call the model to improvise. +- **`escalated`** is control transfer (irreversible step, locator miss with a handoff that aborts, or an `on` → escalate), not “we are stuck internally.” + +Happy-path evidence: `evidence/replay-success.json` (`savingsBalance: "$1,240.50"`). The same artifact with `memberId=10002` returns `$50.00` — parameterization, not a second recording. Locator miss without a handoff: broken Search name → `failed` on `click-search` with expected/observed. With a handoff, the same miss cedes the live page; the operator clicks Search and replay still extracts `$1,240.50`. + +## Heterogeneity & multi-tenant + +**Surface.** The artifact stores intents (`click`, `fill`, `extract`) and locator *candidates*, not Playwright selectors. `Surface` is how we perceive and act (goto, click, fill, extract, a11y `inventory`, observe). Today one implementation: Chromium. A desktop adapter would implement the same type and resolve `role_name` against the OS accessibility tree; `table_cell` would mean “row/column in the focused grid.” CSS candidates would no-op or be ignored. The replay engine would not change. + +Framesets, extra document contexts, and screenshot+coordinates are the next surface problems, not schema problems. Coordinates were rejected as the default locator: they fail under DPI and layout shift; role+name matches how a human finds the control. + +**Tenants.** The base artifact has `app.family` + `surface`, **no `tenantId`**. Hundreds of credit unions on the same vendor product should share one capability. Drift belongs in an overlay later: per-tenant locator inserts (an extra `label` candidate), copy variants for `on` match strings, entry URL. Detection: replay `failed` with the same `stepId` and a changed `observed` across tenants is a locator/copy drift signal, not a reason to re-record the whole flow. Re-record when the *intent* changed (a new confirmation step), not when a button’s accessible name gained a suffix. + +## Escalation & handoff + +Stuck means: policy refuses an irreversible act, an `on` handler says escalate, a locator miss when a handoff is present, or discovery `give_up` / max steps. Locator-miss with no handoff stays `failed`. With a handoff, the same session is ceded **once per step**; a second miss on that step fails instead of looping. + +`RunSession` owns the live `Surface` and `owner: automation | human`. Handoff is `cede` → `intervene` → `resume` on that object. A new browser is not a handoff. + +The intervention carries capability/goal, current step, page text, optional screenshot, and why it stopped. The operator acts on `session.surface`. Resume is `skip_step` (human finished the blocked step — correct after an irreversible click), `retry_step`, or `abort`. Human work is `events[].type === "human"`. + +Operator UI is mocked: `ScriptedOperator` in tests (Playwright runs for irreversible Search and for a broken Search locator; the operator clicks the real Search on the **same** page and replay extracts `$1,240.50`); `PromptOperator` is headed browser plus stdin `skip|retry|abort`. A co-browsing console would subscribe to the same `Handoff` interface. + +## Safety + +Policy (`src/policy`) runs before the surface moves: allowlisted action types, host, path prefix. Default allowlist is `127.0.0.1` / `localhost`. Irreversible defaults to **escalate and do not click**; `onIrreversible: "block"` is the stricter option. Discovery is bound by the same gate. + +Artifacts and logs must not persist secrets or raw identifiers. Param values with sensitivity `identifier` / `financial` / `secret` / `full_pii` are stripped from failure `observed` (`[memberId]`, not `10001`). Success **outputs** stay intact — that is the capability contract (the caller asked for the balance). Screenshots are not pixel-redacted (cut). Discovery prompts redact inventory values the same way so the model sees “filled” without a raw member number in our logs. + +## Cuts + +- **Operator UI** — mock; control transfer is real. +- **Desktop / framesets / multi-tenant overlays** — designed, not built. +- **Screenshot redaction, capability catalog API, codegen, N-run flakiness, bounded LLM fallback on replay** — stretch; skipped until this thread was evidenced. +- **CI model** — `ScriptedModel`. The NIM path is real; CI does not call it. + +Next, if this were production: a tenant overlay for copy/locator drift. diff --git a/apps/core-servicing/server.ts b/apps/core-servicing/server.ts index 659b18e..5c4f444 100644 --- a/apps/core-servicing/server.ts +++ b/apps/core-servicing/server.ts @@ -119,6 +119,13 @@ function memberPage(rawId: string) { ) .join(""); return shell(` +
+ + + +
Your session is about to expire.
+
+ +`); } const isDirectRun = diff --git a/evidence/lookup-member-savings.json b/evidence/lookup-member-savings.json index ea5ea59..ae5e103 100644 --- a/evidence/lookup-member-savings.json +++ b/evidence/lookup-member-savings.json @@ -69,6 +69,25 @@ "type": "business_outcome", "code": "member_not_found" } + }, + { + "match": { + "kind": "dialog", + "value": "Your session is about to expire." + }, + "then": { + "type": "recover", + "action": "dismiss", + "target": { + "candidates": [ + { + "strategy": "role_name", + "role": "button", + "name": "Continue" + } + ] + } + } } ], "action": "click", @@ -103,8 +122,6 @@ "kind": "heading", "value": "Member detail" }, - "outputs": [ - "savingsBalance" - ] + "outputs": ["savingsBalance"] } } diff --git a/evidence/replay-success.json b/evidence/replay-success.json index ad9321c..b7bc1aa 100644 --- a/evidence/replay-success.json +++ b/evidence/replay-success.json @@ -21,6 +21,11 @@ "stepId": "click-search", "detail": "click button \"Search\"" }, + { + "type": "recovered", + "stepId": "click-search", + "detail": "dismiss (1/1)" + }, { "type": "acted", "stepId": "extract-savingsbalance", diff --git a/src/discover/compile.ts b/src/discover/compile.ts index 3ef4f03..d223cbc 100644 --- a/src/discover/compile.ts +++ b/src/discover/compile.ts @@ -70,6 +70,7 @@ export function compileCapability(input: { continue; } if (act.action === "click") { + if (act.name === "Continue") continue; lastClickIndexByName.set(act.name, steps.length); steps.push({ id: `click-${slug(act.name)}`, @@ -96,31 +97,44 @@ export function compileCapability(input: { }); } + const searchIndex = lastClickIndexByName.get("Search"); + const outcomeIndex = searchIndex ?? [...lastClickIndexByName.values()].at(-1); const outcomes = input.done.businessOutcomes && input.done.businessOutcomes.length > 0 ? input.done.businessOutcomes - : lastClickIndexByName.has("Search") + : searchIndex !== undefined ? [{ matchText: "Member not found", code: "member_not_found" }] : []; - for (const outcome of outcomes) { - const searchIndex = - lastClickIndexByName.get("Search") ?? - [...lastClickIndexByName.values()].at(-1); - if (searchIndex === undefined) continue; - const step = steps[searchIndex]; - if (step?.action !== "click") continue; - const existing = step.on ?? []; - steps[searchIndex] = { - ...step, - on: [ - ...existing, - { - match: { kind: "text", value: outcome.matchText }, + if (outcomeIndex !== undefined) { + const click = steps[outcomeIndex]; + if (click?.action === "click") { + const handlers = [ + ...(click.on ?? []), + ...outcomes.map((outcome) => ({ + match: { kind: "text" as const, value: outcome.matchText }, // biome-ignore lint/suspicious/noThenProperty: handler verb in the artifact, not a thenable - then: { type: "business_outcome", code: outcome.code }, - }, - ], - }; + then: { + type: "business_outcome" as const, + code: outcome.code, + }, + })), + ]; + if (searchIndex !== undefined) { + handlers.push({ + match: { + kind: "dialog" as const, + value: "Your session is about to expire.", + }, + // biome-ignore lint/suspicious/noThenProperty: handler verb in the artifact, not a thenable + then: { + type: "recover" as const, + action: "dismiss" as const, + target: roleTarget("button", "Continue"), + }, + }); + } + steps[outcomeIndex] = { ...click, on: handlers }; + } } const parameters = [...paramNames].map((name) => ({ diff --git a/src/discover/scripted.ts b/src/discover/scripted.ts index 250dba5..c10e71e 100644 --- a/src/discover/scripted.ts +++ b/src/discover/scripted.ts @@ -33,6 +33,12 @@ export function lookupMemberScript(memberId: string): Decision[] { name: "Search", reason: "submit the inquiry", }, + { + type: "click", + role: "button", + name: "Continue", + reason: "dismiss the session warning", + }, { type: "extract", rowText: "Savings", diff --git a/src/replay/engine.ts b/src/replay/engine.ts index bb1c2c2..104bcdb 100644 --- a/src/replay/engine.ts +++ b/src/replay/engine.ts @@ -129,7 +129,38 @@ export async function replay( return outcome.resume; }; + const settle = async ( + step: Step, + ): Promise => { + const observation = await surface.observe(); + const handler = matchingHandler(step.on, observation); + if (!handler) return "continue"; + + const handled = await applyHandler(handler, step, { + surface, + events, + }); + if (handled === "continue") return "continue"; + if (handled.status === "business_outcome") { + return parseReplayResult({ + status: "business_outcome", + capabilityId: capability.id, + revision: capability.revision, + code: handled.code, + events, + }); + } + if (handled.status === "escalated") { + const next = await escalate(step, handled.reason); + if (next === "skip_step") return "continue"; + if (next === "retry_step") return "retry"; + return next; + } + return fail(step, handled.expected, handled.observed); + }; + let index = 0; + const locatorHandoff = new Set(); while (index < capability.steps.length) { const step = capability.steps[index]; if (!step) break; @@ -153,8 +184,13 @@ export async function replay( if (denial?.kind === "escalate") { const next = await escalate(step, denial.reason); if (next === "skip_step") { - index += 1; - continue; + const settled = await settle(step); + if (settled === "continue") { + index += 1; + continue; + } + if (settled === "retry") continue; + return settled; } if (next === "retry_step") continue; return next; @@ -178,7 +214,22 @@ export async function replay( stepId: step.id, detail: "locator miss after retries", }); - return fail(step, error.expected, error.observed); + if (!options.handoff || locatorHandoff.has(step.id)) { + return fail(step, error.expected, error.observed); + } + locatorHandoff.add(step.id); + const next = await escalate(step, `locator miss: ${error.expected}`); + if (next === "skip_step") { + const settled = await settle(step); + if (settled === "continue") { + index += 1; + continue; + } + if (settled === "retry") continue; + return settled; + } + if (next === "retry_step") continue; + return next; } if ( error instanceof Error && @@ -195,40 +246,13 @@ export async function replay( detail: describeStep(step), }); - const observation = await surface.observe(); - const handler = matchingHandler(step.on, observation); - if (!handler) { - index += 1; - continue; - } - - const handled = await applyHandler(handler, step, { - surface, - events, - }); - if (handled === "continue") { + const settled = await settle(step); + if (settled === "continue") { index += 1; continue; } - if (handled.status === "business_outcome") { - return parseReplayResult({ - status: "business_outcome", - capabilityId: capability.id, - revision: capability.revision, - code: handled.code, - events, - }); - } - if (handled.status === "escalated") { - const next = await escalate(step, handled.reason); - if (next === "skip_step") { - index += 1; - continue; - } - if (next === "retry_step") continue; - return next; - } - return fail(step, handled.expected, handled.observed); + if (settled === "retry") continue; + return settled; } const checkpoint = capability.success.checkpoint; @@ -365,8 +389,28 @@ async function applyHandler( case "recover": { const attempts = then.maxAttempts ?? 1; for (let i = 0; i < attempts; i += 1) { - if (then.action === "dismiss" && "target" in step) { - await ctx.surface.dismiss(step.target); + if (then.action === "dismiss") { + const target = + then.target ?? ("target" in step ? step.target : undefined); + if (!target) { + return { + status: "failed", + expected: "recover dismiss target", + observed: (await ctx.surface.observe()).text.slice(0, 240), + }; + } + try { + await ctx.surface.dismiss(target); + } catch (error) { + if (error instanceof LocatorError) { + return { + status: "failed", + expected: error.expected, + observed: error.observed, + }; + } + throw error; + } } else if (then.action === "wait") { await new Promise((resolve) => setTimeout(resolve, 250)); } @@ -377,6 +421,9 @@ async function applyHandler( }); const again = matchingHandler(step.on, await ctx.surface.observe()); if (!again) return "continue"; + if (again.then.type !== "recover") { + return applyHandler(again, step, ctx); + } } return { status: "failed", diff --git a/src/schema/capability.ts b/src/schema/capability.ts index d3be14f..2759736 100644 --- a/src/schema/capability.ts +++ b/src/schema/capability.ts @@ -100,6 +100,8 @@ export const ExceptionThenSchema = z.discriminatedUnion("type", [ type: z.literal("recover"), action: z.enum(["dismiss", "retry", "wait"]), maxAttempts: z.number().int().positive().optional(), + /** Control to dismiss; defaults to the step target when omitted. */ + target: LocatorSetSchema.optional(), }), z.strictObject({ type: z.literal("escalate"), diff --git a/tests/app/core-servicing.test.ts b/tests/app/core-servicing.test.ts index 54465e8..e6bafb6 100644 --- a/tests/app/core-servicing.test.ts +++ b/tests/app/core-servicing.test.ts @@ -49,6 +49,18 @@ describe("core-servicing app", () => { expect(html).toContain("AccountBalance"); expect(html).toContain("Savings"); expect(html).toContain("$1,240.50"); + expect(html).toContain('role="dialog"'); + expect(html).toContain("Your session is about to expire."); + expect(html).toContain('id="cmd2" value="Continue"'); + expect(html).toContain('id="member-body" hidden'); + expect(html).not.toContain("Member not found"); + }); + + it("shows a different savings balance for member 10002", async () => { + const html = await page(await start(), "/member?id=10002"); + + expect(html).toContain("

Member detail

"); + expect(html).toContain("$50.00"); expect(html).not.toContain("Member not found"); }); diff --git a/tests/discover/compile.test.ts b/tests/discover/compile.test.ts index adffcc8..b1c0009 100644 --- a/tests/discover/compile.test.ts +++ b/tests/discover/compile.test.ts @@ -16,6 +16,7 @@ describe("compileCapability", () => { value: "10001", }, { action: "click", role: "button", name: "Search" }, + { action: "click", role: "button", name: "Continue" }, { action: "extract", rowText: "Savings", @@ -47,10 +48,13 @@ describe("compileCapability", () => { name: "memberId", }); const click = capability.steps.find((step) => step.action === "click"); - expect(click?.on?.[0]?.then).toEqual({ - type: "business_outcome", - code: "member_not_found", - }); + expect(click?.on?.map((handler) => handler.then.type)).toEqual([ + "business_outcome", + "recover", + ]); + expect(capability.steps.some((step) => step.id.includes("continue"))).toBe( + false, + ); }); it("normalizes bad extract names, drops duplicate cells, and rejects a money checkpoint", () => { diff --git a/tests/discover/loop.test.ts b/tests/discover/loop.test.ts index d5896da..c8070e5 100644 --- a/tests/discover/loop.test.ts +++ b/tests/discover/loop.test.ts @@ -45,6 +45,9 @@ describe("discover lookup-member-savings", () => { if (replayed.status === "success") { expect(replayed.outputs.savingsBalance).toBe("$1,240.50"); } + expect(replayed.events.some((event) => event.type === "recovered")).toBe( + true, + ); }, 30_000); it("escalates when the model gives up and there is no handoff", async () => { diff --git a/tests/fixtures/lookup-member-savings.json b/tests/fixtures/lookup-member-savings.json index 6fa3991..f1b2f50 100644 --- a/tests/fixtures/lookup-member-savings.json +++ b/tests/fixtures/lookup-member-savings.json @@ -57,6 +57,25 @@ { "match": { "kind": "text", "value": "Member not found" }, "then": { "type": "business_outcome", "code": "member_not_found" } + }, + { + "match": { + "kind": "dialog", + "value": "Your session is about to expire." + }, + "then": { + "type": "recover", + "action": "dismiss", + "target": { + "candidates": [ + { + "strategy": "role_name", + "role": "button", + "name": "Continue" + } + ] + } + } } ] }, diff --git a/tests/handoff/hitl.test.ts b/tests/handoff/hitl.test.ts index 68e94cd..88a3438 100644 --- a/tests/handoff/hitl.test.ts +++ b/tests/handoff/hitl.test.ts @@ -3,6 +3,7 @@ import { ScriptedOperator } from "../../src/handoff/scripted"; import { replay } from "../../src/replay/engine"; import { parseCapability } from "../../src/schema/capability"; import { RunSession } from "../../src/session/session"; +import { LocatorError } from "../../src/surface/surface"; import { RecordingSurface } from "../helpers/recording-surface"; const clickTarget = { @@ -145,4 +146,88 @@ describe("HITL handoff", () => { } expect(result.events.some((event) => event.type === "human")).toBe(true); }); + + it("cedes the same surface on a locator miss when a handoff is present", async () => { + class MissingClickSurface extends RecordingSurface { + async click() { + throw new LocatorError('button "Go"', "(empty page)"); + } + } + const surface = new MissingClickSurface(); + const session = new RunSession(surface); + let reason = ""; + + const result = await replay( + capability({ + steps: [ + { + id: "submit", + action: "click", + target: clickTarget, + risk: "read", + }, + ], + }), + { + surface, + session, + handoff: new ScriptedOperator(async (owned, request) => { + expect(owned.owner).toBe("human"); + reason = request.reason; + return { + actions: [{ type: "other", detail: "clicked the real control" }], + resume: "skip_step", + }; + }), + }, + ); + + expect(reason).toContain("locator miss"); + expect(result.status).toBe("success"); + expect(session.owner).toBe("automation"); + expect(result.events.some((event) => event.type === "human")).toBe(true); + }); + + it("fails a locator miss if retry still cannot find the control", async () => { + class MissingClickSurface extends RecordingSurface { + async click() { + throw new LocatorError('button "Go"', "(empty page)"); + } + } + const surface = new MissingClickSurface(); + const session = new RunSession(surface); + let interventions = 0; + + const result = await replay( + capability({ + steps: [ + { + id: "submit", + action: "click", + target: clickTarget, + risk: "read", + }, + ], + }), + { + surface, + session, + handoff: new ScriptedOperator(async () => { + interventions += 1; + return { + actions: [{ type: "other", detail: "asked to retry locators" }], + resume: "retry_step", + }; + }), + }, + ); + + expect(interventions).toBe(1); + expect(result.status).toBe("failed"); + expect(session.owner).toBe("automation"); + if (result.status === "failed") { + expect(result.stepId).toBe("submit"); + expect(result.expected).toContain("Go"); + } + }); }); diff --git a/tests/replay/lookup-member.test.ts b/tests/replay/lookup-member.test.ts index d25ac61..c7d4234 100644 --- a/tests/replay/lookup-member.test.ts +++ b/tests/replay/lookup-member.test.ts @@ -46,6 +46,30 @@ describe("replay lookup-member-savings", () => { if (result.status === "success") { expect(result.outputs.savingsBalance).toBe("$1,240.50"); } + expect( + result.events.some( + (event) => + event.type === "recovered" && event.stepId === "click-search", + ), + ).toBe(true); + }, 30_000); + + it("replays the same capability for member 10002 without rediscovering", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + + const result = await replay(fixture, { + surface, + entryPoint: `${app.baseUrl}/`, + params: { memberId: "10002" }, + }); + + expect(result.status).toBe("success"); + if (result.status === "success") { + expect(result.outputs.savingsBalance).toBe("$50.00"); + } }, 30_000); it("returns member_not_found as a business outcome, not a failure", async () => { @@ -64,6 +88,9 @@ describe("replay lookup-member-savings", () => { if (result.status === "business_outcome") { expect(result.code).toBe("member_not_found"); } + expect(result.events.some((event) => event.type === "recovered")).toBe( + false, + ); }, 30_000); it("fails with step, expected, and observed when a locator misses", async () => { @@ -148,4 +175,61 @@ describe("replay lookup-member-savings", () => { } expect(result.events.some((event) => event.type === "human")).toBe(true); }, 30_000); + + it("hands the live page to a human when a locator misses, then resumes", async () => { + const app = await startApp(); + apps.push(app); + const surface = await PlaywrightWebSurface.launch(); + surfaces.push(surface); + const session = new RunSession(surface); + + const broken = parseCapability({ + ...fixture, + steps: fixture.steps.map((step) => + step.id === "click-search" + ? { + ...step, + target: { + candidates: [ + { + strategy: "role_name", + role: "button", + name: "Definitely not a button", + }, + ], + }, + } + : step, + ), + }); + + const result = await replay(broken, { + surface, + session, + entryPoint: `${app.baseUrl}/`, + params: { memberId: "10001" }, + handoff: new ScriptedOperator(async (owned, request) => { + expect(owned.owner).toBe("human"); + expect(request.reason).toContain("locator miss"); + await owned.surface.click({ + candidates: [ + { strategy: "role_name", role: "button", name: "Search" }, + ], + }); + return { + actions: [ + { type: "click", detail: "clicked Search after locator miss" }, + ], + resume: "skip_step", + }; + }), + }); + + expect(result.status).toBe("success"); + expect(session.owner).toBe("automation"); + if (result.status === "success") { + expect(result.outputs.savingsBalance).toBe("$1,240.50"); + } + expect(result.events.some((event) => event.type === "human")).toBe(true); + }, 30_000); }); diff --git a/tests/replay/recover.test.ts b/tests/replay/recover.test.ts new file mode 100644 index 0000000..2dd9b4f --- /dev/null +++ b/tests/replay/recover.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "vitest"; +import { replay } from "../../src/replay/engine"; +import { parseCapability } from "../../src/schema/capability"; +import { LocatorError } from "../../src/surface/surface"; +import { RecordingSurface } from "../helpers/recording-surface"; + +class SessionWarningSurface extends RecordingSurface { + private warning = false; + + async click() { + this.clicks += 1; + this.warning = true; + } + + async dismiss() { + this.warning = false; + } + + async observe() { + if (this.warning) { + return { + url: this.gotos.at(-1) ?? "", + text: "Your session is about to expire.", + dialog: "Your session is about to expire.", + }; + } + return { url: this.gotos.at(-1) ?? "", text: "Member detail" }; + } + + async extract() { + if (this.warning) { + throw new LocatorError("Savings / Balance", "session warning"); + } + return "$1,240.50"; + } + + async checkpointMet() { + return !this.warning; + } +} + +const recoverHandler = { + match: { kind: "dialog" as const, value: "Your session is about to expire." }, + // biome-ignore lint/suspicious/noThenProperty: handler verb in the artifact, not a thenable + then: { + type: "recover" as const, + action: "dismiss" as const, + target: { + candidates: [ + { strategy: "role_name" as const, role: "button", name: "Continue" }, + ], + }, + }, +}; + +function capability(recover: boolean) { + return parseCapability({ + schemaVersion: "1.0.0", + id: "recover-probe", + name: "Recover probe", + description: "Search then extract after a session warning", + revision: 1, + app: { + family: "core-servicing", + surface: "web", + entryPoint: "http://127.0.0.1:4173/", + }, + parameters: [], + outputs: [ + { + name: "savingsBalance", + type: "money", + sensitivity: "financial", + description: "Balance after the warning is dismissed", + }, + ], + steps: [ + { + id: "open-app", + action: "navigate", + url: { kind: "entry" }, + risk: "read", + }, + { + id: "submit", + action: "click", + target: { + candidates: [ + { strategy: "role_name", role: "button", name: "Search" }, + ], + }, + risk: "read", + on: recover ? [recoverHandler] : undefined, + }, + { + id: "extract-balance", + action: "extract", + target: { + candidates: [ + { + strategy: "table_cell", + rowText: "Savings", + columnHeader: "Balance", + }, + ], + }, + output: "savingsBalance", + risk: "read", + }, + ], + success: { + checkpoint: { kind: "text", value: "Member detail" }, + outputs: ["savingsBalance"], + }, + }); +} + +describe("recoverable interstitial", () => { + it("dismisses the warning, logs recovered, and still returns the output", async () => { + const surface = new SessionWarningSurface(); + const result = await replay(capability(true), { surface }); + + expect(result.status).toBe("success"); + if (result.status === "success") { + expect(result.outputs.savingsBalance).toBe("$1,240.50"); + } + expect(result.events.some((event) => event.type === "recovered")).toBe( + true, + ); + }); + + it("fails extract when the warning is not recovered", async () => { + const surface = new SessionWarningSurface(); + const result = await replay(capability(false), { surface }); + + expect(result.status).toBe("failed"); + if (result.status === "failed") { + expect(result.stepId).toBe("extract-balance"); + } + expect(result.events.some((event) => event.type === "recovered")).toBe( + false, + ); + }); +});