From acd62f8848efae620679fb5d5a9c443624994a97 Mon Sep 17 00:00:00 2001 From: kevin Heifner Date: Wed, 30 Sep 2026 13:39:44 -0500 Subject: [PATCH 1/3] epoch: record one delivery result per batch operator per duty epoch sysio.epoch::advance walked the expiring group once per active outpost and sent sysio.opreg::recorddel and termcheck for every (outpost, member) pair, so each duty epoch wrote one dellog row per outpost. termcheck's thresholds and the min_terminate_window_ms bound count duty epochs, so the extra rows scaled the miss ladder with the outpost count. With the two launch outposts a full outage tripped the consecutive-miss rail after three duty epochs instead of six. An operator that kept missing one outpost interleaved a hit with every miss, so the consecutive rail never fired for it; only the percent rail could, and with enough outposts its miss rate falls under that threshold too. advance now combines each member's result across the active outposts before recording it. A member counts as delivered only if it delivered to every active outpost, and gets exactly one recorddel and one termcheck per duty epoch. The non-canonical-delivery slash collection stays per outpost and is still flushed ahead of the delivery records. An epoch with no active outpost records nothing, as before. The sysio.opreg docs now describe that cadence and termcheck's configured thresholds. sysio.epoch.wasm is rebuilt; its ABI is unchanged. --- .../include/sysio.epoch/sysio.epoch.hpp | 6 +- contracts/sysio.epoch/src/sysio.epoch.cpp | 63 +-- contracts/sysio.epoch/sysio.epoch.wasm | Bin 79839 -> 80240 bytes .../include/sysio.opreg/sysio.opreg.hpp | 45 ++- contracts/tests/sysio.msgch_chain_tests.cpp | 371 ++++++++++++------ contracts/tests/sysio.opreg_tests.cpp | 2 +- 6 files changed, 330 insertions(+), 157 deletions(-) diff --git a/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp b/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp index 4f39e90c27..2aeb372e24 100644 --- a/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp +++ b/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp @@ -145,9 +145,9 @@ namespace sysio { static constexpr uint32_t MAX_BATCH_OP_GROUPS = 255; /// Upper bound on `operators_per_epoch` (the size of one schedule group). - /// Every member of the expiring group costs two inline actions per active - /// outpost in `advance` (opreg::recorddel + opreg::termcheck) plus an inline - /// transfer in sysio.system::payepoch, so group size directly scales the + /// Every member of the expiring group costs two inline actions in + /// `advance` (opreg::recorddel + opreg::termcheck) plus a payclaims + /// credit in sysio.system::payepoch, so group size directly scales the /// epoch-boundary transaction; an absurd value (which the product equality /// alone cannot reject -- UINT32_MAX groups of one is internally consistent) /// would abort `advance` on its vector reserves and halt epoch advancement diff --git a/contracts/sysio.epoch/src/sysio.epoch.cpp b/contracts/sysio.epoch/src/sysio.epoch.cpp index e8711f0935..b5653ef829 100644 --- a/contracts/sysio.epoch/src/sysio.epoch.cpp +++ b/contracts/sysio.epoch/src/sysio.epoch.cpp @@ -484,9 +484,13 @@ void epoch::advance() { // For each (outpost × member of the expiring group): // - scan `msgch::envelopes` (`byoutepoch` index) for any row matching // (chain_code, current_epoch_index, batch_op_name == member) - // - collect the delivery result and every non-canonical deliverer - // - slash all non-canonical deliverers before delivery accounting can - // terminate them, then record the result and run `termcheck` + // - count the outposts the member delivered to and collect every + // non-canonical deliverer + // Then slash all non-canonical deliverers before delivery accounting can + // terminate them, and record ONE result per member: delivered only if it + // delivered to every active outpost. `sysio.opreg`'s miss thresholds and + // window bound count duty epochs, so a record per outpost would scale the + // miss ladder with the outpost count. // // The outpost set is sourced via a cross-contract read of // `sysio.chains::chains` (no local mirror) filtered to @@ -513,18 +517,23 @@ void epoch::advance() { // operator, which would abort advance and stall the chain). std::vector to_slash; - /// A delivery result retained until non-canonical offenders have been - /// slashed. `recorddel` remains an audit record even for a newly + /// One member's delivery result for the expiring epoch, accumulated across + /// the active outposts and retained until non-canonical offenders have + /// been slashed. `recorddel` remains an audit record even for a newly /// slashed operator, while `termcheck` safely skips non-ACTIVE rows. struct delivery_observation { name member; - bool did_deliver; + uint32_t outposts_delivered = 0; ///< Active outposts this member delivered to. }; std::vector observations; + observations.reserve(expiring_group.size()); + for (const auto& member : expiring_group) observations.push_back({member}); + uint32_t active_outposts = 0; sysio::chains::chains_t chains_tbl(CHAINS_ACCOUNT); for (auto op_it = chains_tbl.begin(); op_it != chains_tbl.end(); ++op_it) { if (!is_active_outpost(*op_it)) continue; + ++active_outposts; const uint64_t chain_code = op_it->code.value; const uint128_t composite = @@ -563,17 +572,17 @@ void epoch::advance() { } } - for (const auto& member : expiring_group) { + for (auto& observation : observations) { bool did_deliver = false; checksum256 member_checksum{}; for (size_t i = 0; i < delivered.size(); ++i) { - if (delivered[i] == member) { + if (delivered[i] == observation.member) { did_deliver = true; member_checksum = delivered_checksums[i]; break; } } - observations.push_back({member, did_deliver}); + if (did_deliver) ++observation.outposts_delivered; // Single slash path (dispute-vote design, per-operator outcome table): a delivered // NON-canonical checksum is a fault -> slash. Silence (no delivery) is never slashed; it @@ -581,9 +590,9 @@ void epoch::advance() { if (did_deliver && have_winner && member_checksum != winner) { bool queued = false; for (const auto& s : to_slash) { - if (s == member) { queued = true; break; } + if (s == observation.member) { queued = true; break; } } - if (!queued) to_slash.push_back(member); + if (!queued) to_slash.push_back(observation.member); } } } @@ -615,19 +624,25 @@ void epoch::advance() { // Preserve the delivery history after slashing. A non-canonical operator is already // SLASHED here, so opreg::termcheck returns without converting the punitive outcome into a // termination/remit. Other group members retain their normal delivery accounting. - for (const auto& observation : observations) { - action( - permission_level{get_self(), "owner"_n}, - OPREG_ACCOUNT, - opreg_actions::RECORD_DELIVERY, - std::make_tuple(observation.member, state.current_epoch_index, observation.did_deliver) - ).send(); - action( - permission_level{get_self(), "owner"_n}, - OPREG_ACCOUNT, - opreg_actions::TERMINATION_CHECK, - std::make_tuple(observation.member) - ).send(); + // + // One record per member for this duty epoch; missing any active outpost makes it a miss. With + // no active outpost the group had nothing to deliver, so nothing is recorded. + if (active_outposts > 0) { + for (const auto& observation : observations) { + const bool delivered_everywhere = observation.outposts_delivered == active_outposts; + action( + permission_level{get_self(), "owner"_n}, + OPREG_ACCOUNT, + opreg_actions::RECORD_DELIVERY, + std::make_tuple(observation.member, state.current_epoch_index, delivered_everywhere) + ).send(); + action( + permission_level{get_self(), "owner"_n}, + OPREG_ACCOUNT, + opreg_actions::TERMINATION_CHECK, + std::make_tuple(observation.member) + ).send(); + } } // The envelope rows read above are left in place: `sysio.msgch::deliver` diff --git a/contracts/sysio.epoch/sysio.epoch.wasm b/contracts/sysio.epoch/sysio.epoch.wasm index 3583ddc36b03401edcaecef5ff887efea340647a..17159c0bb1d2b82f46000ba885c087fe3949e683 100755 GIT binary patch delta 5757 zcmb7IdvH|M8Q*j7-MhQV?%uPT-DI-~aPAFY0t9(TvcUkk0TM%yNFx&72?1$0S_HxA zgC;Ph4r8&R$2v?$G1JG?KeXEJOshC`sC7nK(W0deR!8lOZPO`^v1UfB&a}Vp+)Kh^ zowhK!=bn4-`M&eLe!uVRyB`{3|1_R%)#H^~Ni3#mnqFC163Y~B((h&33vU$OHeSe9 z-dkX$Y={kN>Q5hJL)u^|?)5@p+Uwesk|-OwA(RVZJ!--19oX)#*7o%kk$f$rZC*- zZF$G2)eJ`$7y^#j77GaGo-y)57v_Mg?{{f_P|v&Jyc6f^@u80I1*_GDbtgpQLESM2 z9BaVg4VnSDmTT;H*$oiHp~K?AtXg7oZ;>4mLu!?6mXZ=I@`ixW>$I>P#4OaHg)!i> z!+L`j6-+?(K|Bz_M(Bl-kWgF2r_d1^aIL(1$DTZvk*my=P;T-WrM^)#7)}DV>3CVE z_XO;qa)D=sugEdFuuO*v9fKR7Juq+2E{Eu09nSW41(TuE*za(cC1o>i>ymJQRIaUK+dkW58^rspjT!%9U9C+a@Ih0R0Gz9gW9I9m7 zmbW|GUuG*qa%T-YR4S4Jo_%Ddg@cjdI6M z;r6U|oAy6E%<^_Q?jgdZ`wHBTu)Lee+Zp6)9Klkl$T!G4YUc(kL^ZDiQz{r0qSZV0-N})&}Zy#2L@V!g=?V-v#(4vX{Ofw>_=%M$AYb?6Z(ZcsQzw zv^+j*Ljt1e^bytw4|2R0mJH8679q2si*J^{zpYvRWp;o0MXyj5*p!!X@|#>gN3`l# zhmsQUw#9*Q({uFG2*w!Q>)2bPK8jmPk5R;@74!FKJa};A?CayNpE@xl=PggkSLe9$ z5A{psjrI4it&-O-k%_shnJ%}^eTZ!^ z=_>2OdEPI#F3je4&{I`!a47}XF-S&6BrmZKlL_OPNUB|rpaAgPx8ZRVIAF2OP7sVh z5EVL&z&Z;#r?u$vgN3t(L1Xu2YA6Tl7uUt@u&Mx|4$ zozjF!c%jtUfH)EpQIVj#81B>#?Mmchs8$ODUI>PgqLM&KRRavlikO;;NTYs+YCGJh z14|OfJ$e6ixm>t4@Fleu-i}R>OxlvF-|$UTIknmmct|BIme*=W0pT!Dx{GG9pxnLa zdO(aRvht}#6~QuOSCxEuQI=)pUl&c?P_1%oJPZd_UzTIlJ}_Mf+X|qL>QJs!dB8Lt zwgD_vm*CPk0W1>s2qAk`ZfWQ?0gNGJ>n|E+Yg&u^b3^5nIDBBo33;gWc4ng<41tn( zE+B)AvpB4Oh9m@APYy2`6k)ln@v}OBOs#fCcTyfy5C$MJz!dB|O;o4TU^ra%=e9Rn z!VviDtaAU2e+==vfHBljg*e3F03Xf$x`hWxI&hqzL$ao+Es06EPE2@(xG8Z3T$2}Q z@mSdNj*r%ITT_Nv@@q{!RlF^x`9@;bFtSvn-DuuPiAoMx zg9l7^3?oXEs9W79m8FKu_8{P(DH@X4%|`jl#ZAmAe6;u)7UGJ;;&NJZMI8|icJ4qj zh+Fqv5fCAF%y&gxr1*fHmfM;ICyg+hG+FjBpq3Q z-XxAn%XgYH5cNs(os1U-mb5Y!mp@&aH3_^BnOB!C9a0QyDu&g%kknptV!7IWCqeEN z=0wn}iJ%b$PX^6Um4129t|Z^{rlZ0#YEmWq8=sIwh@otJ9z$v^-Q`1_pSv^|y?r<)Rfe%^9j6&}AIsaKxics)?wV zHK#)!LlcuyCZT{QGO1cVv?3R*7Fp!ZYb(lGN{+605@`JJ%Gn&OUaP&YIn`~k6woju zUt3v@SEDO)xY})5GeZqhR=RCDBb#@DFsstKAu{rNZ9RCWQ~7+{mYae0EwDHMz;>}m zg7_3QnW_2!>eOX-0mImyD*5ZqN^n@dJ%_8^mNpTo5M({yely%)-&+3_j_d;50&G!w z;2tFscdkih7Ln6wvZH`HZOTqwQ)VUi?dM;?NcB#D!xdZY{&7Cj6R zqe7U@4T4}L2{dAK#1#LIX#mGV9p#?Abw%v;d>4hf6mXw_E=|ZzaY@MFDstV@RarAZ zaf!>Vt8$UJ@B5^DY!&(U*;PHbCOdtn7E?lHm#0u!P)F=ZjZRYjs&fi3?5)n6fp{k6 zQ(YC9Uc1^Cw7ke}0lE;R5ricX0rZ&|&@^MqSI2^gVSM$nDoY(s$kjMrU@5p-Yk{*Z zk&tU{%wB8R3GYO9X)F?{JDFS6)T8Q{$1Zs-e zL~REApY}TDlw7@r+5VFr*LLwz#XK1gYDf#2bR!RabWQ}60}#DtDS`Y5o!m7Ite9&L5(q-6cN^w zt@@x7Y!!!pf*JMu^207(ZE z%WTx&mRoWWF4Ft}Fb9W*TZZwzW_@lbp%`18Gt?b9I))6w5QV!i0G?=8;tdh+Vc462 z73m$0#&HZmZw8=AUu5yN3>ThFtsG2@v_ClK`aN{TkZTxM4ZRv$P8QUNJJlCIE1A^vII zNz1N{^>D_wHa0u>ka{gTK2E1R`PRmo4m2=cGjzxYj zo70=ab!YE2xZc;B3#V}I1oIg(acgy4B!&Txp6JU>B_1VlDZ)+S$_$}=k_H1| z!CGWuUti8h39!c+)&E1~BP!F6sh@E|z|eQL69F{#`$#kYN>7mT zq6B~q3=o}4yT1%9Z#}rfXM$<{!~{3*@FmVHCh=+}ke~0U#|{_J9#zYOwy7)HaP=qO>QCXn{hy#ZQ%IwI3SM;0M1m#6#uy#ZK@ zH3TVu0s-sAz5O>}&CuZ?17N*lmzPx!@1m^w&93RVj_&efvZI(;DW9EFQ3X$_sDh^y z!IK566g;J()CXaPYL8<8Jb4xap0HK#te)cgkf>Gi)i2jBq~D2#(Ql&Hbw9_MjDD1% zzLQ|;iy-Uxib=@&1}nUg&ofpo-P=9{)Xl$r8t0Sn25BQq&hM!U!g&hgzRP6w?!&C4 z@Z;SNF#M#@xTk|f3y1bp>#Rf`-#3Rv*(EEqVI7qE^@KsQ;L#>@Mt$$=%I>yQ$w%W13@9fP@&@gS2 zx%=(eJ?A^$>-@g&+rfY6fBINI-m0Z|X)?(<*D5MXljS_WLVJ|)6Yu3e)L*G|e*7GZ zvp%+us~_J-_o@8z8NYnP1Bi@|?d&ONfMY zt7j^@w0gFp4^%fSN{F~ftgiCog4Jp*z8c;{!zf-vS66v)Vjuq+du(N3YuZtMUfqRV zuAi{fRLeztb=Cb|LLQy)?POxLZv=d|7Yia@X})&iNiOf0QqgA!JuvUcd8S~&K`j@Q z zTu`3#GFnSG$?gUP_Xhdtl*!RCqRj5ekFR@@`A%Chf>)RHa5v%siD<)%*oE!#((>Z9 znMrHCSolb=8h%`}8!@6iN^-s{@T!InaFLdk^*w2uzY&q@8NP;Yk#rrj1elS(ste7|OK|?U1)oPJOZNLy3xYS1?9v-&tKmONe zpE-TGt}11OF+5d~Tra30Md0 zqzet|{Bh<)=16VU52jj5#0VrT&q3MpTybpE*d89k4wn4&tXt>X_6E-(m5(uB6T=K> z0`$Bp1KL2|GJE=!f<>{xkQ&nC7)uC* zdz~EZ+lL6$Ljq{>e*eWY`%*^&B5ryT%9TL;)Mh!JLp)FV1r5~_V(_hJc=l4s|YGgl7G>#nlek{qg$;~=%&6iEm> zFmt{wOnJ-v$C^!{F99@aOFSKNeiX@pu@MY{;zX-(r2CZ8R1err#I6;R6r|~qlNZck zZhqr}$t-5|2KpX9EgxT)t+k*5B=^Bt;8!q3M8tx@f+^C%w0k{Uo?R%IB?lLdYe_4u zFB#A}4Nf6E#lRq4(0b4y5U=5{>JS$6G(wZ;C4;oKWmj`KSZ!;*n_2lkG`BLAmg9m- zgRl;sSr{zrgXlSAu>JVs%W)i(h5$pp$9KrQ!YDe;EjrD*!fE=*;0>MTG;oE}5aqlQ z?2-|lqyfV>dVRpI6?(I81nyec&mDHB!cQoo)(;p(cUo|#FdC3}5J$uX6x@(Pn83c5 zRJ4bNAya$^fLECYb3UZG7+;_`f_K}J`TJVS7)~KjMBxTQURe%E+L@#jEa)CR`77 zR2ms_Usj&zXwtK}AR(8(x_CD?0&EW;Bc%v}XJI1W0&Lkm zKtHy#O3vu2KxX-%vkBKj9gYD{#lUOo^*6!{hrUt2-9mKf3L$aykg60*+yw@yISNnT zsPZ++KzLW;#jr8-xy&IEhm!;NDHfWCdUX}Uh4EF0j6}}S&wLvCEOGTx>^)7W)9h>btH1hAr)C(&5iCIet|Z!za56 zj24X|EkGBHI6+tn9zdc8(S1$j<*)jJa=@15%~dYUBMdS%&Lfs0t@9Qb+ZB#HwY+kg z>p5XfKGK))fFs<32S|K!ZD-Pk2iA2JJdhs61K4251E^?~2hzhHs2)uxsz_jeTUC|- z?Uz=NS%0^JY;Ucs#C6Kbf=!&!)|>KID=Xs4#GDSuUsg83j$>C<;@Y^Xu(EHIl|?%! zo5Glx-e4b?)0so4^fODG`!5$jRf4ow!K29Fm;4ZWi^Zx7w)sI+CD$p zDjt3sBdX!$`c-zln}E@?G|NJTU5r~G7mEG_QU~Hz!6f1~J8ZSFqsWeM6*=D&74rSn zyI4BkabuimY5N|zpxXnM+;dZBhEux}3BEWNv5^OAwE^AZ^1YkZFrKfwc?PQsvl)p2 zl__DcG{l^Ia!q!Oh4LKn8&)5jz|)eiuW5sy?CvLkpGUjv zaNgeTZi-vNw(keV*fP~~0Pkf_Q=g+Uv}#n;9T_%>GYf)l%mqOzr%`E0;^D)RaD*&K z?~H^6 zevo-qzH37%Wz3$9t)68+#43jhslLi$9o3#>Fl$1T$(J^MH^fr=t1&6RvuPZ1-H}aA zxW2z>GOqvL)KraxX(7`gK|@M}8Db2f#x_dMvIkVm;PSr$HMYQGtILEd-@gM|N!`{C|3D0cd7?)iAXA zsQ3>7dMTUALtDon$-DBKw-++O<69{cyn1_q<)qPAemxV&JGPC*vPZW4e~@3)=PHmN zgNIIS{of!TSXaXjqn;fGue#BMzXI~fvq(oMI8-`vJhOB&xSfLPU)w3DX5}W}+MO$SWXotunT6bJnhLB; zQ-PI9VC6z51y-g>RS-rv`)Lq>Rj4n(3WA26zG?49sFllwy*0DwyV3zvo2YbyFYt|r zKIl-@NuYH#086zG1K52m-?k&iSeZO==f?oPqj!zBtzobOw|*v%uC3Ez3h%zpif!GVm_W%Jp?qm``Mn2Evi14ahkieg_2kFCa+i+L f!Pky3wq725?Vox7^;+EKm%VYS1|;$6*0%ouJCH4{ diff --git a/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp b/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp index 020a4d5c02..00d745ae8f 100644 --- a/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp +++ b/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp @@ -111,14 +111,15 @@ namespace sysio { /// Minimum accepted `terminate_window_ms` for a given consecutive-miss /// threshold and epoch schedule. Delivery records accrue only on an - /// operator's DUTY epochs: `sysio.epoch::advance` runs `recorddel` for - /// the expiring group alone, and the schedule is a sliding window of - /// `batch_op_groups` groups, so a resident operator's duty interval is + /// operator's DUTY epochs, one per duty epoch: `sysio.epoch::advance` + /// runs `recorddel` once per member of the expiring group, combining its + /// result across every active outpost. The schedule is a sliding window + /// of `batch_op_groups` groups, so a resident operator's duty interval is /// `batch_op_groups` epochs. The rolling window must span the full - /// terminating run -- `consecutive_misses` duty-epoch records -- plus - /// one duty interval of boundary slack, or records age out (and are - /// pruned) before `termcheck` can observe the run, leaving the - /// consecutive rail structurally vacuous (SEC-28 residual). Operators + /// terminating run (`consecutive_misses + 1` records, `consecutive_misses` + /// duty intervals end to end) plus one duty interval of boundary slack, + /// or records age out (and are pruned) before `termcheck` can observe the + /// run, leaving the consecutive rail structurally vacuous. Operators /// benched by a surplus roster accrue no records while benched, so no /// finite window observes them; the bound governs resident operators, /// whose duty interval the sliding schedule pins at `batch_op_groups` @@ -319,18 +320,23 @@ namespace sysio { [[sysio::action]] void releaselock(name account, sysio::slug_name chain_code, sysio::slug_name token_code, uint64_t amount); - /// Record per-batch-op delivery hit/miss for the rolling 24h buffer. - /// Called inline from `sysio.epoch::advance` after each delivery cycle. - /// Each write also sweeps up to `MAX_DELLOG_PRUNE_PER_WRITE` rows that - /// have aged out of the rolling termination window, keeping the buffer - /// bounded without a dedicated crank. + /// Record a batch operator's delivery result for one duty epoch in the + /// rolling buffer. Called inline from `sysio.epoch::advance` once per + /// expiring-group member; `delivered` is true only if the member delivered + /// to every active outpost. Each write also sweeps up to + /// `MAX_DELLOG_PRUNE_PER_WRITE` rows that have aged out of the rolling + /// termination window, keeping the buffer bounded without a dedicated + /// crank. [[sysio::action]] void recorddel(name account, uint32_t epoch, bool delivered); - /// Evaluate the rolling 24h delivery buffer for an operator. If it - /// breaches the threshold (>3 consecutive misses OR >5% missed in - /// the trailing 24h), inline `terminate(...)`. Called by - /// `sysio.epoch::advance` after every `recorddel`. + /// Evaluate an operator's rolling delivery buffer and inline + /// `terminate(...)` when it holds more than + /// `terminate_max_consecutive_misses` consecutive misses, or a miss rate + /// whose whole-percent value (rounded down) exceeds + /// `terminate_max_pct_misses_24h`. Both rails count duty epochs, one row + /// each. Called by `sysio.epoch::advance` after every + /// `recorddel`. [[sysio::action]] void termcheck(name account); @@ -507,9 +513,10 @@ namespace sysio { sysio::const_mem_fun> >; - /// Per-batch-op rolling delivery buffer. One row per (operator, epoch) - /// recording whether the operator delivered on schedule. Rows older - /// than `TERMINATE_WINDOW_MS` are discarded by `prune` / on-write. + /// Per-batch-op rolling delivery buffer. One row per (operator, duty + /// epoch), `delivered` only if the operator delivered to every active + /// outpost. Rows older than `terminate_window_ms` are discarded by + /// `prune` / on-write. struct delivery_key { uint64_t log_id; uint64_t primary_key() const { return log_id; } diff --git a/contracts/tests/sysio.msgch_chain_tests.cpp b/contracts/tests/sysio.msgch_chain_tests.cpp index 6e06e2b82c..ff9c9bae17 100644 --- a/contracts/tests/sysio.msgch_chain_tests.cpp +++ b/contracts/tests/sysio.msgch_chain_tests.cpp @@ -37,6 +37,9 @@ #include +#include +#include + #include "contracts.hpp" #include "contract_test_support.hpp" @@ -57,6 +60,8 @@ constexpr uint64_t ETH_OUTPOST_ID = "ETH"_s.value; constexpr uint64_t SOL_OUTPOST_ID = "SOL"_s.value; constexpr std::string_view ETH_CHAIN_CODE = "ETH"; constexpr std::string_view SOL_CHAIN_CODE = "SOL"; +constexpr uint32_t ETH_EXTERNAL_CHAIN_ID = 31337; ///< External chain id the fixtures register ETH under. +constexpr uint32_t SOL_EXTERNAL_CHAIN_ID = 1; ///< External chain id the fixtures register SOL under. constexpr uint64_t BATCH_OPERATOR_MINIMUM_COLLATERAL = 1; constexpr uint64_t TABLE_SCAN_LIMIT = 64; constexpr uint32_t ONE_OPERATOR_PER_TIED_VERSION = 1; @@ -79,13 +84,33 @@ constexpr name SET_CONFIG = "setconfig"_n; /// sysio.opreg ABI field identifiers used by the WNS-16 fixture. namespace opreg_fields { constexpr const char* STATUS = "status"; +constexpr const char* STATUS_REASON = "status_reason"; constexpr const char* RECENT_ACTIONS = "recent_actions"; constexpr const char* ACTION = "action"; constexpr const char* ACTION_TYPE = "action_type"; constexpr const char* SUCCESS = "success"; constexpr const char* CHAIN_CODE = "chain_code"; +constexpr const char* ACCOUNT = "account"; +constexpr const char* EPOCH = "epoch"; +constexpr const char* DELIVERED = "delivered"; } // namespace opreg_fields +/// sysio.opreg tables read by delivery-log assertions. +namespace opreg_tables { +constexpr name DELIVERY_LOG = "dellog"_n; +} // namespace opreg_tables + +/// sysio.opreg ABI type names used to decode table rows. +namespace opreg_abi_types { +constexpr const char* DELIVERY_LOG_ENTRY = "delivery_log_entry"; +} // namespace opreg_abi_types + +/// Termination reason `termcheck` records when the rotation fixture's consecutive-miss rail fires. +constexpr const char* CONSECUTIVE_MISS_TERMINATION_REASON = "rolling-window: >5 consecutive misses"; + +/// Payload of the envelopes BATCHOP delivers while the rotation fixture walks its duty epochs. +constexpr std::string_view ROTATION_DELIVERY_PAYLOAD = "rotation-duty"; + /// sysio.opreg configuration ABI field identifiers used by the WNS-16 fixture. namespace opreg_config_fields { constexpr const char* MAX_AVAILABLE_PRODUCERS = "max_available_producers"; @@ -229,6 +254,28 @@ class sysio_msgch_chain_tester : public tester { static constexpr auto TIER_ONE_OWNER = "tierone"_n; static constexpr const char* EMPTY_ROA_TOTAL_SYSTEM = "75496.0000 SYS"; static constexpr uint64_t EMPTY_ROA_BYTES_PER_UNIT = 104; + static constexpr uint64_t MS_PER_SEC = 1000; ///< Milliseconds per second, for window spans. + + /// Rotation fixture schedule: single-operator groups, so a resident operator is on duty once per + /// `ROTATION_GROUPS`-epoch rotation. + static constexpr uint32_t ROTATION_GROUPS = 3; + static constexpr uint32_t ROTATION_OPERATORS_PER_GROUP = 1; ///< Operators seated in each rotation group. + + /// Consecutive-miss threshold the rotation fixture installs (the production default; see + /// `CONSECUTIVE_MISS_TERMINATION_REASON`). + static constexpr uint32_t ROTATION_MAX_CONSECUTIVE_MISSES = 5; + + /// Percent-miss threshold the rotation fixture installs: the accepted ceiling, so an anchored run + /// terminates on the consecutive rail. + static constexpr uint32_t ROTATION_MAX_PERCENT_MISSES = 99; + + /// Exact minimum window the span bound accepts for the rotation schedule: (misses + 1) duty + /// rotations. + static constexpr uint64_t ROTATION_WINDOW_MS = + (uint64_t{ROTATION_MAX_CONSECUTIVE_MISSES} + 1) * ROTATION_GROUPS * EPOCH_DURATION_SEC * MS_PER_SEC; + + /// Outposts the rotation fixture registers. + enum class rotation_outposts { eth, eth_and_sol }; /// Construct an OPP integration fixture, optionally with an activated ROA that has no /// registered node owners to exercise the msgch electorate preflight. @@ -403,8 +450,8 @@ class sysio_msgch_chain_tester : public tester { ("is_bootstrapped", op == BATCHOP ? batchop_is_bootstrapped : true))); } - register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, ETH_CHAIN_CODE, 31337); - register_chain(opp::types::ChainKind::CHAIN_KIND_SVM, SOL_CHAIN_CODE, 1); + register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, ETH_CHAIN_CODE, ETH_EXTERNAL_CHAIN_ID); + register_chain(opp::types::ChainKind::CHAIN_KIND_SVM, SOL_CHAIN_CODE, SOL_EXTERNAL_CHAIN_ID); // A non-bootstrapped batch operator starts UNKNOWN and becomes ACTIVE // only after a collateral update re-evaluates its role eligibility. @@ -603,23 +650,27 @@ class sysio_msgch_chain_tester : public tester { return fc::variant{}; } - /// Every inbound `envelopes` row in primary-key order, read straight from chainbase so the walk - /// does not depend on ids staying inside a fixed probe range. - std::vector envelope_rows() { - const auto table_id = compute_table_id(msgch_tables::ENVELOPES.to_uint64_t()); + /// Every row of `code`'s `table` in primary-key order, decoded as `abi_type`. Read straight from + /// chainbase so the walk does not depend on ids staying inside a fixed probe range. + std::vector table_rows(name code, name table, const abi_serializer& abi, const char* abi_type) { + const auto table_id = compute_table_id(table.to_uint64_t()); const auto& kv_idx = control->db().get_index(); std::vector rows; - for (auto itr = kv_idx.lower_bound(boost::make_tuple(MSGCH_ACCOUNT, table_id, std::string_view{})); - itr != kv_idx.end() && itr->code == MSGCH_ACCOUNT && itr->table_id == table_id; ++itr) { + for (auto itr = kv_idx.lower_bound(boost::make_tuple(code, table_id, std::string_view{})); + itr != kv_idx.end() && itr->code == code && itr->table_id == table_id; ++itr) { std::vector raw(itr->value.size()); if (!raw.empty()) std::memcpy(raw.data(), itr->value.data(), raw.size()); - rows.push_back(msgch_abi.binary_to_variant( - msgch_abi_types::ENVELOPE_ENTRY, raw, - abi_serializer::create_yield_function(abi_serializer_max_time))); + rows.push_back(abi.binary_to_variant( + abi_type, raw, abi_serializer::create_yield_function(abi_serializer_max_time))); } return rows; } + /// Every inbound `envelopes` row in primary-key order. + std::vector envelope_rows() { + return table_rows(MSGCH_ACCOUNT, msgch_tables::ENVELOPES, msgch_abi, msgch_abi_types::ENVELOPE_ENTRY); + } + /// `envelopes` rows recorded for `epoch_index`, across outposts and operators. uint32_t envelope_rows_for_epoch(uint32_t epoch_index) { uint32_t n = 0; @@ -837,17 +888,32 @@ class sysio_msgch_chain_tester : public tester { return n; } + /// Every `dellog` row for `account`, oldest first. + std::vector dellog_rows(name account) { + std::vector rows; + for (auto& row : table_rows(OPREG_ACCOUNT, opreg_tables::DELIVERY_LOG, opreg_abi, + opreg_abi_types::DELIVERY_LOG_ENTRY)) { + if (row[opreg_fields::ACCOUNT].as_string() == account.to_string()) rows.push_back(std::move(row)); + } + return rows; + } + + /// `account`'s `dellog` rows for `epoch_index`. + std::vector dellog_rows(name account, uint32_t epoch_index) { + std::vector rows; + for (auto& row : dellog_rows(account)) { + if (row[opreg_fields::EPOCH].as() == epoch_index) rows.push_back(std::move(row)); + } + return rows; + } + /// Count DELIVERED dellog rows still present for `account`. recorddel PRUNES (erases) rows that /// have aged out of the rolling window, so a surviving delivered row proves that record is still /// inside the window -- the direct check that the edge anchor was not pruned. - uint32_t delivered_dellog_count(name account, uint64_t scan_until = TABLE_SCAN_LIMIT) { + uint32_t delivered_dellog_count(name account) { uint32_t n = 0; - for (uint64_t id = 0; id < scan_until; ++id) { - auto data = get_row_by_account(OPREG_ACCOUNT, OPREG_ACCOUNT, "dellog"_n, name{id}); - if (data.empty()) continue; - auto row = opreg_abi.binary_to_variant("delivery_log_entry", data, - abi_serializer::create_yield_function(abi_serializer_max_time)); - if (row["account"].as_string() == account.to_string() && row["delivered"].as()) ++n; + for (const auto& row : dellog_rows(account)) { + if (row[opreg_fields::DELIVERED].as()) ++n; } return n; } @@ -944,35 +1010,48 @@ class sysio_msgch_chain_tester : public tester { ("original_message_id", std::string(64, '0'))); } - /// bootstrap() variant for a real rotation: THREE single-operator groups (so a resident op is on - /// duty once per 3-epoch rotation), the SEC-28 percent rail disabled up to its accepted ceiling - /// (99, so an anchored run terminates on the CONSECUTIVE rail), and `terminate_window_ms` set by - /// the caller (the exact span bound for this schedule). ETH outpost registered; genesis advance run. - void bootstrap_rotation(uint64_t terminate_window_ms) { + /// bootstrap() variant for a real rotation: `ROTATION_GROUPS` single-operator groups (so a resident + /// op is on duty once per rotation), the percent rail at its accepted ceiling (so an anchored run + /// terminates on the CONSECUTIVE rail), and `terminate_window_ms` set by the caller (the exact span + /// bound for this schedule). Registers `outposts` with BATCHOP bonded on each, runs the genesis + /// advance, and returns the registered outposts' chain codes. + std::vector bootstrap_rotation(uint64_t terminate_window_ms, + rotation_outposts outposts = rotation_outposts::eth) { + const bool with_sol = outposts == rotation_outposts::eth_and_sol; + BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "setconfig"_n, mvo() ("epoch_duration_sec", EPOCH_DURATION_SEC) - ("operators_per_epoch", 1) - ("batch_operator_minimum_active", 3) - ("batch_op_groups", 3) + ("operators_per_epoch", ROTATION_OPERATORS_PER_GROUP) + ("batch_operator_minimum_active", ROTATION_OPERATORS_PER_GROUP * ROTATION_GROUPS) + ("batch_op_groups", ROTATION_GROUPS) ("epoch_retention_envelope_log_count", 200))); BOOST_REQUIRE_EQUAL(success(), setemitcfg_defaults()); BOOST_REQUIRE_EQUAL(success(), push(SYSIO_ACCOUNT, sysio_abi, SYSIO_ACCOUNT, "initt5"_n, mvo() ("start_time", fc::time_point_sec(control->head().block_time())))); + fc::variants required_batch_collateral{ + make_chain_min_bond(ETH_CHAIN_CODE, ETH_CHAIN_CODE, BATCH_OPERATOR_MINIMUM_COLLATERAL)}; + if (with_sol) { + required_batch_collateral.push_back( + make_chain_min_bond(SOL_CHAIN_CODE, SOL_CHAIN_CODE, BATCH_OPERATOR_MINIMUM_COLLATERAL)); + } BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, "setconfig"_n, mvo() ("max_available_producers", 21) ("max_available_batch_ops", 63) ("max_available_underwriters", 21) ("terminate_prune_delay_ms", 600000) - ("terminate_max_consecutive_misses", 5) - ("terminate_max_pct_misses_24h", 99) + ("terminate_max_consecutive_misses", ROTATION_MAX_CONSECUTIVE_MISSES) + ("terminate_max_pct_misses_24h", ROTATION_MAX_PERCENT_MISSES) ("terminate_window_ms", terminate_window_ms) ("req_prod_collat", fc::variants{}) - ("req_batchop_collat", fc::variants{ make_chain_min_bond("ETH", "ETH", 1) }) + ("req_batchop_collat", required_batch_collateral) ("req_uw_collat", fc::variants{}))); - register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, "ETH", 31337); + register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, ETH_CHAIN_CODE, ETH_EXTERNAL_CHAIN_ID); + if (with_sol) { + register_chain(opp::types::ChainKind::CHAIN_KIND_SVM, SOL_CHAIN_CODE, SOL_EXTERNAL_CHAIN_ID); + } // BATCHOP is the termination target: NON-bootstrapped (bootstrapped operators are exempt from // rolling-window termination -- see opreg::termcheck) and collateralized so it activates. @@ -981,7 +1060,13 @@ class sysio_msgch_chain_tester : public tester { BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, "regoperator"_n, mvo() ("account", BATCHOP.to_string())("type", opp::types::OperatorType::OPERATOR_TYPE_BATCH) ("is_bootstrapped", false))); - BOOST_REQUIRE_EQUAL(success(), depositinle(BATCHOP, "ETH", "ETH", 1)); + BOOST_REQUIRE_EQUAL(success(), depositinle(BATCHOP, ETH_CHAIN_CODE, ETH_CHAIN_CODE, + BATCH_OPERATOR_MINIMUM_COLLATERAL)); + if (with_sol) { + BOOST_REQUIRE_EQUAL(success(), depositinle(BATCHOP, SOL_CHAIN_CODE, SOL_CHAIN_CODE, + BATCH_OPERATOR_MINIMUM_COLLATERAL, + opp::types::ChainKind::CHAIN_KIND_SVM)); + } for (const auto& op : {BATCHOP_B, BATCHOP_C}) { BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, "regoperator"_n, mvo() ("account", op.to_string())("type", opp::types::OperatorType::OPERATOR_TYPE_BATCH) @@ -995,6 +1080,74 @@ class sysio_msgch_chain_tester : public tester { BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "schbatchgps"_n, mvo())); BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "advance"_n, mvo())); produce_blocks(); + + return with_sol ? std::vector{ETH_OUTPOST_ID, SOL_OUTPOST_ID} + : std::vector{ETH_OUTPOST_ID}; + } + + /// Walk the rotation through advance() until BATCHOP terminates. On its first duty epoch BATCHOP + /// delivers to every outpost in `outposts` (the anchor that keeps the percent rail below its + /// ceiling); on each later duty `n` it delivers only to `later_deliveries(n)`, which must leave an + /// outpost out. Requires exactly one dellog row per duty epoch, a miss unless BATCHOP reached every + /// outpost, and termination on the consecutive rail at the first duty past the threshold, with the + /// anchor still inside the window. + void walk_rotation_to_consecutive_termination( + const std::vector& outposts, + const std::function(uint32_t duty)>& later_deliveries) { + // The anchor, then one more consecutive miss than the threshold allows. + constexpr uint32_t kTerminatingDuty = ROTATION_MAX_CONSECUTIVE_MISSES + 2; + std::map tips; + uint32_t batchop_duties = 0; + bool terminated = false; + for (uint32_t step = 0; step < ROTATION_GROUPS * (kTerminatingDuty + 2) && !terminated; ++step) { + const uint32_t duty_epoch = current_epoch(); + const bool on_duty = duty_member() == BATCHOP; + bool reached_every_outpost = false; + if (on_duty) { + ++batchop_duties; + // Any outpost left out makes advance() record this duty epoch as a miss. + const auto reached = batchop_duties == 1 ? outposts : later_deliveries(batchop_duties); + if (batchop_duties > 1) BOOST_REQUIRE_LT(reached.size(), outposts.size()); + for (const uint64_t outpost : reached) { + auto& tip = tips[outpost]; + const auto envelope = encode_chained_delivery(ROTATION_DELIVERY_PAYLOAD, tip); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP, outpost, envelope)); + tip = next_stream_tip(envelope); + } + // Finalize the deliveries before advance_to_next_epoch's epoch-length jump, or the pending + // transactions expire across it. + produce_blocks(); + reached_every_outpost = reached.size() == outposts.size(); + } + advance_to_next_epoch(); + + if (on_duty) { + // One record for the duty epoch, however many outposts are active. + const auto rows = dellog_rows(BATCHOP, duty_epoch); + BOOST_REQUIRE_EQUAL(1u, rows.size()); + BOOST_REQUIRE_EQUAL(reached_every_outpost, rows.front()[opreg_fields::DELIVERED].as()); + } + + auto op = get_operator(BATCHOP); + BOOST_REQUIRE(!op.is_null()); + const auto status = op[opreg_fields::STATUS].as(); + if (status == opp::types::OperatorStatus::OPERATOR_STATUS_TERMINATED) { + terminated = true; + BOOST_REQUIRE_EQUAL(kTerminatingDuty, batchop_duties); + BOOST_REQUIRE_EQUAL(CONSECUTIVE_MISS_TERMINATION_REASON, op[opreg_fields::STATUS_REASON].as_string()); + // The delivered anchor sits exactly on the window edge at this miss and MUST survive the + // terminating advance's prune. This is what pins the exact boundary: an off-by-one in the + // span or the prune bound would erase this row (the surviving window would then be all-miss), + // whereas termination + reason hold either way. BATCHOP reached every outpost exactly + // once, so exactly one delivered row must remain. + BOOST_REQUIRE_EQUAL(1u, delivered_dellog_count(BATCHOP)); + } else { + // Still ACTIVE: BATCHOP must not terminate before its terminating duty. + BOOST_REQUIRE(status == opp::types::OperatorStatus::OPERATOR_STATUS_ACTIVE); + BOOST_REQUIRE_LT(batchop_duties, kTerminatingDuty); + } + } + BOOST_REQUIRE(terminated); } abi_serializer sysio_abi, token_abi, epoch_abi, opreg_abi, msgch_abi, chalg_abi, chains_abi, uwrit_abi, roa_abi; @@ -1622,7 +1775,7 @@ BOOST_FIXTURE_TEST_CASE(late_confirmation_after_consensus_recorded, sysio_msgch_ /// WNS-16: a non-canonical delivery must be slashed before its historical /// miss window can terminate it. The fixture builds its historical anchor and -/// miss through real `chkcons -> advance` transitions, then lowers the +/// misses through real `chkcons -> advance` transitions, then lowers the /// consecutive-miss threshold before the divergent epoch. Pre-fix, `termcheck` /// marked BATCHOP TERMINATED before `slashop` rejected that state and rolled /// the entire epoch back. @@ -1632,13 +1785,17 @@ BOOST_FIXTURE_TEST_CASE(noncanonical_delivery_slashes_before_termination, sysio_ constexpr uint32_t kPermissiveMaxConsecutiveMisses = 5; constexpr uint32_t kPermissiveMaxPercentMisses = 99; constexpr uint32_t kTerminatingMaxConsecutiveMisses = 1; + // Each missed epoch records one miss, so the run needs one more epoch than the terminating + // threshold allows. + constexpr uint32_t kHistoricalMissEpochs = kTerminatingMaxConsecutiveMisses + 1; constexpr const char* kHistoricalAnchorPayload = "history-anchor"; constexpr const char* kHistoricalMissPayload = "history-miss"; constexpr const char* kCanonicalPayload = "canonical"; constexpr const char* kNonCanonicalPayload = "non-canonical"; constexpr uint32_t kExpectedSlashActionsPerOutpost = 1; constexpr uint32_t kEpochAdvanceCount = 1; - constexpr uint32_t kExpectedDeliveredLogCount = 4; + // One delivered row per duty epoch: the historical anchor and the non-canonical delivery. + constexpr uint32_t kExpectedDeliveredLogCount = 2; bootstrap(/*n_batch_ops=*/kBatchOperatorCount, /*batchop_is_bootstrapped=*/false); BOOST_REQUIRE_EQUAL(kExpectedInitialEpoch, current_epoch()); @@ -1658,13 +1815,13 @@ BOOST_FIXTURE_TEST_CASE(noncanonical_delivery_slashes_before_termination, sysio_ } // Create reachable history under permissive rails. The first epoch's canonical anchor prevents - // the 99% percent rail from terminating BATCHOP while the next epoch records two real misses - // (one per active outpost) through advance()'s inline recorddel/termcheck calls. + // the 99% percent rail from terminating BATCHOP while each following epoch records one real miss + // through advance()'s inline recorddel/termcheck calls. Both outposts receive identical envelopes, + // so their inbound streams share one tip. set_termination_thresholds(kPermissiveMaxConsecutiveMisses, kPermissiveMaxPercentMisses); + inbound_stream_tip tip; const uint32_t anchor_epoch = current_epoch(); - const auto historical_anchor = encode_delivery(anchor_epoch, kHistoricalAnchorPayload); - const auto historical_anchor_digest = oracle::epoch_digest(decode_envelope(historical_anchor)); - const auto historical_anchor_message_id = delivery_message_id(historical_anchor); + const auto historical_anchor = encode_chained_delivery(kHistoricalAnchorPayload, tip); BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP, ETH_OUTPOST_ID, historical_anchor)); BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP, SOL_OUTPOST_ID, historical_anchor)); BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_B, ETH_OUTPOST_ID, historical_anchor)); @@ -1674,35 +1831,30 @@ BOOST_FIXTURE_TEST_CASE(noncanonical_delivery_slashes_before_termination, sysio_ BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_C, SOL_OUTPOST_ID, historical_anchor)); advance_via_consensus(); BOOST_REQUIRE_EQUAL(anchor_epoch + kEpochAdvanceCount, current_epoch()); + tip = next_stream_tip(historical_anchor); - const uint32_t missed_epoch = current_epoch(); - const auto historical_miss = encode_delivery( - missed_epoch, kHistoricalMissPayload, - oracle::digest_bytes(historical_anchor_digest), historical_anchor_message_id); - const auto historical_miss_digest = oracle::epoch_digest(decode_envelope(historical_miss)); - const auto historical_miss_message_id = delivery_message_id(historical_miss); - BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_B, ETH_OUTPOST_ID, historical_miss)); - BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_B, SOL_OUTPOST_ID, historical_miss)); - elapse_epoch_boundary(); - BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_C, ETH_OUTPOST_ID, historical_miss)); - BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_C, SOL_OUTPOST_ID, historical_miss)); - advance_via_consensus(); - BOOST_REQUIRE_EQUAL(missed_epoch + kEpochAdvanceCount, current_epoch()); - BOOST_REQUIRE_EQUAL(opp::types::OperatorStatus::OPERATOR_STATUS_ACTIVE, - get_operator(BATCHOP)[opreg_fields::STATUS] - .as()); + for (uint32_t miss = 0; miss < kHistoricalMissEpochs; ++miss) { + const uint32_t missed_epoch = current_epoch(); + const auto historical_miss = encode_chained_delivery(kHistoricalMissPayload + std::to_string(miss), tip); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_B, ETH_OUTPOST_ID, historical_miss)); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_B, SOL_OUTPOST_ID, historical_miss)); + elapse_epoch_boundary(); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_C, ETH_OUTPOST_ID, historical_miss)); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_C, SOL_OUTPOST_ID, historical_miss)); + advance_via_consensus(); + BOOST_REQUIRE_EQUAL(missed_epoch + kEpochAdvanceCount, current_epoch()); + BOOST_REQUIRE_EQUAL(opp::types::OperatorStatus::OPERATOR_STATUS_ACTIVE, + get_operator(BATCHOP)[opreg_fields::STATUS].as()); + tip = next_stream_tip(historical_miss); + } // The accumulated real misses are now terminating. A non-canonical delivery is still recorded // as delivered, so only slash-first ordering prevents termcheck from blocking this advance. set_termination_thresholds(kTerminatingMaxConsecutiveMisses, kPermissiveMaxPercentMisses); const uint32_t epoch = current_epoch(); - const auto canonical = encode_delivery( - epoch, kCanonicalPayload, - oracle::digest_bytes(historical_miss_digest), historical_miss_message_id); - const auto divergent = encode_delivery( - epoch, kNonCanonicalPayload, - oracle::digest_bytes(historical_miss_digest), historical_miss_message_id); + const auto canonical = encode_chained_delivery(kCanonicalPayload, tip); + const auto divergent = encode_chained_delivery(kNonCanonicalPayload, tip); const auto canonical_checksum = fc::sha256::hash(canonical.data(), canonical.size()); const auto divergent_checksum = fc::sha256::hash(divergent.data(), divergent.size()); BOOST_REQUIRE_NE(canonical_checksum, divergent_checksum); @@ -1753,6 +1905,32 @@ BOOST_FIXTURE_TEST_CASE(noncanonical_delivery_slashes_before_termination, sysio_ BOOST_REQUIRE_EQUAL(kExpectedDeliveredLogCount, delivered_dellog_count(BATCHOP)); } FC_LOG_AND_RETHROW() } +/// With two active outposts, advance() writes exactly one delivery-log row per expiring-group member +/// for the epoch, marked delivered only for the member that reached both outposts. +BOOST_FIXTURE_TEST_CASE(advance_records_one_row_per_member_per_epoch, sysio_msgch_chain_tester) { try { + constexpr uint32_t kBatchOperatorCount = 3; + constexpr const char* kPayload = "one-row-per-member"; + bootstrap(kBatchOperatorCount); + + const uint32_t epoch = current_epoch(); + const auto envelope = encode_delivery(epoch, kPayload); + // BATCHOP reaches both outposts, BATCHOP_B only ETH, BATCHOP_C only SOL. + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP, ETH_OUTPOST_ID, envelope)); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP, SOL_OUTPOST_ID, envelope)); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_B, ETH_OUTPOST_ID, envelope)); + BOOST_REQUIRE_EQUAL(success(), deliver_as(BATCHOP_C, SOL_OUTPOST_ID, envelope)); + produce_blocks(); + BOOST_REQUIRE_EQUAL(epoch + 1, advance_one_epoch()); + + const std::vector> expected_delivered{ + {BATCHOP, true}, {BATCHOP_B, false}, {BATCHOP_C, false}}; + for (const auto& [member, delivered] : expected_delivered) { + const auto rows = dellog_rows(member, epoch); + BOOST_REQUIRE_EQUAL(1u, rows.size()); + BOOST_REQUIRE_EQUAL(delivered, rows.front()[opreg_fields::DELIVERED].as()); + } +} FC_LOG_AND_RETHROW() } + // SEC-28 (huang review): terminate on the CONSECUTIVE-miss rail through the REAL rotation -- a // materialized three-group schedule driven by advance() with one outpost, at exactly the minimum // window the span bound accepts. A resident operator is on duty once per three-epoch rotation, and @@ -1764,57 +1942,30 @@ BOOST_FIXTURE_TEST_CASE(noncanonical_delivery_slashes_before_termination, sysio_ // span (or the prune/scan bound) ages it out, failing that assertion. Driven end to end by // advance()'s inline recorddel/termcheck rather than fabricated timing. BOOST_FIXTURE_TEST_CASE(terminate_at_duty_rotation_via_advance, sysio_msgch_chain_tester) { try { - constexpr uint32_t kGroups = 3; - constexpr uint32_t kMaxConsecMisses = 5; - // Exact minimum the SEC-28 bound accepts for this schedule: (misses + 1) duty rotations. - const uint64_t window_ms = - (uint64_t{kMaxConsecMisses} + 1) * kGroups * EPOCH_DURATION_SEC * 1000ULL; - - bootstrap_rotation(window_ms); - - // Walk the rotation via advance(). BATCHOP delivers on its first duty epoch (the anchor) and - // never again; every later duty epoch is a miss recorded by advance()'s inline recorddel. The - // sixth consecutive miss is BATCHOP's 7th duty (anchor + 6 misses). - constexpr uint32_t kTerminatingDuty = kMaxConsecMisses + 2; - uint32_t batchop_duties = 0; - bool terminated = false; - for (uint32_t epoch = 0; epoch < kGroups * (kMaxConsecMisses + 4) && !terminated; ++epoch) { - if (duty_member() == BATCHOP) { - ++batchop_duties; - if (batchop_duties == 1) { - // Anchor: one delivered inbound envelope (stream genesis) for the current epoch. - BOOST_REQUIRE_EQUAL(success(), - deliver_as(BATCHOP, ETH_OUTPOST_ID, - encode_delivery(current_epoch(), std::string("\x01", 1)))); - // Finalize the delivery before advance_to_next_epoch's epoch-length jump, or the pending - // transaction expires across it. - produce_blocks(); - } - // otherwise: withhold delivery, so advance() records this duty epoch as a miss - } - advance_to_next_epoch(); + const auto outposts = bootstrap_rotation(ROTATION_WINDOW_MS); - auto op = get_operator(BATCHOP); - BOOST_REQUIRE(!op.is_null()); - const auto status = op["status"].as(); - if (status == opp::types::OperatorStatus::OPERATOR_STATUS_TERMINATED) { - terminated = true; - BOOST_REQUIRE_EQUAL(kTerminatingDuty, batchop_duties); - BOOST_REQUIRE_EQUAL("rolling-window: >5 consecutive misses", - op["status_reason"].as_string()); - // The delivered anchor sits exactly on the window edge at this miss and MUST survive the - // terminating advance's prune. This is what pins the exact boundary: an off-by-one in the - // span or the prune bound would erase this row (the surviving window would then be all-miss), - // whereas termination + reason hold either way. BATCHOP delivered exactly once, so exactly - // one delivered row must remain. - BOOST_REQUIRE_EQUAL(1u, delivered_dellog_count(BATCHOP)); - } else { - // Still ACTIVE: BATCHOP must not terminate before its sixth miss (its 7th duty). - BOOST_REQUIRE(status == opp::types::OperatorStatus::OPERATOR_STATUS_ACTIVE); - BOOST_REQUIRE_LT(batchop_duties, kTerminatingDuty); - } - } - BOOST_REQUIRE(terminated); + // BATCHOP delivers on its first duty epoch (the anchor) and never again; every later duty epoch is + // a miss recorded by advance()'s inline recorddel. The sixth consecutive miss is BATCHOP's 7th duty. + walk_rotation_to_consecutive_termination(outposts, [](uint32_t) { return std::vector{}; }); +} FC_LOG_AND_RETHROW() } + +/// The same rotation with two active outposts. advance() writes one record per duty epoch, not one +/// per outpost, so a full outage keeps the duty-epoch ladder: BATCHOP is still ACTIVE after five fully +/// missed duty epochs and terminates at the sixth, with the anchor on the window edge. +BOOST_FIXTURE_TEST_CASE(terminate_at_duty_rotation_via_advance_with_two_outposts, sysio_msgch_chain_tester) { try { + const auto outposts = bootstrap_rotation(ROTATION_WINDOW_MS, rotation_outposts::eth_and_sol); + walk_rotation_to_consecutive_termination(outposts, [](uint32_t) { return std::vector{}; }); +} FC_LOG_AND_RETHROW() } + +/// Delivering to one of two active outposts is a miss for the duty epoch, recorded as a single row. A +/// member that keeps reaching only one outpost therefore climbs the consecutive ladder and terminates +/// at its sixth such duty epoch. +BOOST_FIXTURE_TEST_CASE(partial_delivery_is_a_duty_epoch_miss, sysio_msgch_chain_tester) { try { + const auto outposts = bootstrap_rotation(ROTATION_WINDOW_MS, rotation_outposts::eth_and_sol); + // Alternate the outpost reached, so a miss on either one counts. + walk_rotation_to_consecutive_termination(outposts, [&outposts](uint32_t duty) { + return std::vector{outposts[duty % outposts.size()]}; + }); } FC_LOG_AND_RETHROW() } // WNS-15(a) / WNS-08 (WIRE-346 / WIRE-322): consensus thresholds must derive from the group that diff --git a/contracts/tests/sysio.opreg_tests.cpp b/contracts/tests/sysio.opreg_tests.cpp index 401749a03f..d361e04f56 100644 --- a/contracts/tests/sysio.opreg_tests.cpp +++ b/contracts/tests/sysio.opreg_tests.cpp @@ -717,7 +717,7 @@ BOOST_FIXTURE_TEST_CASE(termcheck_terminates_at_duty_rotation_cadence, sysio_opr // resident operator is recorded once per 3-epoch rotation, so a // consecutive-miss run reaching the threshold spans // (threshold + 1) * kDutyRotationMs of wall clock — the arithmetic the - // SEC-28 span bound protects. Drive one outpost's records at exactly that + // span bound protects. Drive one record per duty epoch at exactly that // cadence under a window sized by the bound (with one rotation of margin // for block-time skew) and require the run to stay observable end to end. BOOST_REQUIRE_EQUAL(success(), set_epoch_config(kWindowBoundEpochDurationSec)); From 3b78bc7aa898db8c3c96e0be36ec8581a74225c5 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 23 Oct 2026 14:02:27 -0400 Subject: [PATCH 2/3] epoch: tighten the WNS-54 comments and describe the record in the README Change-Id: I993837341880f71d7d165fe1c739abdd2a4c398a --- contracts/sysio.epoch/README.md | 5 ++- contracts/sysio.epoch/src/sysio.epoch.cpp | 15 +++---- .../include/sysio.opreg/sysio.opreg.hpp | 44 +++++++++---------- 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/contracts/sysio.epoch/README.md b/contracts/sysio.epoch/README.md index 011fbdec97..d7f76ee02b 100644 --- a/contracts/sysio.epoch/README.md +++ b/contracts/sysio.epoch/README.md @@ -38,8 +38,9 @@ Operator registration, collateral and status live on `sysio.opreg`; the outpost 1. Emissions readiness gate (block-and-retry via `blocklog`, never a throw). 2. Delivery audit of the expiring group for every active outpost: every operator that delivered a - non-canonical envelope is slashed through `sysio.chalg::slashop`; each member's delivery result is - recorded with `sysio.opreg::recorddel`, followed by `sysio.opreg::termcheck`. + non-canonical envelope is slashed through `sysio.chalg::slashop`; then each member gets one + `sysio.opreg::recorddel` (delivered only if it reached every active outpost), followed by + `sysio.opreg::termcheck`. 3. Bump the epoch index, drop the expired group from the front of the window and append a new tail group drawn from `ACTIVE` batch operators that are not already resident (non-bootstrapped first). The resident groups stay disjoint, so an operator serves at most once every `batch_op_groups` diff --git a/contracts/sysio.epoch/src/sysio.epoch.cpp b/contracts/sysio.epoch/src/sysio.epoch.cpp index 6f09fc4f57..526300a7e6 100644 --- a/contracts/sysio.epoch/src/sysio.epoch.cpp +++ b/contracts/sysio.epoch/src/sysio.epoch.cpp @@ -414,11 +414,10 @@ void epoch::advance() { // (chain_code, current_epoch_index, batch_op_name == member) // - count the outposts the member delivered to and collect every // non-canonical deliverer - // Then slash all non-canonical deliverers before delivery accounting can - // terminate them, and record ONE result per member: delivered only if it - // delivered to every active outpost. `sysio.opreg`'s miss thresholds and - // window bound count duty epochs, so a record per outpost would scale the - // miss ladder with the outpost count. + // Then slash the non-canonical deliverers before delivery accounting can + // terminate them, and record one result per member, delivered only if it + // reached every active outpost: `sysio.opreg`'s miss thresholds count duty + // epochs, not outposts. // // The outpost set is sourced via a cross-contract read of // `sysio.chains::chains` (no local mirror) filtered to @@ -551,10 +550,8 @@ void epoch::advance() { // Preserve delivery history and ordinary termination accounting. Inline // slashing completes before these actions, and all mutations complete - // before finishadv reads the registry for outbound attestations. - // - // One record per member for this duty epoch; missing any active outpost makes it a miss. With - // no active outpost the group had nothing to deliver, so nothing is recorded. + // before finishadv reads the registry for outbound attestations. With no + // active outpost nothing was due, so nothing is recorded. if (active_outposts > 0) { for (const auto& observation : observations) { const bool delivered_everywhere = observation.outposts_delivered == active_outposts; diff --git a/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp b/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp index 96fa1eebb3..aa344d1b38 100644 --- a/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp +++ b/contracts/sysio.opreg/include/sysio.opreg/sysio.opreg.hpp @@ -92,16 +92,16 @@ namespace sysio { "production default percent-miss threshold must lie inside the accepted bounds"); /// Minimum accepted `terminate_window_ms` for a given consecutive-miss - /// threshold and epoch schedule. Delivery records accrue only on an - /// operator's DUTY epochs, one per duty epoch: `sysio.epoch::advance` - /// runs `recorddel` once per member of the expiring group, combining its - /// result across every active outpost. The schedule is a sliding window - /// of `batch_op_groups` groups, so a resident operator's duty interval is - /// `batch_op_groups` epochs. The rolling window must span the full - /// terminating run (`consecutive_misses + 1` records, `consecutive_misses` - /// duty intervals end to end) plus one duty interval of boundary slack, - /// or records age out (and are pruned) before `termcheck` can observe the - /// run, leaving the consecutive rail structurally vacuous. Operators + /// threshold and epoch schedule. Delivery records accrue one per DUTY + /// epoch: `sysio.epoch::advance` records each member of the expiring + /// group once, combined across the active outposts. The schedule is a + /// sliding window of `batch_op_groups` groups, so a resident operator's + /// duty interval is `batch_op_groups` epochs. The rolling window must + /// span the full terminating run (`consecutive_misses + 1` records, + /// `consecutive_misses` duty intervals end to end) plus one duty interval + /// of boundary slack, or records age out (and are pruned) before + /// `termcheck` can observe the run, leaving the consecutive rail + /// structurally vacuous. Operators /// benched by a surplus roster accrue no records while benched, so no /// finite window observes them; the bound governs resident operators, /// whose duty interval the sliding schedule pins at `batch_op_groups` @@ -251,23 +251,19 @@ namespace sysio { [[sysio::action]] void slash(name account, std::string reason); - /// Record a batch operator's delivery result for one duty epoch in the - /// rolling buffer. Called inline from `sysio.epoch::advance` once per - /// expiring-group member; `delivered` is true only if the member delivered - /// to every active outpost. Each write also sweeps up to - /// `MAX_DELLOG_PRUNE_PER_WRITE` rows that have aged out of the rolling - /// termination window, keeping the buffer bounded without a dedicated - /// crank. + /// Record a batch operator's result for one duty epoch in the rolling + /// buffer. Called inline from `sysio.epoch::advance` once per + /// expiring-group member; `delivered` only if it reached every active + /// outpost. Each write also prunes up to `MAX_DELLOG_PRUNE_PER_WRITE` rows + /// that have aged out of the window, so no dedicated crank is needed. [[sysio::action]] void recorddel(name account, uint32_t epoch, bool delivered); - /// Evaluate an operator's rolling delivery buffer and inline - /// `terminate(...)` when it holds more than - /// `terminate_max_consecutive_misses` consecutive misses, or a miss rate - /// whose whole-percent value (rounded down) exceeds - /// `terminate_max_pct_misses_24h`. Both rails count duty epochs, one row - /// each. Called by `sysio.epoch::advance` after every - /// `recorddel`. + /// Terminate the operator inline when its rolling buffer holds more than + /// `terminate_max_consecutive_misses` consecutive misses, or a + /// whole-percent miss rate (rounded down) above + /// `terminate_max_pct_misses_24h`. Both rails count duty epochs. Called + /// by `sysio.epoch::advance` after every `recorddel`. [[sysio::action]] void termcheck(name account); From 1cf719fce466964f7da5a6cb27b18f142d0d42a1 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 23 Oct 2026 18:58:30 -0400 Subject: [PATCH 3/3] epoch: test that advance records nothing without an active outpost With no active outpost, advance must send no recorddel or termcheck, since "delivered to every active outpost" would hold vacuously. The test also activates an outpost afterwards and checks that recording resumes. Change-Id: I5937099759bcbc567754afdbeebdaec5d68a9e4b --- contracts/tests/sysio.msgch_chain_tests.cpp | 77 +++++++++++++++++++-- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/contracts/tests/sysio.msgch_chain_tests.cpp b/contracts/tests/sysio.msgch_chain_tests.cpp index 503b723dc6..216722912d 100644 --- a/contracts/tests/sysio.msgch_chain_tests.cpp +++ b/contracts/tests/sysio.msgch_chain_tests.cpp @@ -80,7 +80,9 @@ constexpr uint32_t ENVELOPE_PRUNE_BUDGET = 4; /// sysio.opreg action identifiers used by the WNS-16 fixture. namespace opreg_actions { -constexpr name SET_CONFIG = "setconfig"_n; +constexpr name SET_CONFIG = "setconfig"_n; +constexpr name RECORD_DELIVERY = "recorddel"_n; +constexpr name TERMINATION_CHECK = "termcheck"_n; } // namespace opreg_actions /// sysio.opreg ABI field identifiers used by the WNS-16 fixture. @@ -194,6 +196,11 @@ constexpr const char* DISPUTE_ID = "dispute_id"; constexpr const char* OWNER = "owner"; } // namespace chalg_vote_fields +/// sysio.chains action identifiers used to activate a chain registered after genesis. +namespace chains_actions { +constexpr name ACTIVATE_CHAIN = "activchain"_n; +} // namespace chains_actions + /// sysio.epoch ABI field identifiers used by the WNS-16 fixture. namespace epoch_fields { constexpr const char* BATCH_OP_GROUPS = "batch_op_groups"; @@ -278,7 +285,7 @@ class sysio_msgch_chain_tester : public tester { (uint64_t{ROTATION_MAX_CONSECUTIVE_MISSES} + 1) * ROTATION_GROUPS * EPOCH_DURATION_SEC * MS_PER_SEC; /// Outposts the rotation fixture registers. - enum class rotation_outposts { eth, eth_and_sol }; + enum class rotation_outposts { none, eth, eth_and_sol }; /// Construct an OPP integration fixture, optionally with an activated ROA that has no /// registered node owners to exercise the msgch electorate preflight. @@ -959,14 +966,31 @@ class sysio_msgch_chain_tester : public tester { /// of next_epoch_start spaces every epoch's records exactly epoch_duration apart -- the exact-span /// timing the SEC-28 window bound is defined against, with no block-granularity drift. uint32_t advance_to_next_epoch() { + advance_to_next_epoch_trace(); + return current_epoch(); + } + + /// `advance_to_next_epoch`, returning the `advance` trace so a test can inspect its inline actions. + transaction_trace_ptr advance_to_next_epoch_trace() { auto st = read_epoch_state(); BOOST_REQUIRE(!st.is_null()); auto next = st["next_epoch_start"].as(); auto delta = next - control->head().block_time(); if (delta.count() > 0) produce_block(delta); - BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, MSGCH_ACCOUNT, "advance"_n, mvo())); + auto trace = sysio_system::test_support::push_contract_action_trace( + *this, EPOCH_ACCOUNT, epoch_abi, MSGCH_ACCOUNT, "advance"_n, mvo()); produce_blocks(); - return current_epoch(); + return trace; + } + + /// How many times `account` executed its `action` in `trace`, inline actions included. + static uint32_t action_count(const transaction_trace_ptr& trace, name account, name action) { + BOOST_REQUIRE(trace); + uint32_t n = 0; + for (const auto& at : trace->action_traces) { + if (at.receiver == account && at.act.account == account && at.act.name == action) ++n; + } + return n; } static fc::variant make_chain_min_bond(std::string_view chain_code, std::string_view token_code, @@ -1028,6 +1052,7 @@ class sysio_msgch_chain_tester : public tester { /// registered outposts' chain codes. std::vector bootstrap_rotation(uint64_t terminate_window_ms, bool batchop_is_bootstrapped = false, rotation_outposts outposts = rotation_outposts::eth) { + const bool with_eth = outposts != rotation_outposts::none; const bool with_sol = outposts == rotation_outposts::eth_and_sol; BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "setconfig"_n, mvo() @@ -1054,7 +1079,9 @@ class sysio_msgch_chain_tester : public tester { make_chain_min_bond(WIRE_CHAIN_CODE, WIRE_CHAIN_CODE, BATCH_OPERATOR_MINIMUM_COLLATERAL) }) ("req_uw_collat", fc::variants{}))); - register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, ETH_CHAIN_CODE, ETH_EXTERNAL_CHAIN_ID); + if (with_eth) { + register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, ETH_CHAIN_CODE, ETH_EXTERNAL_CHAIN_ID); + } if (with_sol) { register_chain(opp::types::ChainKind::CHAIN_KIND_SVM, SOL_CHAIN_CODE, SOL_EXTERNAL_CHAIN_ID); } @@ -1084,8 +1111,10 @@ class sysio_msgch_chain_tester : public tester { BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "advance"_n, mvo())); produce_blocks(); - return with_sol ? std::vector{ETH_OUTPOST_ID, SOL_OUTPOST_ID} - : std::vector{ETH_OUTPOST_ID}; + std::vector registered; + if (with_eth) registered.push_back(ETH_OUTPOST_ID); + if (with_sol) registered.push_back(SOL_OUTPOST_ID); + return registered; } /// Walk the rotation through advance() until BATCHOP terminates. On its first duty epoch BATCHOP @@ -1939,6 +1968,40 @@ BOOST_FIXTURE_TEST_CASE(advance_records_one_row_per_member_per_epoch, sysio_msgc } } FC_LOG_AND_RETHROW() } +/// With no active outpost the expiring group had nothing to deliver, so advance() sends neither recorddel +/// nor termcheck; "delivered to every active outpost" would otherwise hold vacuously (0 of 0). Once an +/// outpost is active, the next advance records and checks the expiring member again. +BOOST_FIXTURE_TEST_CASE(advance_without_active_outpost_records_nothing, sysio_msgch_chain_tester) { try { + BOOST_REQUIRE(bootstrap_rotation(ROTATION_WINDOW_MS, /*batchop_is_bootstrapped=*/false, + rotation_outposts::none).empty()); + + // One full rotation, so every group expires once. + for (uint32_t step = 0; step < ROTATION_GROUPS; ++step) { + const auto trace = advance_to_next_epoch_trace(); + BOOST_REQUIRE_EQUAL(0u, action_count(trace, OPREG_ACCOUNT, opreg_actions::RECORD_DELIVERY)); + BOOST_REQUIRE_EQUAL(0u, action_count(trace, OPREG_ACCOUNT, opreg_actions::TERMINATION_CHECK)); + } + for (const auto& member : {BATCHOP, BATCHOP_B, BATCHOP_C}) BOOST_REQUIRE(dellog_rows(member).empty()); + BOOST_REQUIRE(opp::types::OperatorStatus::OPERATOR_STATUS_ACTIVE == + get_operator(BATCHOP)[opreg_fields::STATUS].as()); + + // A chain registered after genesis starts inactive until activchain. + register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, ETH_CHAIN_CODE, ETH_EXTERNAL_CHAIN_ID); + BOOST_REQUIRE_EQUAL(success(), push(CHAINS_ACCOUNT, chains_abi, CHAINS_ACCOUNT, chains_actions::ACTIVATE_CHAIN, + mvo()("code", ETH_CHAIN_CODE))); + produce_blocks(); // land both before the advance's epoch-length jump expires them + const name expiring_member = duty_member(); + const uint32_t expiring_epoch = current_epoch(); + const auto trace = advance_to_next_epoch_trace(); + BOOST_REQUIRE_EQUAL(ROTATION_OPERATORS_PER_GROUP, + action_count(trace, OPREG_ACCOUNT, opreg_actions::RECORD_DELIVERY)); + BOOST_REQUIRE_EQUAL(ROTATION_OPERATORS_PER_GROUP, + action_count(trace, OPREG_ACCOUNT, opreg_actions::TERMINATION_CHECK)); + const auto rows = dellog_rows(expiring_member, expiring_epoch); + BOOST_REQUIRE_EQUAL(1u, rows.size()); + BOOST_REQUIRE(!rows.front()[opreg_fields::DELIVERED].as()); +} FC_LOG_AND_RETHROW() } + // SEC-28 (huang review): terminate on the CONSECUTIVE-miss rail through the REAL rotation -- a // materialized three-group schedule driven by advance() with one outpost, at exactly the minimum // window the span bound accepts. A resident operator is on duty once per three-epoch rotation, and