From b523961cee44c5a0ac97c786b3dda595a7f6fe63 Mon Sep 17 00:00:00 2001 From: naiba Date: Mon, 31 Aug 2026 07:05:06 +0000 Subject: [PATCH 01/48] docs: add sponsors to README --- .github/sponsor/99cdn.jpeg | Bin 0 -> 10865 bytes .github/sponsor/vmrack.jpeg | Bin 0 -> 18994 bytes README.md | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 .github/sponsor/99cdn.jpeg create mode 100644 .github/sponsor/vmrack.jpeg diff --git a/.github/sponsor/99cdn.jpeg b/.github/sponsor/99cdn.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..d3e098652992b7adfd4c552dcc0f1f465f3827f6 GIT binary patch literal 10865 zcmd6MWmsEHw{~zZ#T{B)3KVxJr9}%Aid$NwKylZS0xj~h05Z|vJ|Zg65`g-5ydI*iMifNm zKj;5dV?+S}7>E|s=e)c;(MW&!NXRIFtbeuhJb#{r^xqk?kpGntNiYlLUp^o=5B={L zM6y4Ij)d}e3p(P0|KLVK{%_n!*#84J(!a7H5hDL{6+)uF^T{LX-@O0S@LBi*;Opz`(x;T8^#~Q{B2nO*q^hJkx>x`7CIXGKkfgq17AX5 z#)h{72rvN}fGiXwdH^y35()tlyb~!0ffwx$z~8X{7y>5(7zQR5HqK)JG7<_3GAarh z0xu%+^Y2baB|sy5$|H?Vr1k-W-hr6cH#!@W;blb|iTW6XkXfa3G@R z^~EH98J%6xhQ+|A4k0mi9K$AMZg{-*SY(SJtB@BfR@KZO3_1D^-rq97p#421w7 z1%Sa=b3y?BNr8|4jx9OCnNE4>Ny=K@fj@r&!?s&j#^|~NWE5(^$eaURB zQ%-A;qyxsdexT#MRhyoRh+(Rq3df0MqX&npRLMNY&FpnB9Dw%ex^0~fb9NUFz&|em z;_nwoogYlVqJ$!(#&SXN#1SBmrU?H6t8EZIs7(fBwS=QZnFS3?oV9Kui_-7I@ggB8 zQiJUQKc%Ee2ZmP8Y&w1fDRNO}Yre}X6ey?&sl1!_^z@{gdLV&B9s=(^?t*HB0)ITS zTQ%hFyjABgnk!-!3Xeb?B^qvbXZMi#;)p45v_c2Ahzx)OsuKNG!zAeqlRCfVmz4qq zx(2_Pn<}i(%&uv;2pq@|sAxPCd(u^I-CM=4J9ee1mo|SZAF5Qa^BJ`Y zh6DWS?wzMkmVBu9L|wm`HC}-%WE<-_7Mw`sdq!C{3|9`D?|fk6VB!`Ikcs;`tXrA1 z0rnt#&H)*`w`jy59R9tn;w7L0Lg^YO%Dm6ov0|KAe`P zvk#X!;Pmo2H~@xCa;NL9bv?7D8aQ(s9ysSxrnrpNuG3BaLS@h1GfWm=q9g)3b5DGC znfVhsTw}6kD5I*(g>#p5*rA)7`IN~l^qGpnJi0RX(L8n9?L!a8t)l(OlooeFAxCUv zd!2&NWrHEzcn!(@g*PXHUW2##xC`r13%5mrXLm*?Myl#N8%O`UHr*7Ko+}^K@8t_8 zUxDOJNYJ`AL74cpi%;Dp867v>vuJs+rwCE8=+(OmYU9S|9^csEfDRDDi7-Q#L2M8= zZcBMf%Yatm(#)1~q;?LN4Gvh%1=g%|0q4MakRRV5Is4PgHtIM<$FMQvL08$aJbgFk zP4Jcx${jVRO-_>S{^j+Y8RTFid@3)2a-PacwO5R!eyh^0A9+D`0?{((tr&Kfbc>bp%s8F3rx5INj1!6>c`IGR0c+eS*D%8dm3;3ar^JPAK>k;+MvpLL z^Kx;g5ZD)xHE6Ez(hCj-vhJ_4x>4fS;w4czqDOW5nUQhPXu58Dj50*9eon3a3A58Jquxzt2F;YJp;d=Q$zPGBTiqC;&N_M+aNtL+n>SF6O9o9vxcT#ve1 z-`YF6)2=}~F4eBd?ilU?(5udofYIgG#fG~vd)QY(oV;JbF~Sj}FFI(SgRmc%uhlln zalPfPopRhK+6hY=TyrgnqYk@?Z2;LlFTdI_RFIdJhpnJDQ9u`3&+f&cx5KxFgV74? z`u!h1KL6=_lbB`;*qCo=5HTbldzjQb%9W~u_~38OT1R)VWn*dCd&kor)z2xV3@@B7 zWZ!4OhJhsKnSONS&Y2;U-RaClq;XYowIOOaWC^6fIwD%#OLKd`EKsFeD|pWQvJxs- zQm|))F?}A&$wiW|eQRhs*kWCX^;q+p<4}NY6Vfq9G@@GqU;`jDsgDn0?X`5+)+c0Y z(>wKNsCD=bN!c1mn)Z3!UD6z!NT}8}xoMud1R!VcC4UrTh z$Qll~X2FYW<3?A7>aQMc4IM|mM*+?)lXzr>MTzfjY8v+bqH3c+iL&Q0dJPAl&Ry@7 zct3l{ekM-BziH9J?eeoe=wZxJ`i?)#)n=t}#D2LQ>qjtAig98pHHfWSOtwR!1 z47BgbbBI7Mc=d}q%?|alDb70aHS3;j%9Ug_qW?NHvsTn11;KE8tQq1uK>A zY$)%v@7(OXT!=_TvqYnat_lvo_0jH1HsTN;t}^JrDX(-)tvf9`-P;9YW|n3E9===q zu);c1%Da;pH72W%RC$zSyxUtipy_Bj#s4RVwdy5U{@k_)b-`vIa3c zt_som=R+>wM?S*Wp=&!Gh=6C+=*nWpc02N;Q;cPLk4dSiByn)DSw6O8b)5 z2;!ZGsqzE54ZAiv)O#<8+x-*J<8Om0z>GvZlIkP7=G?pN{)@LB3lmRh+nfAXGY_Tj zr0&t7QP7i)Bf)hEwfA1mG3#49m@y`8@Qo!*8 zV)*2+aVcuj4#m0%31C-B<^VM-DXaJP2zEWPaBQsal@_>EH5`Q6L;^i&D{Ret44Yt1 zm|#5lgFENWH^fnc)&YTgY6Gz#1$}@6=B`#%!xPsr{vcO=>Q1PXnwUo47v4ouul2-) zA&pQ}W81?_@zgK;7eQo52JyDIba>kut78X-;>^k1oZkFvJ<-gU-kjG~Yv6BEkNE8Z zVB|qe36VB5P2P@OK4+Pi>6JsrR*)SSw)P#hyMG6azvUrYl3J&EIkE_Lr4>N3I9&(8 zn>eCQ%G>Y6kFQk4h}>mECq{Y`gK4H-`Tf27lL($Y!=y!24)se_m1Vg-1GlwW6p4N< zO*Kc>h3N91(;>@R<4Y}a&=ZLMWx@3x1YLx)h0Lnl(oD?GOF@mL)_m7M?#WZtA_Mal zE5z%_lkUiej759mr_??jiO<~qtA{UefEjnNUS~sh)tW`=@s?PIvZw;55Yc4it5>FX zN9DXy5GCLoL86IeL!>ytId6@8DD3hDHqjA*Oh^3T05Wkl^VLjpeFHe)<7^*pFiA~EM|0UKDI^)&5oRER zjjsp+Uc2-h2H%Ui=E4EKXWB*_NFk07gJln(_{g$#5DsG7;c;rMUXWj!DIS;4!Fo_o z%oC;#PTdDd(NTabfT3^zor|yle#rb{O6FYfojAEpR;Zg{R5yvV1fSnh6|f8hQC^v& zGPw3Md&qC$)U62{o9}ywHGjUq=s+@<$jjE<-gnwvn4T9SLUM}(DY>M7NP<27q;}v# z0BIrwb~@2a9Y`#GmviPL#r!;p0#vvTl&*l1270Xxy%3r!UyWHd479S2R}{qDwRi7P zbjOWRD>^DsJQeenfZ&jP9h9m)o8PZ-Db-xi-rZs;se5gMzuwbH8NU;Dulcz&t){!e z<5`D<(xSZnO%2GH=bj4!D};FB``?};8}ghdo&U;}IvadJFMQ5Gfl)!N-KQsYPsDMl zHYQ*tq1U4X=A9j)hMs`q6P@2sI5uPUZfftokEreEEgIc%u_-!mf4asjMt{3_W~(b& zt7ll^HPbx4yaNq~rmlSiLsZv&svsSmFCKCg>MasfGOSsW@@YpXFwT#FIdl;DOdJ~R zjm#i#jdu)+V_}NRkuJPQH>JwJaylxoF&yv(Tum491e(~{Ea@SD$=UTYX5P);`I8c> zeHxEJ8l4Kq_iL29XWq0Aso)y?_63Eb;j*pIc_U&fs49oP-XEpMQ%JS!T5&*-P01ic`=vW&m@Gz;*;lJkj4{Q(lciZ;S+!DpKZJL zWV>k4L_+|!Q|EcJxmkr{FLYk;wB{rB*0y75V-<3rT!;c~dL_kDNOh8CR_z0hbnuLskf{KdjF)A#d z7Xt&Ak^rB8@_%iA2|~F@g8wcE0jLrB)n5wA-*ONDfY5@_P?1p(auGoBF#rh_p=JF> zi}73JLqz|PiwH0fnvmKDUm{{&X@_V`5;FRinvQ-k*%f2sqzvlsjC0!BJNRTYoZ>1c z7+-0{t|4+!Ap{yk)_-&-L^2dqWHfXn48)JTzoi-!WE3C`__HI#6hu*A6SFiDLhW{kl+)Iy(>gyKY;w#|kj>j(9JGGEYte$e zICAaJ7=)HI_!JfC<*&?d8l(=1MY3~TV>Nn0=aLrG zt)Ef9ZRr#Zb0_O1^oN$y%jnad-Zb$y{Ua|pGG%ue5!MLS8`dxhZc+@D)wU6LH@!(-Xe9Soz>!j3Y4w=407uot*D*E42S1yOJH9a$ zi^(n0wnMdYk@!UFcv$poq$dXs$X|%Rly2`ySbH9$vcjKGNB}#K30bEzMfY0Wa9A#c zF{$U2?hjq3#a*P{G+id0D?tpdRfKjt zl^v%}W^RBJ5X&-=?sTzKuO|(A&o2!J(Q;62B_gSpo8v=fFuv%?q{yy*3nKCDXkjrN zJL^(?vd6hb?ZFk9Kk73f6{-O#Ut{=ea)g=|$Cdb~9fC90t2#X7U-o*}l1Rj}tM8jg-IM%GvbuT3KIgM`BZ8Y4kDOUn-+frs! zdp-}EY9*bKBqgZ~y6>+H)pjXP8?M4iM4dF{^g7d0C|*8^Evf{$p6LJLP~(UzY!O#E zQD{D??N|3XU4;Xr9)CC3q2VJxTvm)1C1&(1EBauvOxKt)L+a=1X57rFW2QHu$`VaP zedP&QdTn}9B~x2#Rz zxsduERGO~y>J<{fV>E5(`rZES_mZDy294B?_;VIcuxSB3?DCB+jQKG&9TPX=;~vIL zrnbR5qbVBI(V>q;jrCIt6L+^1HHJ-iV>O)GnQvEuWBdEU1Sp|k|IL77I6#3tM%xah zCSEa0xv-MAvZ?ZoS@BN#*RKJm$$Mz;^1}~UP;y-*E+xu4B)p~4mHG~?OF7*inFMsw z+P@r3Gae^#Zt-lYuf?=9HFoM`$d2JZTwN}>+veoIs$~{A73K-2it+e@!|UL$JvF6! z3pHM5&W|w3C5#B{-DnK;k~nx-1*CFM2UgQWI$yy7T3Vtv57UF1uVTJ+0W@zgePiFX}YvcxYN zY)N?AoC(3kE+v*l9z51&Xr+)DsjqvzRi1f0I^^6#btd__H5v78TzO=1`>UEwAwNys zQdU63xID^u#j43By>ODL*YF_T9^*M$mT(WlPj89N4>@l9F>QtqrQxY?z^}5m<6auc zg<~cC^Wj~ZtBE5K*c{vmRcU<-eWt*HDzmLqx}T3)Mf{*S74>`tIV1bApAJytpiol7h_;VmVxysph7XDykmY)W&XTmWG$1y_lYA)+H%BbnT?{mS#jm*=4ncR zgV~`KW>`HxdU!xtWV+G>M#nJCC*4hdjQC!I1VP6| z3O3uJ-Mwcxb8(ka>-j-xk}6xewcF4ybGrKgx-xlTnLIdv`Mk+#-kcdGXSuVIUv`aq zY&T!ZG$>+No4vTtIjRKTmJ`o!|%2Z$Ym`=+@Vlbt^HMww-& z4JT>w_zcJObDP#4Rka1DU`ipBQBVAQ(=0ct;`pZ(X|_d$SJ?e*+uR}dWQj-@R?F6w z1Cd(-;gIvbJ(RVonPIW1@BQZL{=y0j(wLUss-N=`whQZVB9B;(af5=h6 zWJaE@yqh}f!C7X@+7A32KJd03KYN+z_LHX*958f4^C;DWC4Xx1x{be?rvvg)f+q`J%j}6+;<31qn9&SK$ zYrA*CEMx%=NEn9$HuaW1E~AV?Zil^4tvSY`S44*2-wEj{C`%hNpUlmB*-$ssU1A26 z(g^=G6|N4hm+bIpL1&}|j$ZAq)XAedM@*f(+BRV=DkSSm>Ug`#wHMYL_E9-i8SAQ? z_D!h-zTF3db|gK-9J5=hRh7rFl|wIUw`wix8pAN_*+@c_wrZQ(qqq_1Eb{5jiL^~F_WbqeOEl0@NQU#~O7}1VXlHehW**36w-f829 zs`{Z1_8>mb039o>lJ9JU*){7UAN{HmC(Lpi?zU1N?N{0&2`vRiV}ev-7~rpTOkZiE zYda%q9uL|pX>8=V1-(Ys!spA5lRawGTPOYz7c-2PugrX!E<8CqZzV2Vv;QlKN*5uS2=VnE9ew-M5Je!UYj$ggU z+cX2|UH4Q~p0roKcI=)iHuXKdr*MFMv*BEbGR?X3m5hA4xYv-`?OD(0fa1ZW_-uHs z3ofcJ$DH92%J8Tu#sPbr;SgO>;kd~DoZ=Cr>?3rDOge^P3u(T%UEegh*K&rSrtzDu zqXCSA&dG#+>%9xQl=?<4IPm8J$;zb1Jm_`Dpck2@^-ZSE*A3yK*IRs<$(@=bCe|jE zoKE|*cFZq&5@Li_@;r$-7M0}S0RIYD864nQp=P{6Hbk;NuRoKp|5M$^irIAiIHtoA zC#kre6-ZPVA~&?g%zaiZR9NTIw-yn1UpghQmLaK$oq0jp5JZD}!co>kl{4KKBN<6t zgv$bUg9Fs`n;6H<-|XpRTBfDmAV1SNu!Sj!r4+NWX?}i0A0NkcVcE^2_}E*0t0O>? z`&c!az);f9<5({uT;Xt5JSjVTT>Jtc#6i$47@qN^JcGx9@@KAOTH7#l1<$5o0$-zv zdB$_^V+q9|4XZw(uij_#IK?gp-D$M0=G)M__q*?Noo^O3-kfH-lHx14WA9wPT-uyy z9eXgX9QHBM8-raStk_%>X*jb0OZ)3$K+9h<(bjwL~}+{le+MW_eMI{{~8_Dnvg>dHVQ?fSjt&C^cA zE3)`RGZT65x;DUiW&K!*yW}H?zKXOxrjy`&i9XZZMA^f9r{|b}K+(qUU%SNd zaZ2N*%Y^HNK)>P%NWL52XG4j8^(tTfG{`}BoN!!3T-c?JR-;0$@m1WC3 z7LN=P_r1x=fxZS3$I)9O9?>U^iHWuf5{yNGxYdPQMnz?~W2fIW=DS*QFl5B~134Ai zm06vK0x!UGrWpSlO2ZzOhTQgm^xO51oV^?z-JJp%gAF;7a(`=|*AgYrx*}7TOQzM+vr3 z6U5L>p>BCdZJH$Uj~%y?++UixG~?$Yc!VR(21N>9Ki40;$`kAma#Hwvfjr<6c)r383il-v61o_uZr;H+rl|r2NC&p?D|lC^!U-`4?Yip@UIK zmq`7sU~pZ-arHF>4#@L|3Gl4>G){@8>Sz_l_ownN;KmHmg|;uXTA-wtVf3>nX`KK{ z1d^?f6@gwfdNyGhSyztWy!=8ok^ESjg2g4Rra~;uSY0^4e66QE8XDO~1HxP!J|!q(?sl@WD?s>dJcB0K966h`*1J1mR6~J zlP*_e>KyqXJa99T8#S@G!!GT`CS1y87tFNOr6kX8ANwwKg}Gi*JaHhDdHJAWVxt2V zgMU3E#n#FBF8OO$^w7MwTz)hXrbJ*ARYO=#sCPq3vXQ}%u_oh8$^EwrYP$s49wHjX z$fsc@RKFO{0DwIv7kg14Pti8P>aLQtT}70T#lmTZhu^5U#M*0R#%)vHNV9t|OL7}H zHL&8#&7IcULbl4Oh*YCnrB9un*WRKT$D|hixjz|~tUgyayh>2r#cMt^smMC~9>Hcr zc5&(W!p+(V^TiE8KxQfD#|jm#lUfh!ZzoV!v(&x%U#j{9At9o8o{wv)^X*gCZ*3<^c7QxX5% zmVVf=YHXhi$=LW22R*c=^viocB4<(R+ODeVuydlW6|lRLT{}~H*9EpBIgz=9-Y4G> ze3nBwlJM}8!bBNcqzTCKvI!oezIs7G08o>&pHL7^6hnEF=zKhJRiwvF zshUip$iKhg{jNtdhIN_Jl3C=3Epn8T%Ic{oaTx)jQdFUA`m@6j-QAx)e60h|6>vM^ zslz$l4^XW&%oj)G@1>P?>6(=Y5P3Ft$4hS4GmHDki#nFLWhVuT*4Sl?F@ou%OLt5H zrLk3lf|+{z)7hO9o)R~cBGYp#AmlocWKOBzA*7=g-t=`*ypW@)h*L6h$b@|CuU7Sy z&EJ{b*E9L0#XJRQ%~4lUUa9_bO&14B6MWwYcgXcFTaSqy?(D)(P-^CM!)}wBQjBalsA&y)Of{N?m&WEz%B8STFbu9d*m%x_pPDvyz52#2uDrP=m<8pYrW9cx_Z9q*uJ-Qr8dMSw0ufUT-Lz(HF_a8`y>TlrU=1Qd=sX z$?=o3JM>Zc&#n&pJz6bNr)NxjsuC{>%wh9FRP1UUCmh^cZTita3zh!Yxc^zA O{&frdA9X?Sx&H#mw>=U?v~*0PH;^K8r&ri+=2vqGg)h|Bk$Ya zJ?H&>w`Y$rYPMHZYgP5j;>#w0E+-``1wcSR0N&sq@Un$DDd}x%1ptbQ03!eZ@Bj=1 zApir0Ai$3t1kpdR8U#H6^(Pz>03fmchQX;KYys%M)9Zo1P2dMy@}K;Fec|2%030|3 znw^uABN5^+7y=RsDEL=62RnNK#DA1n0Qs*HA#Dnv{slu66~X?U23*)52MYo9cL*%F z!hhNg0r?;8hCuu;?S}ZLG)%~UYQbpyyF3~2_cz{uzAuX}D?sRPT7exT*h4`=LP9}9 zfgLmq^lt|P3-iap{_WuYIQZWV;UD)WHzXu9*dxHg!2T2dKL%dbz->l+=>;(00Y{(! z3W5@V#DIXpfOr{%_yBG%%%2ASZuTDsZYQ{5aPSC-Naz401QZk`G!zWDz2KC7zljcw z0fR}wAqI=3Y6?f`ip?36SO`xgUf+kKHhoUbW#$%)fQXBSPe4dROGnSZ$j!sc$1fl# z@mf+!T1Hk*T|-k#TSr&V+``hz+Q!z--NV!CjknL+kkGL3h{&kuq~w&;wDgS3tPdZH zic3n%$}1Wgo0?l%+uA$&2L^|RM@GlSXJ+TV%`Yr2EpKh_?CyQvKlpKYad~z9^XB&M z{uj6x{Dltq{r|!T1B?$OG&B@6+;4m!Am4x=C=6&A3JzFIF;zHIS1d}-Ab4!?#KQVM z1S&4Ia~w0bX+&IV?k$>&-zfdT=zoS#@c$=_{z2#;d|s9TWGDzQ!=Nw#5b*p={~;Rq zpFA&9|BJNK&rg}b3qyH)FF%Ir zkCnbzbK5vg)(uK%dm=XB4QTz?An>Ag3q}IF67XZ5Esr_TVhOyw$ z0G>F~w+JTO(xTO%wvvEwow^_S?Ub>X%?~|~GB;9}rD@U}0V{S9EjJ=(v^RTq&h`)K z-#6A5MSJYflbbM@lY;1hRxDimt5Zk5u1~Ih#MZikHjm}f2KknM2?Nf@cvE%j&fV^ldU$VL`fqf2NQ}8-*$NSVgT9Q%#qd^-+ z$LteP_X`jb0@{pS_B>wp+ZcHPP^;=*No}&M0HZE0TC2gFg7L!iP&ZQ1k27}&F91v& zc#Ii6&#r?S@3hbPcAUdKdoT`BsTyZLP0cs1j=q(KG$=B%&l}A&U)28L2w-au8VP{2 zBs>oWpaj?}qTS$KOb)Vul8yz;^CZVpcIF$+WzHMv<$;-U zUubY;UtWMdwjSFTAX+b=PdRXdz=I9ru2fTZeNrjp1Y?Z7C2T+4HVxZ2iGFExa!vh* zC|N1C_IN-;?a9Cct^dk0Y2&u?5@vUa_O6jW9^Bwx|u1`5Z2!OH%Tri50)J zkI;t67VQVM$?7M=0xZauu7*Al&s?_^Jvc`>ojGL*@G#Y;4hyCD4qM*WE%q1LPiZZj zfafRZrHeNsplFvz{X066*f)A9;=)Gj-<8R^rva*5crEA#Qv=squ~YjkvO5p56XWF# z@O0RF_Pk*HqcUWjt^AS_V?dYOxSEpcJX@BkpFw*o=qvFo&9-mt ztBEu&hVs~70}bV{I}a^qHq8WjM`vvJ^D<20P`-ye{k9Ogvy+7;(}_+!+2?(Dh}?9 z^;pB{>Rj;R;pg?!Wqfdt=nG)EvG{IBE96{a7}N-Nf`trUJsM+0RF4feZ9NS%<3N3k12w5s= zL@Yz<9)<$WQWeSmvl=z`)gt!qroL;CRK z>x|gDhiS6yBTlx(^*w`z?NRZbJ8RL5Zn`Je@)HfiE_hAz_zju?9c z^v!IXO9w2&-2`5M+1}$kxkxwgwxsz2c1bE|`xP5nINs$7{L1m011iSd!OoLnrpdHv zw2tr|;{^96tf55T@xFMy5w!wax{wiED(&9(KtbPF7m(2>)ER(hmz&!t?tL9}$% zQaEw0BKMNka%8m$IBX~$e0|qOe zj}32PlssVTQ=OkaJ)9&iKtvJ`7Kct2if@z5ItNcpk%aL*7r2TpEZ?egPxg*gmx>(5 zK-oev(*`WPk>XUoE&6-+Vq`yFZG)DRR&E5r$U81Qe=}}wI)m8{dBiX+_YSh4u*z1~ zrw>g#%D;CMUb=iA;1?D^kl&*Hh`zs*|A=Y*jA(q`c?0gh!3K{cU3UZL>+>=-HIHmU zP^MIX z`Rt(G(NFdv70o^dIB(rG2`f5!`_UP<^9ORH6Y{YTC(R;RxCEi{)$3qkYC+~-((-c)Q`;*+l z0j25#Cn#!AV2d8=2<=aQkbOXVUNWxJ4i9k1Utv8mxKNIKrV?IzMAtoJvxdp-(pFn3 zvx@aeR;GGABXfF36dVSP8Q=%9&cEZ+enyCRK=M_JxhcAX+L|z17+S}#pHs@`Ijwc~ zM0WJFgZKjxAEc{E&ASNHzNE~02H&ag!GIUc9DG}_7we0=Rf=c zG_DPJn;YKEZrj`Nkl}o$TiTwzE-PS{$Hqc6OH=1Nvl=!FBWG0*$-)Vxf1MaBl z9*Isa&~75Zg?K;uK4pTOUjX1R0OjYJ_Fzj^e(+B}r6sk~;a}@BG81!1Io+MOGW!al zsr>5gn&~ICcYOO=;GDhBy&xQ?p3N-dmWBQREc+frhgv5VA-)|ZZ^eo5_ZO!ol^!x^ z#RfqgY}-MLtW#B=@!?)(pRa`6w@{cy)-mRMacW?B-xhbJ&D>j5!X&z;QoSE26OOFi zw;m#^OFwZgxQo7(p5Z<^QJY!Ul40HM5>1vu8{OpYjg7Uy=-cioW@P$ljN)tj<<59n z8U}xyt9gfdx#O_vTc_Vz!ho;&ITHPjvbm6->`X3GF{2~x?7azuCD;new z@C)xJ+apR0c*0$n?r9)<#Cq6f=^1)ZCVX_BR?087iDlghe-ka?=~2poOvN`IvG4Gc z9QAvTcUkR}`$jPz`i_gHi)yJ-zTjRLcCzfvZKrKZF4}mT`GMkqfS0*6M;>LfvBg}9 z)#mo{@((fFJwLCvC}Lrdxso`KGXdsd*s7v1|K=fknfqUyZNL35(k_7oS}TAA0G84G z_FzE)SWW-~iwO5yK7)dQ2#<({f{u=cf`*ELg^!JaiHC`bhC_^lM?gSCM1+n_LPkPJ zhEGUD_`47YXc(|~2NoF)4w(=G4TJE1?Oyu90vm`TNSpsGumOm{@}<8d9Dj>z001nj zfq{aC1dDS3BCycoPwYRD0J!~7kPvYHk|Ur)p@RiEJ-z>t;Q!YUYCx@!ceC@=Y2!_V zRa&R%ZI;Bl_U#VEw8~|@PH|naC)W;J>w#NS4~naUe?zEJ2SzGI#ogG@4~L+El*xg zGMsPy<@BeJz6 zsd#kaWj*m@xwA4$UL;Qn$ymX6UP3#s#_3t1A!%M%)#L6q<#tsf-O}o;@pJ3w@}B?7 zEJ$b0YaST06|Hpx3wuUG1ME!Be{1K$Sfj1P1weSy#XbrEb1+mx${; zim%;gWkH(;5J7#ITYZ@OeFLma^|hBP_u81;n#BE66(S>i-ur->!$QS{xvw4c_vfc| zBhtlB1ap?)Emd9LP)@_h zxWeeLyz#>$^V?hP>_;S``0hzTJO%US0tzhGKnpCyw$;VTdR&pX zdTCpT9BzL;idTFLC$kW%(~ArxZeKoU*yaqixdlxw>d<*(7wb}#jV;5|1w``SKKnPT z(xkxUv3I$nFM2372(%@8O%1`dxa@t^Ye#e2nt7Pw?=;Id0W1wQQ{^g8?wScQcxa7? zwhVMO5w}O~EeDIRjK5YgAS+D1AN(RgCn8c`G-=!Ufk)9@vzrLnuW+`~2|2z4O^8AX z{!$c@Cicy`5hGdWR*!sb4H+-`LfdiEPfHZvR4MKCJD0r!L?p)~Lh#~VrI6cDUz0?^ zL(;CAEzx;Cit1$ZO~@R1od-^6qelplUUk*@m3LS9*t#~0aZ)pyGe(OI+Sg3>H+DXn zX^`ExnB9cqAF?E(^If+l#=7x?!}oCbB|L?8VReNt~eWa!SV9-0_{{WJH) z)z#fkV{6}P8J1Fsk5ADk>;Q20+=rg>0EqCf_WI`Q@C$DdBtuP3y?!UGI=J_FB)>ST5PMDZ=k(B_PXr_ zm~vAiiXsU@UH?)Q8Gfl%I7JtUV!dL)u@O2w_2%j2@k|-RiP=_K9|?kOk`wDF}n0GK=lNZ90JIk zZGp<5Wxy>Nn~9SO*A6!wV{HWUv&9vqT8UdSUtx9vS;p?HIW|cqRvZhGeB@c0Wik;h z9UJwR=OuqlYUUb$v-bPw{qIcq8I_*EPz9YDN%K%J?Tm zP-;nloXMJW#?v0Xun2PE1LikO>i8Od1S`lSQ-sP|DR@c-)FC{St+%TLAZyi85%oP? zT@l%MhD2OKmvwbt2=>W2hupA>QHEFBxq6UlU2A&c-3o^%`fAiSuF%1K(z70jKA9aL zT=`_4Y@lwZIbUm(r4JT)6g)q{Ji3Q3!XZb&jmyb7b#CkAhSTPHGPiQdmB^eVi2l+| z`ejy6u{ZWMDZ5;PLJwMLvVjCWOj zVlES9FkT}j%(@WKRp&e#7S~B@P}+ezKaOXmtmvn<1ElskY*bjV(JDTTi%Duk`~vdK}{~mGx{tw^9sh+M%^xqWfU_)$svW&Z+F-vbOAw8&Z!yp$EDK-EP}z` znKdBl!|LbAHcBGv=twwCuhd!WEAG;0Ovg?VCr(*WZ#1imBYQ8AFWzb6HH$&6@}0e- z9sF+OQ87~L=fpU#871CDU|2uav9iE{wjDhWn0vx@1Y2)s^@cd@YpYoLksm}VetPw& z#jpH1(r1o+T6UaP0v};-wI*9`M!QG#dvjw=cJn*blWc;Ek4#u7-w}U8gZeyYcgXac zGrF(MQ)e%Jq^PfyfqJAAJG81D?J)*hL^7ceU$6I;v|>zB;C6-TU1HdeN|qEgJc%r< zfBsS)VawN{A!u1L%mL?v*CN!P%C^9O_FXan8$T%&BSkUtqqTxdXp62vL9j>ZqyBqxL*wRxz1h6c;%735_5@F{Smk)eBeJ++Bt1%F|hqs_nj%3e1&9E{7#_QR!<8&@#v`#WEt8tMrGq( zP=o*2W@pTPShjE(z@7bckDpD5U9^T+mY9=GZ2>#fTe{xlMdGUbDp2YnVgBtaFRE50 zX|F>Cr^S@&b>TVVwSxC%ReT9iMj|v;1wt0feGQ<(aldORi$0n(%9J&`5$GcFBtOy2 z&KKZ4(p=7FPTTeJ+)Qt(Y+`2btKNJSxi9!NPHye(hfS{R{ zf4(1d^V+_&@Mn)hzt#e2O}6B8hhK_}MJP>tR?3<-yh#6$);m1cgaySt-%aogVS~5r zMKAE7&RX2VqtjS!VA5a?wGgjUVIG#~t;8!1*Bw6>@r6(&11W3iSE2I6l`HbUJmwdE z35MA(p1>Y7q9}5Q6P?g9aS?3yRR~#`@hqFD?h+>?AVr^97P8XCM0AT2p}Sc{21^>~@KE52 zzx#p$1mD%!h4PZ`PGCSsx7FE2p+quI5TM6_!R@}v7}y$SLNFMY5F37LOSz@kgNp*r z>T2?BC`3aiR@};F)WTt%o2K~Y*;c35b0Y?}?ES}H`>wl%IJ&>O?Vr9o6EV0p`V1;a zj_NS~FTt;u>!08Qo1d};a2gZ!|LaR4ob_u!Z7X=~WW@j=puv_Ba8NM+(ozGz)$SOW zSR88DI24p(Duw8xffA~&LG|Z*^6mu~62j$gs^T3^Kgl=l88qckteh&d zV(${J?B4xhct$5z?x1fA?On;9b(zM9>t#85uEk*dGyADZ5M{~pEm}`AP>*4E%-exy z+BnsTk{4#w-A03ktP|&W(2OcRp_-!?B5{w{g;Fzta)Z>8t6JFCUh_+sjOkN?^!VAw z>C1R-KZ47~36t|qE{12}aGez{B$C6DN^Uk57JW63)Mh%_-yam+5^o#3c;3O@;d2l} z511P)f4BiY=|lguj{zT$|KbJ~2PTe~8a9PWVZEqo;4~Fy&|k~|8*}_-hQ0hMAvQjb zmK1adXFa2bou~oPcdsiZgn<=@^v_S?d)V)UCr@_zqy8|@VJCu`SyCazjyc;m(5*H znD+E6EJSnDsk7s_IJ0ip=rmee-u5(|CRG1hjOeVhkr@ft{H%z>Yi;;Ehrt=;&qsLly$FxNBgEh{jBr`^5G2Nm~gJ4I+NCUbAaF zZPvY2(~al7z|6a%ib>ex12azsybXnff`UPUhll-ZANrS(3OWS`CMBnsDhw7m6}Pyl zYhvFtHW!auP*U<1yM&rYNa4o@YIPH{;QE4d9P^70e{WG?LEtTFjF)(j>qcG1Zb0Q_ zy!5XgdSo9Cr^5x-7H6@BtE?egnSUaF1HmbpsIe-w2Yh7i(D94<{d--CmFlJX4qnf-JzLH7oZ)?_} zmIeaxM)5rdI1D1L+Eg`+4hS+8Q3dGX?Tb|>>~3hpdt(>P%pZ;DOoD0V>P|l>v8&Zf zLWwiO(1w~KxaYCz{CEze5wXRioaNq}kQ_YkS6!EDSr)yarp-H@Ldvu4-`T$nQ;&gHqJ1vbUNi6j^SV^1>VfiwKLw3(kpC zSC@i6D|}DNwF^gHU0ye-g2U`iYi6QMu`DOBQyU66#V8sXh@%xDw&ghAkxhueJA*sl zPQJ8JS)M7s$KRkf94|>yV%Xvz-ur^9W3&5>c&7$^Ct2PP*r)Kuw=)gVT2oTYFoA&T2iE@w)oO8v|y{yhiO7)2{ zme&U9D!9TQ##eX8A6NV?z}W&*oEcSnVGF}1sNVU>zziPqI-}%qtiHvvBXGtc9hFdA zO^Fswb=YT9u9(u|FFDgzmW-Egso_9x$(hzEVoB#ta$HaIp07C~rXv#lh-%&Ztzm%r z$Bt#wTh(5r{e~Kjht&hgJe7k=%%wHEZx*|~eIpvunof1>jl<(P2^ZfSzAV=}3Uo7m zL)81KmSU%v&ttC!U)3dn+2k+OI@@ifW7hHYrjCUNL;1-i-iqvP^faFmj)IFM^A6iiu^noD!1zBH7uwMYSeX|^wX$Bnj*7UEv4!HLcA;J@+K}enx*pd1*w)FG8 zn2d`3m>VFRybJ-jIK#1vP{H*i1&aA=4JJs-`keK2%`#8HrtoP5JruP<)jm}lEi+?b zl2?+k6Da65_057?th{dT3pEm}bau{i(pHqzVuiVHSc$^>%0%U0)lzJLaLeaL+1X}Q z!Dkp0#@x5n_RQ0zcg45DJnMYHF^jUfLQod$;j1V#ly*Wl`3^#yYd+IrHj3sCGge| zZjc_TRwt(9oZaaKmfZrFXucy8xfhEJ0^SeCOlO(Drb5s*J_Za4@}Ip^YB~Q zM7Ktgqd5no?_E_3T+2uE%(+yUknc2@4F-bC(-hT@NXfv&87AuHygYz+B>iF*j3wM5 z^Gt`k0w&0&bIt%nxtaEBB%9oh3N#u1iI*$WyQ?bLF(bB(8h?2urLrv zhGGz@yBAFr+o6pE4Vs(Vt9(orGV5seSLGirSyv&^jN&RE=+1ZkI$TVR*LC4zX9&Tc z55ISKi?v1EZM5x6m0r239P0M-UDSCr!OsWG#5I_av{GU%M%8lm6yBubvT1VrC?E3@ z80{k>#56axL^TCrj-PK?&`#)7lPii2VMgvlMgmLj z7mKd@5V5I92}z{+u0q>lW(rjyEg`R0KULa%9BOwxXk>wkaRXb6K^JIb2ZlCr?30v6 z^fd8DWtDj_8OA!9bjiP?N+OB*JwL22)cm5`{>K0%@tg{w@&RXP;|RTKUBvo%xEA?`;vr_j_oYUTyd;*Ky{!>QdsT zSW7m|RvqyQ9$xFqs@{mN8YRtXmP33t3u;LoAj>vvxpEXEX1h`v2 zEm2q7NZkYsAK0IQ)#-MP?kVk78XuXLs9QuvsMssvP!Tkcckz7S;$&kZUo=f4Tdu8O zHZl-_?<-&4ij`E@y_nkKrj@LbSV53U=2)@w)8=}rr*Mf*s3kiNn~`d z=R*Sge|rUdrZ+SdwS4AtuS0O49^Tu!c<3aagqMImZ$~as3oso&j@Y58me%d zz>%^qD^>8B-}%}~>e+TxJ(BArbmWo>_a2_7I@9(lZe6fiKC4b!1BUk8XEw`05 zY1m@-tPdaZ@ViOR#!$2R?gYo=MrvE^0-YV!=)%F2G7pK%njhQ^d2JAU$$?R}Ms67S zNA~sh_;uUbYa153t(U7vCuH?-6)lys0rGs4cm>m;?nkH(_o1zf+N)F!dFjX z&{%DiN_sn}veUN)RaQ=Np5($zn_DslU9nBG0=I_C=5Z^GXZ8#3v>uByX4Dg3G}FO? zo;V$=-T?WvE&L2d5tWS8`#1_hi5A2h9l)bM5*vDX{r>py7vBgVnaJ8~Z8WjzOzlxY1=N^;zn=i`)j@>Mbo5m-TE{WGb7 zMcI*pJJZKE;>nc#_=qFa<_H?smCogqfo0MkmyQEdfRva(VxWCVD^I7BLHzFeo$&jj z(YkQR`Z4@|O<~5BLz#j<)lj`nwqLnsVA(MrMGQl|=?r(!uN2sAJf-_FR`SO$cGRp|?P~%zLjhSW2EvbD z1Wu(h-B%Mh_uEM=C*6cT=k5BW3y)s0zDxX0d`td{ogNEpWs>mwpvUz4tN;xS_eV+k zN5~BUV1N$_VydRDLFB)!zRxeF*-f?z{*hRNC|-cHP4_lmh$vo#;WTr36EQ)0$55iX z2rT_%j^d1&Ra-ME(!zRJuw42qByh+i4<9;84j_RcGDvh1$NCgNtPH&RGM%UYk@^9a z>K7LG?7=7 zuP9~QYP?lMIkaK2Km)IJdiSMvQ4kq2Y?Yrb$caa($h^AVL>ZWdyo(IdsB{pETwKLp4sg28{kS%>kO;OQO{VI1$Z&(kJ(3RHLXcCVhn7OhXc_z9 z>V;qeiP@KPq_{r$LB`+~FzA&y*|?MqPY z9P)Dh3a?5ON<$ZHA$$mHsP(D>AG#=xUf+GsL*-wQHqiLnqcZ72UOF335gk<4lGM5gh@2m%yKjQPWUDACZ$ zq;S!|MGN_RZJku5lKX~;FG|Y|YLRbfJF4tt)Q6oVJ^#k2!ki-7fGn3Ui_8Sj;?nG> zvJxMd@+MGBf3i%nIFuUDD5s;TqERFgd4-W=y}Z`} zavDpMi%qtE0T_}Q_8?81puxuIZCkZABd~|?{N)a(Q(?@8^vSSj(X8??$=y=sD2*?uqlBiRQi)Z;?yL_fhStE0`3(fJ~7)6bSEsTEHC=8L{!60ZDLJ7 zl(kLvOoPXft*{zpu_`*BckY+SR#_1xg>b}hYZ>k6)HyQc&cg6Mqo={UbvmeW$q-Nn z`0SFt8|7&8Q*R8H%pcJKbU2r{qsUqCUS{4l%Jr70s{SvAS$8!s*eXFd=n#>X zuZbmd$m2L^U(4tR56QSJCvMH}MJi;f>$|!0ZXcCwUcSaOr_3)1A9@?gTo|wNIz*dz zHW}`M0NOS2^<~c*($wq;DIRNtT zt)5$ml4ls7lrHxoR$}aU?{S2FJjNHh276KHey0NNjnQ}u4C3pM0y_5Eu2Gs`NK0FyvYRi7NcW z{(ekTr_hcFO_70%s99&s#*^~3JNZNJW>^+J2~NUS@Xmn%H?QV?nA6@tx@QVbqtV!}d_iWJ*Vy+EQiWA$o3H>^O7T~7 z4_&x6ELh|alkX~hpWdPx3>0IS?2+)Xf0r+<$W9Dj+^NsQ$Og-#5Z1?`zK}FLZd!{{ z!pFh^$wl2~{kJ`c&7yU3&xetC9z_odFr$6Eh^4pXjs@@IO=#sz#OErpvgS7WoX#maAq_jL${YYGpYkgf9{vHg%?QaV&Q(;9 zXkH<=U|NW`v5^2<0@@4%{tYbV<2-$A2w&h&s3Ow*6ue`%iZY!HwCtckc5Nyt7}@4c zy<#)M?L&u16Lt+!j2KLI4@6FKQPIW_b0}Wm`tx^K7yC2;Vlh2fwO#^Cj$Ua>mW63! zNk5!x?P+5v`rz}BSZHJ{TJeq&!z|+==lV>)hV(^I0{C_DK*R@2Td>mOHAH6b@&`(| zJeo-Fz=wK%WJEdvg9FE*QpOYVK)hLUKXiydd=~^p&ajFH3lHAy*;KDUQGB)*vHnX7 zv8iq;g&Dtw2g{q!C+RB;feo&B>v&<>vurKK4xg435XCa5&@89)jh)xkbrq2 zEl6MbtJuHlJ5H`L&nwz_0YtzizJFiX`g5-kY-{y@-ERs&Uxo2D1pXUZ;L*O4=lnO3 z5crU>t`*1tx=A3qc`&ev2#ARTreF(ifI3K#y#P(z{*@RwYmgW= zFzCSm#|Sc%i6aL9O>Y*&-y2CE2+bXg%HAsuK!{=kFhVcDuZYWDQ2-t~B$^p)fkFYk z)*1-Z^#Uql=V(ae=;Jr9_D{iXl+IJ4VQ#!qMA*vEskN%JL3;nUzMGArht2gi) z5;lDXf?EYV5}s&F?{RR{wy4c0d_3AzD=947WDHYYZVl-H5u8;5 zDf62dgV-i=X4|+M{(?eZ?tz+k&Zj%a~K@E+ipZ}JG->G z84^Z7kT)^;gnCiq7zP}@*Q+L`3LjoXqpeSL>Nv@Pr~{aRA)#OS#j+xdd>)}kkWAGJ z#8?9RUVpLI>}pjih$Ue3+{@xThdZfYh@5K*d_N^C8Ot=?)FN&K<&!dZh2)F+>FG{~ z_cTu-5(;8G{DMlWm-|6H76^RA`so@Kglfw_kSCkhWvDUb>cK&chsWsgMG&L}hmeBb zdc6DmsBnNn?-d?dCIS1num^K5_R6Kmg)&##f>-IaiTzFJq=A8kkH&e{lSIuyC_8ug zkF((@nY~>VcCU+HT0;J*31mh194VVN79U8s%q3>8rWoL{Vh9uHW4@1-i2!lEaOiJ_ z3w>UtC%5~&O0L_nT!S2jcZepXcD?XJ7sF#eY40qQ3xpZ5E6sEFE>PfO>=P|qkCDG{&Oc!CS)S%wyOP3DI9mE66;?T)vY6`dZG1x!}u4=|3- zSK{1o?N||EQctf4Nb}E?SzDyohZ_j@Nkx`=969O5&41Lr4YJl90AElf&OfJ~&U`cu z*Q9pRma93DOQ@+~WQQ@8NjOI)VdOX=b-Y87E0*!q$CS8dfOKSgzrClwy#N-bNe?Z6 z+AmyeNLe;HfU^OcXZTa@Dw)&Cjd-ReWY-p|PZ;GAKTyLB z!VVus8g{?)1NesjwT1;}BuqcNp7-!iG*r>Z1b~9XCI{*8;f^X=^3frV(oI=hG*98} z_eFizf~HaNLJpG+O^~KSd}7BF)8O;bzx)9)A2h>|13?})2F@Z z)e+Ns?Xe7m?zmO0#)Tt%l0x(FO2M&=Bp3kgx>K$LkJN*WZ%r#`osv*tP{^|?!Mut;|Tqs zG9|I-hALZ*#SfK3rawCagYlY5*@K!3X8)6LQ}=FAe!YItTUclz9sUz($&Z8^nA93drOAT{H4_M z%RADz_wG&+lgb(kTkOYSU7vzfVjS;!LW2CUTkI}qH3f{-sDg64Q7**9EjVdWp_uCB zA1v!#yb6x*gOLFOJzxb^{kvhu*D!+gV+1=h{z#mN!g7amz}WJ!!Y^=u$#lY`d`%lF z&<%+Y&lDml7D7zCU+{DoH5IjM+u6C-s=${nf^e644wo4B$n%TETg}~V>z*|~SUDO{~<0J@+}I6-#$u0W0_h=3pG4(A_?R>Aq9cAU5Gx@8zFUWfggR~QK^xNqBDGL4*eQ3(hx9ooj*y!*_k)p|dS&&`JrOC= zTUFJeFeuiI#5XqhX1*o8uv#&=r4blcB%Tg(Lh`r!s_(n8LCo|Q*gzmJh;%!2RuW2GLaM6$H^&3ztkBt$=dHIt z#U>$UeD~UE6T>NeKaw0AWVx&*#&(|d&d4WWh%yMb9%eCZ4GzXgB-0soMJ~ryAV(#Z zL4GfJGhk^#Y)u^>I5$(PaC}2LWGFQ!%X*v3G$T$5QoMNqAcUp*ix2NQkODv>DllRO zBc%OrY(t1B%Unu4@1GNSVw~(I|Ac?ZH@Aveh!) z7XpIPRt~b9G$<+d8QP`FEA${8-atUzop_r7cN>(-p z&XoN``;jR%o7vpRS>NT23rxPIL038yzP`~x?7idW;gV+F2t(3n9-u!UBwsD3riWR3 z;$_yUbA;;5>nnFMe-uM(7CZ|rgGjY|OmLI*WkQvTyhIHuR_av?;)E$_J6`=Y#RTIV zrX>4wO{*xkjCECLj*c^~3U6tIUfi?jnrM$xgM}5=r4yoR8#G1MjcfD00uF4{PqaIR zx;ft}i|ysx3;SAuMjBlO*ueez=!c-T)pD^kFrVAcBBM*DgU0l(rA4V&0;Mgeas*xR z#>;#~^>dyJ_dCH?A-B38%^pM8aey$aH}hQbn=Sb#Bvpjqy}CA}9fJs32L^V-!&P;w)Sj0bh7e)_}>XmA=%z!O{-atc3{zXa89pZkg{MRm7o*tk?Heh_JKl~$-p+K#> z)G3+3MxcFOL(tfL88;1`DO&%Go{mlbAPA|YVW5EEcw}TlCTI*w?j}Z9{K8muRPaU? zM?crgfg@p1<|=w8_P$;-U-c+UCLUna>|e zUvD;o-Ae%-7Q_}Hd7%Q88;#PTwXM$~GevGnC?M=aR_BtM082E(Ke^S?Q2&XePay6n zit|kYg7Hk{FU%vj6KCF%KUn$vfMo3i(J3DeJCu4x~A-z+f=EZwND*PCP4(V z5fXk7ep2dl*UlToY`uyDm)Fqpd);uZHiKFf=)69nxi+A(Ow~YC7C|vwynyZ^`29V) z&1rVotElv|!|d_zU-6~=T_sfdw#|cDk#cPrgkGDH(lA{%ww$uI=lt*|j_T${O7c$& za6T`gef`S3ckd^oOM&tt!mCuUdFf{S>(Pv&wUl-PGOop%rnRaZrN4;Ojj;DEEV;tj`B+fhLAGs-dEVnWy%Kayna$g$pGpx2}w}7LT8&{hmTKzH=vdp(d$k$F(GV~1vldo}h zjXm&j4CzvbuH~e0_x%+>-vIyxbt9=81zi*dtTGAWD95nA5ifowhf+bbQ5fBD!#&Ob j-{uta>xU+a^FjOOJ^9}~IM7)sX7QuR3$VZOvh@D|{nB3i literal 0 HcmV?d00001 diff --git a/README.md b/README.md index ca743b13..73e11b62 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,40 @@ \>> [Use Cases | 我们的用户](https://www.google.com/search?q=%22%E5%93%AA%E5%90%92%E7%9B%91%E6%8E%A7+Nezha+Monitoring%22) (Google)
+## Sponsors / 赞助商 + + + + + + + + +
+ + VMRack + +
+ VMRack 洛杉矶 VPS｜CN2 GIA + AS9929 + CMIN2 三网精品线路，AMD + NVMe 高性能！ +

+ 👉 vmrack + 		+ + 99CDN + +
+ 🚀 99CDN｜自建 CDN，不止是加速！ +
+ 边缘缓存 + 智能调度 + 自动故障切换 + 商业化运营 +

+ 🌐 99cdn + 		+ 🤝

+ 期待您的支持 + 		+ 🤝

+ 期待您的支持 +
## User Guide From fffbe686506c9483237b3356a62f622272515114 Mon Sep 17 00:00:00 2001 From: naiba Date: Tue, 1 Sep 2026 08:16:45 +0000 Subject: [PATCH 02/48] docs: add AxisNow sponsor --- .github/sponsor/axisnow.jpeg | Bin 0 -> 10272 bytes README.md | 11 +++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .github/sponsor/axisnow.jpeg diff --git a/.github/sponsor/axisnow.jpeg b/.github/sponsor/axisnow.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..ae2c109f6a7807002ba2a3232a6d9721fb2ff6ee GIT binary patch literal 10272 zcmbVy1zc2J*YBC2ySp3dMv#5>qTkdjooL`tMZkVfflk)crmk#6Y*85D6481B*c zd7k&a_x`@`#+o_%tl4L;eb!og?KOM<&$*tvUImCXRW(!r2m}Bi;0L%~BYC0{=;8zb zIywL^007tk280&C06Fj~pb4S-hra{i24H{cp#T6S`-=x##kv6SzuH@X=O*w_|Lga^ z&REX@01K>v3yFvbr$GM2L!dCA;O}~2A)x}uzk4iz{@o+Qr~vji4|tVe{?!KT>;FaU}9kYQ~!U>TrYz<<6d_IB-nsBPymCl z0#FhNj0AGs4=Do8#`vS)@8SAGpq`*&SlBqYc*Fn{0)s)}Fbq&%uw}?^r^87wNLhpx zG06<bub+QF;G@S;(J@bApT@D|pgmAH90tewZ4U$*03I+BI0lO_CaIzUmYp{lt4IVkxl&4DeGd+s=p7`5 z{ljrwN_Md|j+5Ue{juo(&Y;NuFN^*$=pTEo7XW-11RO9J2_O$#1MEDD30X8 zb7wVT# zB*jjgEEy6ly*c|sowNCNqork7^XNsp(=&5T#f#p0wp{-s1tzj_Rbdr?+_W=T`sxMx zX0RnnZ^P!94dJCcXX%<=tbxVo`i;{}xEA#>yRd>edx}HIS0T9T+;PNRA_WYJy)aBP zQLy?|K`5^?6TFMa#j`rFcz~+jWx+bylU~Q$M|hFmZp$GlfKgRf2H(7Ca^|!>;^xcz(>{{9`ySchR3i{u&Gk z`CsVRFozWsH`k%(g#daql=SM=HQ-S@m!pJ=@MJe!Swz+Bt!U32yVsVAvd2j`6Y<#! znVnh1;9?5fjAdq&?00-E#Y`$()Oz&j;kuM<1b(XePIy^7j(retQ8h7Br zJ{WgA%&8W|Q+qgFgt`V+Ra0CuXBK`wGnN&ta*^;3r>NTDGh?gu=*xHe)@-#+udxwAg9&0N5H?zYRrA@o!W4F-o!`ZH%A&l*eZcI zf#a~(+x*Y_d1|-g7bsHKrM5XpUO>AYB359DCkx^7EjEk!XuMKT9hR#%Xi=i1FA`Td zvgRj2Hj@w5C!3B@>OV{-eG|0Ius#q}CrYOF(K}p@<^WLKW{leAIlBR)Nj&GfdHW^O{$orz&+Cqxskdz%ilyts1`%~|N1GM6ue1qEI zs|yK~&shPArNG6eIcz|WWRLRKY%E`}Uim=r?wtWXjho9Bb5FNiGGLr{?t8wAyn{$5 zLh;-5*P?FRAF@fvl;Z-uml}Mt3waCir0`~F!aeumt7O%>R`$^KnSFT?9yPrD6@)JE zi5rGog6aab$F*lZSLMFRD7KwTVXR9spME*xceVWyf=|2n7S)NXiU-wyg~T*}dCXCs z;o!AIJsmhCewiEA!-{aA%U?`GDqb3ViuZ^5vkW-yRbOe#XgB}I*ySOoviYc_B~=VpFJIJ|H0FAf%%P})|{&2m`1<3}yRqbWaXg(S?$EgvOb+qXhoTZ+mq*wv`CKxJx;U!??X+%RX- z&*^<7dPd#DL(G2kQZ`(Zw+`!j!o{u7(L5?t+680iY(7;e_T}vp|JYST?D(FDx%0># zJe0Y<9T9l+q!~{3$a3A-94h#tzFmro{z~{wSdX;)YSuMSk2vf>rfuB6q<{^|6CYoy zoHVL1vWz!*#`?G55N5xT!HCxyey3v1>66>!jd9~W4YmE)994Iy2+AFd@YX8?nkn#D zt7AgE8h5DA|7J|aln(t&tB~tM_PUSJj@C4q&qHpUGW7``{m<23FNM8p!g^&VUNyKRh*ySGO!WgFR{a!{Vid6Y!WzR4Um>Z^RKd7B zMUQkbUS**<7J`kCOmWTE0AXO1`n27*iTy7(K0JHSjD63E#JZ=+litx>cS>62!Vr!7 zXzhp%ji+)Ucd2{zI+(@H9qS0|f^YxVle7&wJoQMqcdNS|4wNs8*ll6_sg*biHf^=< zjh{;Doe-U=T;<6-g8!oY`G+@ZgV~V}F9l9w`)QgO!Un^L);c7;1B3K4eHZD25;Ta@ zsjeo47q?Dqp620Sb)eb62#^MoUR|EpRjDi%>6gAYsJ9kPGfi=AQN|M<9-CTkX|f8< z$%;a?Nw`;!G0hRC}VlHZ&-R=Pa^k`h~73%fM1AM~k~^r?JD*@1Dm=r`7oj zgIuxx`)_AFcG}|U ztQ$Ls-ny|-7`_EeCz+7Og7IipP{mt;)5Tdz%-I|7z>Wf7i8@~a(Gx%78q5x-CoAVh#|r)9(X^J@ETyx zdikUO*b^g)#@tvjaEVlU16lk243}tl0@0F;l1k{wCqmfC&oh~!WRPhF@ds#H#Cpgz zFu=3f3Ku-17g7{2VR(YEHJ%X^JjZYYl~F)+J&vY8V_dqSVlSjFZIL@eHqoucIwyiL zf@@Pvi^a5>tZfmkY}88>cKcOAYH)i*7-IER7=4&C7{q+Dm*g^idBmW5o?m()x=@}4 zc*zujkwfOwl*8J2!&)}C4p$p6Wjt3T##z;9Q#?4>zRrF()FkAF{fa)J>J?ldLlfG8 z(&d3W84Ky4tvJJzBnc|ce1OdwXXyYQ`LdeIoIQv$WkZq=+rjjpr{RzfMvQ%;5#n@MpW z4p^}_JX3{;_KzAR@9@$(3r_3#!{BUrXy6*p^iF$B9@W$nB@2?#%K4vH#WHZQHr zKuqMBo73{AHVS;{wN)dj4rN6hpz$QZzW`QL{RRI4qJC(QDV{bfQ5J8Vs39RC7EGiiJ?yZ%}Pf|-w z`*iof;eMkdyFS8bQ$~$@?d_bY@xTvHf?$8t{OTy}P+u4EiGO|R-Y>_UvzS-SrH9A? z&w$CXIFjBH#8DZ+v6XRcLa_0@^}tJa$8u}sssS%S(sK^KcFXX^{v!Q=IGE2{=~164 zeXNo%mky}bvTMLI%sDXMC?8E)lvH*6HXG4|fBzv}RqJs}D_f|S+e|2K=pv#;@^G=x z3T1N^jC!+|WgX$5(GxO|Z z_OuV2W#bAAr`G^QD3^60E>TRD9LrP*afLn`rue2ifSwSRhJuE($;`Khwg-%UerS)8&z*G?d~;LZjJ&9v6S^T2T92#^_aOc0qb6s$VLOVg zk+jk(+0d^P{;XoF!IcpMUZ?t4o*P7Q`6Qt%YIa1)en-j{I=sv7e3`$twIVz{+VIGg zRAq>5?Mwk%Zz~mYeXQ$=KO4P&E$;tb1u7BO+&@WnyHYrKe|R7vN%MZn8vxxjZ({=A}oWToX^grVa038Si{n?oR1u}pq5MzMDpdbhVkpI0^ z|M$hex4pzLVi0#2>;CtZe{nc3Jc?_Fw8|N6!tR#3z1G~mU$-pV4V(I=)eis~s^K^f zSuRivm-qh&LUo7qa`^mzvA3)2MxRMkXqOm#uzXpY$85FtGJ-#8QTpTX>s|e{Wv-;h zR|aegvE!Ks=A%pTC41!d%lfTTgp^6B`#Ml=2w z=2r^sv7t0}_8*y1X?XWuEI)BmU2h7Y#bC;{;_xYStH7A39TcEIApwBPHu_mgRzPNQ zn_T2#-XkAU;)4J5pvJG;|5*VJ!wnGb0r4Iv1`Ojb=mZ9bf-ELE88a~nDTM^9h(4u| zn7AzmI6Bzim_ab0*MO)+tn3+%24yNCwf}l2_I6$ykzL&>U9F~`s6#9WSVUQUH#@{O z6>m};>-b9gxJqJ8Hn$z#=*5kj*&H>Rc82XEOk}p+dqj($9oju3)$Nnax99M}$MA`o z{Ia61o5P~hV8!g?_NP@+q7J&%8NxUHvyC2kyeFC$Cg_;)oj5nnXXrpw*AyH^aBq_y z-ks5!Jv02oR@$ZfTE{)VePt(f_oKo{fG3giQp{jsTSg{FzEQYt8hlA}==M_MxL{sh zym@Zkjj)k|#Y5jO?`#U~+r7vZSjkF_Vx=q71k+|~Xg(L!3iq`n$(cNo37)&r=cku2 z*^FiAZT@;@{^Aw-q45E0{ys(H4c_TFJd!fBT=50(-7a{T&2b>;$Vbdf zzc@emlqWtUiiAMy?98XKzDO9wir`svc($vow5!ubwO?4l)gU@YN`dX#xY1l~Cz(@y zcb2{QQE(dj$P}uLK>QQZ&eO6bbWa)Hu~aga*|K@%I-3>e-qyG0y%FRZ^xVdhX7#9R zf9OGdQ56@=HT<)Dj80%a^DM1h(}N9 zpU!7cWRvN${7x7zwDJjl;;5^_g=z|Oa>@d=maqE{@ZrBT$n$;Z3p>UXy@*CcCt zS$}F;4~B{Qvy#;9&{n)vTT*zEVVg@im!=&pDy<9UbFld6-qCaWMF6ya(4eZWuORZ1 zGvz_G>6)P(IiY{*t?KGGa+;#zG%bE=4W$L96~=L8gYK@&PxvbCOeUqwKwKe{fgEj^ z;%1zCP^PWUN5QFe3xyN5zgm3Ki@BxyZ1I?Oue5cx4F4k z!33M&gdT1-#lJdiR-V4p-$=hB{V5D)egEdU7H6HGis>7J8&G*a=M$1vA(IaEN8-jY zNr_cBD1{dZ83!==n@mTNRA)G{M%sOF=apE+E;18ApSja4Mn#v8i;Z&kf=Zh( zOP@Y4Vb)J{E;fhq+; zWa}iKWYW(ob~SHrs~%4@Xi%(%wCfxlz3Jy}Yy+l_ju#5g_LNE0tON|K)v1+UtH<-N!O39z|VLVXEKOUM=7Umw2kYxC1RL%zd$dqnNz%^ z#7KMQ4>8Lqmu0KX&x2cYiy~i2RVWS1nzdw4*zkV21wpjCG2mbN0#w#>crwNuJGom( zYQKs*q-eIn#y`+fdO)M>7rv}ddG|cmjBV(=)mg9H_A01Cs?d2_v3Tc!IVz8_rtC6UX}k$7Gik$jzQ=X|;k3)z>W`Ig_T#(g@oB5X$e zC(tlL(hnfmd5BBLd3G=0X zw={nsE?><%NMmO`^r$pj(w$y~FNHmU`|}ckidKwcnha(=OvlA3+4b|cvctX^%PSSi z-hu3^gmvd(E-!ivdKz=--YP+N@hzi%zQlvl*$^$}xliiv9?5ZN4YVaqe}Y>riO_D_ z-Fbf8HThv=q=B({&?3{*yoRSJ+v-P+$W)q}(@mRL)r}ORWQli9rM)aiuFF$wNj}2M zmM=|?#2s&{%kcLm-l=|JXy-8XddzilVE5?mew&22Px&SGJVi#!Ns-cas&YeOL*1B1 zV<7Gw-$cxhP0wDICpz7|`I0cUdbj0=EH{;<2!k4lLb@?t!%Acv>w_w8s&Ul;I-W3| zQq;Y{#-XxXr5mc|7KdTjWDa9?Zg;0Rw!fLXyRH0^2h%ApF}TP8AJ|}m3HDzJ7eWI3 z&XjiE5rs%%=9K!Las9OuA=|$fB=RiRK(<=aPt^h4hO?1<<8|W6n-~r;m{7UBrVsY) z_ub3hWZl!Q2$j+P^n4>Er5&Y?Wk0Vb9T<#kL8^A1_(%!UI({M%D@NeEL7o?YCRD#TWJFeB_$X| zVa;0*_7{<_&qFRI`dg^AGNiuZ4~FcSe$pT`L6tM=1b<9QOjg^5PB9Cl4VBV4_b4`gj9c60v*Qmme)o8G^J>!eP?d=% z;{m4YNOjQn&~oW$HrdhOJFxlL$Hq&8O1|9zWx?%>B|<@|V?%GEH7yA}_^KC=GKZ);OwCQHAPYEa1!AlHU%B9Z7@F2SFp~*-=PPC}Yjcy>{Ttn!TrlaF z14DkrbDhfQ2^x$0PH~1JlD-$T^a@VM_OCN)lJfoea?`Z$SNDp@m;5!Ii@FMKuTE4u6?h27VYX1&u!^w>l;#ggY-xq#i7o!Jm3UfF5HS$el9Y?;lRlhz)1eJ=5jjQNoG^Q7BG-7fv(-3W4#0X#8GVwH6o>Qm$Q zEt@V`W~w#?HRFAU?;eh|KCs@C2^YXxDptaO#-hdMAkpHg@s;j{Ul{HV6|0d$*6lBE z1u|BeOLlt%FgyI2o!wE1^GTC|RY|&Hhi=yZ6PQWaeQ`!Be zgo^bk0IT8nfFd^#4n8BtuY<4y@j!7heTsq@Yd4^*KpX+S?|>}?0f-|3Ay7uS?~mYr zRS^9*%gEpBOyd93I`dBh?%$ace9z$b8wdZo)8Cns*gGPH8GOF2j})?9`zLRL;Gucs zO5-D8+A%4v0d^`02fwvgbh;alBfckA-Bc)f>Nqx`pSwQ4l!Abxv|0T7hUof-C*;M7 z_*0XaM1i!aD|P)ZlDKk(H&9wP_?-Q$*psA539YbKrGjPjyNzTPqej0}@On%-*i0&g z5ML($P*flMinU#~L)^@VOr(Pzn%|Bv+`2 z2eA^uA55wb;hm0jEV&Gww!+j}y)rv&e_lZ{xqGVAG@ zG%CCAhco%=RZS;i_t!h1YzQCb^uLNN$`QFQY8#bM6h9F0UMO!YexoaLv&rRzct;DU zI4gukV~#yrwmJ>-M4ay=(LZ(2)eu`R=+D2BWpWMoj0+J`TWU4(w3pbM%+wj$`ke7Byy1NCekSVDkn{;jj~}ArAwuhmB>c;+!OP+GsYx!6W@2ijm2mrwmI2p zHB!wH-`m@ifK;nxXz8*;TQrI7T?=*);pr3M)7F*e*ff#89hLlFB1+t%(jAJ;44(0_ zLj$3+n0SE$?6=9iU(RPmy7Wx(vlOS$`Hqpl4`Ghc=UJU33m*>ox0=iSGlR6V)Jpb^N2CCplfPmsCIoybup zfnmL6C|em#5V;&lZ+OdV0&TK$nhjc>K<}Ys%IO@_Vl@SGFr>( zjHiaq@pSNQ>io%?U`_H$+co;moBIB35*eOPi2T;8$QJ|hPYWMAx3#<#xkBSCbNGR?xrLI6p<5v{qYvvajq1;!Y1%(b5{`fWN>!}-F&M%)OX(nKv#8&3U zP-4AirBWn;XHL4xvOIl#jI~+DqM@-&r}vmVMF|f|IiEJ9&8Khb!|KIfU0k)d122!j zC~-tnl#XqUrc%_@3JA`CG{75;Ip z+=X=e@8-n0Y#MP8!egLs*42soxh`E{r=Dv#Getx9S!G2No{Nldd#<^}KHamGXBsBv zUpPG#Y0ZE-p_0CL+!W+Ds#;T(W|`U3sWrJeEB2i4V0*FHdXkPXbllpEi{{;8bXgV8 z)k$V2B5!+G*AHpv>})Pv(+9A*g~$l+ufL#IA`wawchJl>$*|uV=kI*(|Eg>(5=-G~ z@79wa2&WIjb&gj0IEpOa!(fY~yF^PAtOn!Du8#zD*Rn(i$PC6`>vLWMkS$Ib+>YWK zS4sz|DNG-`k?F_%o#h=hwMdQfgneN`k$KYm*u^$y-Ee_b9xF4W3}3$fy^Mf3NF9J3Tnigod0_d=ih!po1I%?S4>0VLh z_CFm1yew-+ik_1SO!(_C$ndDQ`%m7cB|wDrKPu99cdn - 🤝

- 期待您的支持 + + AxisNow + +
+ AxisNow｜自建私有部署 CDN｜订阅式高防 CDN +
+ 自主可控、灵活组合的 CDN 网络 +

+ 🌐 AxisNow 🤝

From 8f1147c800c908bbc8051317fd8ac77eff723710 Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 2 Sep 2026 13:30:46 +0000 Subject: [PATCH 03/48] chore(deps): update direct dependencies --- go.mod | 42 ++++++++++++++---------------- go.sum | 82 ++++++++++++++++++++++++++++------------------------------ 2 files changed, 60 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index 65a524cd..9133eb50 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/nezhahq/nezha -go 1.26.5 +go 1.26.6 require ( - github.com/VictoriaMetrics/VictoriaMetrics v1.148.0 + github.com/VictoriaMetrics/VictoriaMetrics v1.151.0 github.com/appleboy/gin-jwt/v2 v2.10.3 github.com/dustinkirkland/golang-petname v0.0.0-20260215035315-f0c533e9ce9b github.com/gin-contrib/pprof v1.5.4 @@ -14,16 +14,16 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-uuid v1.0.3 github.com/jinzhu/copier v0.4.0 - github.com/knadh/koanf/maps v0.1.2 + github.com/knadh/koanf/maps v0.1.3 github.com/knadh/koanf/providers/env v1.1.0 github.com/knadh/koanf/providers/file v1.2.1 - github.com/knadh/koanf/v2 v2.3.5 + github.com/knadh/koanf/v2 v2.3.6 github.com/leonelquinteros/gotext v1.7.2 github.com/libdns/cloudflare v0.2.2 github.com/libdns/he v1.2.2 github.com/libdns/libdns v1.1.1 - github.com/mattn/go-sqlite3 v1.14.49 - github.com/miekg/dns v1.1.72 + github.com/mattn/go-sqlite3 v1.14.50 + github.com/miekg/dns v1.1.73 github.com/modelcontextprotocol/go-sdk v1.7.0 github.com/nezhahq/libdns-tencentcloud v0.0.0-20260628095405-2ab294ec675b github.com/ory/graceful v0.2.0 @@ -31,20 +31,20 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/robfig/cron/v3 v3.0.1 github.com/sqids/sqids-go v0.4.1 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.1 github.com/swaggo/swag v1.16.6 github.com/tidwall/gjson v1.19.0 - golang.org/x/crypto v0.54.0 - golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 - golang.org/x/mod v0.38.0 - golang.org/x/net v0.57.0 + golang.org/x/crypto v0.55.0 + golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa + golang.org/x/mod v0.40.0 + golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 - google.golang.org/grpc v1.83.0 - google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af + google.golang.org/grpc v1.83.2 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.2 @@ -62,7 +62,6 @@ require ( github.com/bytedance/sonic/loader v0.5.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.7 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.1 // indirect @@ -85,17 +84,16 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/compress v1.19.1 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-isatty v0.0.23 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/pelletier/go-toml/v2 v2.3.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect @@ -108,17 +106,17 @@ require ( github.com/ugorji/go/codec v1.3.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fastrand v1.1.0 // indirect - github.com/valyala/gozstd v1.24.0 // indirect + github.com/valyala/gozstd v1.25.0 // indirect github.com/valyala/histogram v1.2.0 // indirect github.com/valyala/quicktemplate v1.8.0 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.27.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.48.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 // indirect + golang.org/x/tools v0.49.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect ) diff --git a/go.sum b/go.sum index 87ad3ce6..36264133 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/VictoriaMetrics/VictoriaMetrics v1.148.0 h1:qfsMLjvkNnIIE3w3o7Sf1ejpYAE3pGwqAz4OqKNqZYI= -github.com/VictoriaMetrics/VictoriaMetrics v1.148.0/go.mod h1:/grcppu6T0Y1k8z6e7ACb43sYUKGkvQ4xVVdCANK/d0= +github.com/VictoriaMetrics/VictoriaMetrics v1.151.0 h1:Svzb/7gPYbzmQnvLjab8h1Jl9olB76mokDE7ykvF/ro= +github.com/VictoriaMetrics/VictoriaMetrics v1.151.0/go.mod h1:Mp7qEBDOXwP50gC93cF8wJmaGjTFSJP2WA6dRtMI7qU= github.com/VictoriaMetrics/easyproto v1.2.0 h1:FJT9uNXA2isppFuJErbLqD306KoFlehl7Wn2dg/6oIE= github.com/VictoriaMetrics/easyproto v1.2.0/go.mod h1:QlGlzaJnDfFd8Lk6Ci/fuLxfTo3/GThPs2KH23mv710= github.com/VictoriaMetrics/fastcache v1.13.3 h1:rBabE0iIxcqKEMCwUmwHZ9dgEqXerg8FRbRDUvC7OVc= @@ -44,8 +44,8 @@ github.com/gin-contrib/sse v1.1.1 h1:uGYpNwTacv5R68bSGMapo62iLTRa9l5zxGCps4hK6ko github.com/gin-contrib/sse v1.1.1/go.mod h1:QXzuVkA0YO7o/gun03UI1Q+FTI8ZV/n5t03kIQAI89s= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.23.1 h1:1HBACs7XIwR2RcmItfdSFlALhGbe6S92p0ry4d1GWg4= @@ -116,18 +116,18 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= -github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= -github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/maps v0.1.3 h1:P1z7EvTqdFBrPYbzSvorvrpib+sjkUMxf0FVvA5NKK4= +github.com/knadh/koanf/maps v0.1.3/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc= github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY= github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= -github.com/knadh/koanf/v2 v2.3.5 h1:2dXJUYaKGm4SGYeoAtBviq9+02JZo/pxQ2ssOd60rJg= -github.com/knadh/koanf/v2 v2.3.5/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= +github.com/knadh/koanf/v2 v2.3.6 h1:JoQPSJmvS4aP0xNc8xMDr5tcrkSEInL23/Il7pITAKo= +github.com/knadh/koanf/v2 v2.3.6/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -142,12 +142,12 @@ github.com/libdns/he v1.2.2 h1:4PMGYsT3g7uFcHIaEkaQK4dENEorUuetPClGCsqi5/s= github.com/libdns/he v1.2.2/go.mod h1:dHvv/0lMlwUXpMvnM63kZEpWYTvhDyK1dduXNVUCUYw= github.com/libdns/libdns v1.1.1 h1:wPrHrXILoSHKWJKGd0EiAVmiJbFShguILTg9leS/P/U= github.com/libdns/libdns v1.1.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= -github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= -github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-sqlite3 v1.14.49 h1:B8jBHC3xhxZgxztrgruTuLucebnULQnx4W7cF7SAE9w= -github.com/mattn/go-sqlite3 v1.14.49/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= -github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= -github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/mattn/go-isatty v0.0.23 h1:cYwCQTQf3HB6xUC+BtyCLZNr7IzbOmoZbmssVNzSyiQ= +github.com/mattn/go-isatty v0.0.23/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-sqlite3 v1.14.50 h1:dmdFvo1XG4MPzA4IkAmE9upVz/Nj31uRoM5+jC8hYbY= +github.com/mattn/go-sqlite3 v1.14.50/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE= +github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -173,8 +173,6 @@ github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.1 h1:0Gmua0HW1Tv7ANR7hUYwRyD0MG5OJfgvYSZasGZzBic= @@ -198,8 +196,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.1 h1:Ri06G4gc9N4t4k8hekMigJ9zKTFSlqj/9paAQCQs7cY= @@ -225,8 +223,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= -github.com/valyala/gozstd v1.24.0 h1:M/9L3h7bVwbj2gZwrmuoaxzwVrmBUvos2jG9cZtuhlc= -github.com/valyala/gozstd v1.24.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/valyala/gozstd v1.25.0 h1:7gS7+5zwidZT1BFQqGPAPII8ekZ3tvYTp5IvOEWC34Y= +github.com/valyala/gozstd v1.25.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= github.com/valyala/quicktemplate v1.8.0 h1:zU0tjbIqTRgKQzFY1L42zq0qR3eh4WoQQdIdqCysW5k= @@ -254,25 +252,25 @@ go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/arch v0.27.0 h1:0WNVcR8u9yFz8j5FvdHpgwNp3FS5U4guYdzHwEiGjoU= golang.org/x/arch v0.27.0/go.mod h1:0X+GdSIP+kL5wPmpK7sdkEVTt2XoYP0cSjQSbZBwOi8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= -golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -294,24 +292,24 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 h1:9HZDLIdYBJXAnaFOr9WHrKVycfpY+75s9HGadC0305A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= -google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= -google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 19daf4ad3cf724808fd6f473c07df96faca384e1 Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 2 Sep 2026 13:30:46 +0000 Subject: [PATCH 04/48] fix(alert): prevent persisted rule crash loops --- cmd/dashboard/controller/alertrule.go | 12 ++ .../controller/alertrule_security_test.go | 67 ++++++++++ model/alertrule.go | 79 +++++++++-- model/alertrule_security_test.go | 123 ++++++++++++++++++ model/alertrule_test.go | 20 ++- model/rule.go | 89 +++++++++++-- service/singleton/alertsentinel.go | 110 +++++++++------- service/singleton/alertsentinel_test.go | 50 +++++++ 8 files changed, 479 insertions(+), 71 deletions(-) create mode 100644 cmd/dashboard/controller/alertrule_security_test.go create mode 100644 model/alertrule_security_test.go diff --git a/cmd/dashboard/controller/alertrule.go b/cmd/dashboard/controller/alertrule.go index 3c54f4a2..85021a47 100644 --- a/cmd/dashboard/controller/alertrule.go +++ b/cmd/dashboard/controller/alertrule.go @@ -172,6 +172,12 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { } if len(r.Rules) > 0 { for _, rule := range r.Rules { + if rule == nil { + return singleton.Localizer.ErrorT("rule is not set") + } + if !rule.IsSupportedType() { + return singleton.Localizer.ErrorT("unsupported rule type") + } switch rule.Cover { case model.RuleCoverAll, model.RuleCoverIgnoreAll: default: @@ -182,10 +188,16 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { if rule.Duration < 3 { return singleton.Localizer.ErrorT("duration need to be at least 3") } + if rule.Duration > model.MaxAlertRuleDuration { + return singleton.Localizer.ErrorT("duration is too large") + } } else { if rule.CycleInterval < 1 { return singleton.Localizer.ErrorT("cycle_interval need to be at least 1") } + if rule.CycleInterval > model.MaxAlertRuleCycleInterval { + return singleton.Localizer.ErrorT("cycle_interval is too large") + } if rule.CycleStart == nil { return singleton.Localizer.ErrorT("cycle_start is not set") } diff --git a/cmd/dashboard/controller/alertrule_security_test.go b/cmd/dashboard/controller/alertrule_security_test.go new file mode 100644 index 00000000..7d68d388 --- /dev/null +++ b/cmd/dashboard/controller/alertrule_security_test.go @@ -0,0 +1,67 @@ +package controller + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestValidateRuleRejectsCrashPayloadsFromMember(t *testing.T) { + ctx := newMemberValidationContext(t) + cycleStart := time.Now().Add(-time.Hour) + tests := []struct { + name string + rule *model.Rule + }{ + { + name: "unknown rule type", + rule: &model.Rule{Type: "attacker_controlled", Duration: 3, Cover: model.RuleCoverAll}, + }, + { + name: "duration overflows int", + rule: &model.Rule{Type: "offline", Duration: math.MaxUint64, Cover: model.RuleCoverAll}, + }, + { + name: "cycle interval overflows int", + rule: &model.Rule{ + Type: "transfer_in_cycle", + CycleStart: &cycleStart, + CycleInterval: math.MaxUint64, + Cover: model.RuleCoverAll, + }, + }, + { + name: "nil rule", + rule: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + alert := &model.AlertRule{ + Common: model.Common{UserID: 200}, + Name: "security regression", + Rules: []*model.Rule{test.rule}, + } + require.Error(t, validateRule(ctx, alert)) + }) + } +} + +func TestValidateRuleAcceptsSafeDurationBoundary(t *testing.T) { + ctx := newMemberValidationContext(t) + alert := &model.AlertRule{ + Common: model.Common{UserID: 200}, + Name: "duration boundary", + Rules: []*model.Rule{{ + Type: "offline", + Duration: model.MaxAlertRuleDuration, + Cover: model.RuleCoverAll, + }}, + } + require.NoError(t, validateRule(ctx, alert)) +} diff --git a/model/alertrule.go b/model/alertrule.go index 37e734c6..886ea0ce 100644 --- a/model/alertrule.go +++ b/model/alertrule.go @@ -64,6 +64,36 @@ func (r *AlertRule) Enabled() bool { return r.Enable != nil && *r.Enable } +// IsSafeToEvaluate validates persisted rule data before it reaches the alert +// goroutine. API validation protects new writes; this second boundary protects +// upgrades from malformed or deliberately poisoned rows already in the DB. +func (r *AlertRule) IsSafeToEvaluate() bool { + if r == nil || len(r.Rules) == 0 { + return false + } + for _, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + return false + } + switch rule.Cover { + case RuleCoverAll, RuleCoverIgnoreAll: + default: + return false + } + if rule.IsTransferDurationRule() { + if !rule.HasSafeCycleConfiguration() { + return false + } + continue + } + duration, ok := rule.DurationInt() + if !ok || duration < 3 { + return false + } + } + return true +} + // HasPermission extends the default owner/admin check with PAT // server_ids whitelist enforcement. AlertRule.Snapshot fans out across // every owner-visible server filtered only by Rule.Ignore semantics @@ -125,6 +155,12 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser point := make([]bool, len(r.Rules)) for i, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + // Invalid persisted rules are ignored instead of being interpreted as + // a failed condition or allowed to panic the sentinel. + point[i] = true + continue + } point[i] = rule.Snapshot(cycleTransferStats, server, db) } return point @@ -134,10 +170,14 @@ func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Ser func (r *AlertRule) Check(points [][]bool) (int, bool) { var hasPassedRule bool durations := make([]int, len(r.Rules)) + validRules := 0 for ruleIndex, rule := range r.Rules { - duration := int(rule.Duration) + if rule == nil || !rule.IsSupportedType() { + continue + } if rule.IsTransferDurationRule() { + validRules++ // 循环区间流量报警 if durations[ruleIndex] < 1 { durations[ruleIndex] = 1 @@ -146,10 +186,21 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { continue } // 只要最后一次检查超出了规则范围 就认为检查未通过 - if len(points) > 0 && points[len(points)-1][ruleIndex] { - hasPassedRule = true + if len(points) > 0 { + lastPoint := points[len(points)-1] + if ruleIndex >= len(lastPoint) || lastPoint[ruleIndex] { + hasPassedRule = true + } } - } else if rule.IsOfflineRule() { + continue + } + + duration, ok := rule.DurationInt() + if !ok || duration <= 0 { + continue + } + validRules++ + if rule.IsOfflineRule() { // 离线报警，检查直到最后一次在线的离线采样点是否大于 duration if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule { continue @@ -157,7 +208,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { var fail int for _, point := range slices.Backward(points[len(points)-duration:]) { fail++ - if point[ruleIndex] { + if ruleIndex >= len(point) || point[ruleIndex] { hasPassedRule = true break } @@ -168,11 +219,7 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { // 常规报警 // duration<=0 是无意义的规则（持续 0 秒）：直接跳过该规则， // 既不污染 hasPassedRule（否则会连带跳过同一 alert 里其它有效 - // 规则），也避免下方 fail*100/total 在 total=0 时整数除零 panic - // —— checkStatus 无 recover，一次 panic 会拖垮整个告警 goroutine。 - if duration <= 0 { - continue - } + // 规则），也避免下方百分比计算在 total=0 时除零。 if hasPassedRule = boundCheck(len(points), duration, hasPassedRule); hasPassedRule { continue } @@ -181,17 +228,20 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { } total, fail := duration, 0 for timeTick := len(points) - duration; timeTick < len(points); timeTick++ { - if !points[timeTick][ruleIndex] { + if ruleIndex >= len(points[timeTick]) || !points[timeTick][ruleIndex] { fail++ } } // 当70%以上的采样点未通过规则判断时 才认为当前检查未通过 - if fail*100/total <= 70 { + if float64(fail)*100/float64(total) <= 70 { hasPassedRule = true } } } + if validRules == 0 { + return 0, true + } // 仅当所有检查均未通过时 才触发告警 return slices.Max(durations), hasPassedRule } @@ -205,10 +255,13 @@ func (r *AlertRule) Check(points [][]bool) (int, bool) { func (r *AlertRule) RetentionWindow() int { window := 0 for _, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + continue + } var need int if rule.IsTransferDurationRule() { need = 1 - } else if d := int(rule.Duration); d > 0 { + } else if d, ok := rule.DurationInt(); ok && d > 0 { need = d } if need > window { diff --git a/model/alertrule_security_test.go b/model/alertrule_security_test.go new file mode 100644 index 00000000..8ded87b5 --- /dev/null +++ b/model/alertrule_security_test.go @@ -0,0 +1,123 @@ +package model + +import ( + "math" + "testing" + "time" +) + +func TestRuleSnapshotIgnoresEmptyMetricSamples(t *testing.T) { + tests := []struct { + name string + ruleType string + hostState *HostState + }{ + { + name: "GPU list is empty", + ruleType: "gpu_max", + hostState: &HostState{}, + }, + { + name: "all temperature samples are filtered", + ruleType: "temperature_max", + hostState: &HostState{Temperatures: []SensorTemperature{ + {Name: "coretemp", Temperature: 0}, + }}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := &Server{Common: Common{ID: 1}, State: test.hostState, Host: &Host{}} + rule := &Rule{Type: test.ruleType, Max: 1, Duration: 3, Cover: RuleCoverAll} + + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Snapshot panicked for missing metric data: %v", recovered) + } + }() + if passed := rule.Snapshot(nil, server, nil); !passed { + t.Fatal("missing metric data must be ignored instead of treated as a failed threshold") + } + }) + } +} + +func TestAlertRuleCheckRejectsOverflowingPersistedDuration(t *testing.T) { + rule := &AlertRule{Rules: []*Rule{{ + Type: "offline", + Duration: math.MaxUint64, + Cover: RuleCoverAll, + }}} + + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("Check panicked after uint64-to-int overflow: %v", recovered) + } + }() + duration, passed := rule.Check([][]bool{{false}}) + if duration != 0 || !passed { + t.Fatalf("invalid persisted duration must be ignored safely, got duration=%d passed=%v", duration, passed) + } + if window := rule.RetentionWindow(); window != 0 { + t.Fatalf("invalid persisted duration must not create a retention window, got %d", window) + } +} + +func TestAlertRulePersistedDataSafety(t *testing.T) { + cycleStart := time.Now().Add(-time.Hour) + tests := []struct { + name string + rule *AlertRule + want bool + }{ + { + name: "normal rule", + rule: &AlertRule{Rules: []*Rule{{ + Type: "cpu", Duration: 3, Cover: RuleCoverAll, + }}}, + want: true, + }, + { + name: "normal cycle rule", + rule: &AlertRule{Rules: []*Rule{{ + Type: "transfer_in_cycle", CycleStart: &cycleStart, CycleInterval: 1, Cover: RuleCoverAll, + }}}, + want: true, + }, + { + name: "unknown type", + rule: &AlertRule{Rules: []*Rule{{ + Type: "attacker_controlled", Duration: 3, Cover: RuleCoverAll, + }}}, + }, + { + name: "overflowing duration", + rule: &AlertRule{Rules: []*Rule{{ + Type: "offline", Duration: math.MaxUint64, Cover: RuleCoverAll, + }}}, + }, + { + name: "cycle without start", + rule: &AlertRule{Rules: []*Rule{{ + Type: "transfer_in_cycle", CycleInterval: 1, Cover: RuleCoverAll, + }}}, + }, + { + name: "nil rule", + rule: &AlertRule{Rules: []*Rule{nil}}, + }, + { + name: "empty rule list", + rule: &AlertRule{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.rule.IsSafeToEvaluate(); got != test.want { + t.Fatalf("IsSafeToEvaluate()=%v, want %v", got, test.want) + } + }) + } +} diff --git a/model/alertrule_test.go b/model/alertrule_test.go index 72cd18df..81ae7b5b 100644 --- a/model/alertrule_test.go +++ b/model/alertrule_test.go @@ -26,7 +26,7 @@ func testCycleRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { - Type: "_cycle", + Type: "transfer_in_cycle", }, }, }, @@ -39,7 +39,7 @@ func testCycleRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { - Type: "_cycle", + Type: "transfer_in_cycle", }, }, }, @@ -130,6 +130,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -143,6 +144,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -156,6 +158,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -169,6 +172,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -182,6 +186,7 @@ func testGeneralRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, }, @@ -210,6 +215,7 @@ func testCombinedRules(t *testing.T) { Duration: 10, }, { + Type: "cpu", Duration: 10, }, }, @@ -227,6 +233,7 @@ func testCombinedRules(t *testing.T) { Duration: 10, }, { + Type: "cpu", Duration: 10, }, }, @@ -240,6 +247,7 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { @@ -257,9 +265,11 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { + Type: "memory", Duration: 30, }, }, @@ -273,9 +283,11 @@ func testCombinedRules(t *testing.T) { rule: &AlertRule{ Rules: []*Rule{ { + Type: "cpu", Duration: 10, }, { + Type: "memory", Duration: 30, }, }, @@ -416,7 +428,7 @@ func TestAlertRule_RetentionWindow(t *testing.T) { {"zero duration only", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}}}, 0}, {"mixed picks max", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 0}, {Type: "cpu", Duration: 7}}}, 7}, {"offline keeps Duration", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 30}}}, 30}, - {"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}}}, 1}, + {"cycle looks back one", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}}}, 1}, } for _, c := range cases { if got := c.rule.RetentionWindow(); got != c.want { @@ -469,7 +481,7 @@ func TestAlertRule_CombinedRuleAccumulatesSamples(t *testing.T) { }{ {"general3+general10", &AlertRule{Rules: []*Rule{{Type: "cpu", Duration: 3}, {Type: "memory", Duration: 10}}}, []bool{false, false}, 9, 10}, {"offline5+general10", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 5}, {Type: "cpu", Duration: 10}}}, []bool{false, false}, 9, 10}, - {"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "net_in_speed_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8}, + {"transfer+general8", &AlertRule{Rules: []*Rule{{Type: "transfer_in_cycle"}, {Type: "cpu", Duration: 8}}}, []bool{false, false}, 7, 8}, {"offline3+offline12", &AlertRule{Rules: []*Rule{{Type: "offline", Duration: 3}, {Type: "offline", Duration: 12}}}, []bool{false, false}, 11, 12}, } for _, c := range cases { diff --git a/model/rule.go b/model/rule.go index e069eef5..2d492b10 100644 --- a/model/rule.go +++ b/model/rule.go @@ -15,12 +15,23 @@ const ( RuleCoverIgnoreAll ) +// MaxAlertRuleDuration is the largest duration that can be converted to int +// on every architecture supported by Go. Alert rules are persisted as uint64, +// but their sampling windows use int indexes; keeping the public bound at +// MaxInt32 prevents a crafted value from wrapping during that conversion. +const MaxAlertRuleDuration uint64 = 1<<31 - 1 + +// MaxAlertRuleCycleInterval also leaves room for the largest integer +// multiplication performed by the calendar helpers (weeks become 7*interval) +// on 32-bit targets. +const MaxAlertRuleCycleInterval uint64 = (1<<31 - 1) / 7 + type NResult struct { N uint64 } type Rule struct { - // 指标类型，cpu、memory、swap、disk、net_in_speed、net_out_speed + // 指标类型，cpu、gpu/gpu_max、memory、swap、disk、net_in_speed、net_out_speed // net_all_speed、transfer_in、transfer_out、transfer_all、offline // transfer_in_cycle、transfer_out_cycle、transfer_all_cycle Type string `json:"type"` @@ -45,8 +56,52 @@ func percentage(used, total uint64) float64 { return float64(used) * 100 / float64(total) } +// IsSupportedType reports whether the rule type has an evaluator. Keep this +// list in lockstep with Snapshot's switch so untrusted strings cannot enter the +// persisted alert pipeline and silently acquire fallback semantics. +func (u *Rule) IsSupportedType() bool { + if u == nil { + return false + } + switch u.Type { + case "cpu", "gpu", "gpu_max", "memory", "swap", "disk", + "net_in_speed", "net_out_speed", "net_all_speed", + "transfer_in", "transfer_out", "transfer_all", "offline", + "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle", + "load1", "load5", "load15", "tcp_conn_count", "udp_conn_count", + "process_count", "temperature_max": + return true + default: + return false + } +} + +// DurationInt converts the persisted duration without allowing uint64-to-int +// truncation. Callers must handle ok=false as an invalid rule. +func (u *Rule) DurationInt() (duration int, ok bool) { + if u == nil || u.Duration > MaxAlertRuleDuration { + return 0, false + } + return int(u.Duration), true +} + +// HasSafeCycleConfiguration guards every value that the cycle evaluator later +// converts or dereferences. It intentionally does not reject an empty unit, +// which is the documented legacy spelling for hours. +func (u *Rule) HasSafeCycleConfiguration() bool { + return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil && + u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval +} + // Snapshot 未通过规则返回 false, 通过返回 true func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool { + if u == nil || server == nil || !u.IsSupportedType() { + return true + } + if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) { + return true + } + // 监控全部但是排除了此服务器 if u.Cover == RuleCoverAll && u.Ignore[server.ID] { return true @@ -71,7 +126,10 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, switch u.Type { case "cpu": src = float64(state.CPU) - case "gpu_max": + case "gpu", "gpu_max": + if len(state.GPU) == 0 { + return true + } src = slices.Max(state.GPU) case "memory": if runtime.Host == nil { @@ -141,14 +199,17 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, src = float64(state.ProcessCount) case "temperature_max": var temp []float64 - if state.Temperatures != nil { - for _, tempStat := range state.Temperatures { - if tempStat.Temperature != 0 { - temp = append(temp, tempStat.Temperature) - } + for _, tempStat := range state.Temperatures { + if tempStat.Temperature != 0 { + temp = append(temp, tempStat.Temperature) } - src = slices.Max(temp) } + if len(temp) == 0 { + return true + } + src = slices.Max(temp) + default: + return true } // 循环区间流量检测 · 更新下次需要检测时间 @@ -187,11 +248,19 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, // IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true func (u *Rule) IsTransferDurationRule() bool { - return strings.HasSuffix(u.Type, "_cycle") + if u == nil { + return false + } + switch u.Type { + case "transfer_in_cycle", "transfer_out_cycle", "transfer_all_cycle": + return true + default: + return false + } } func (u *Rule) IsOfflineRule() bool { - return u.Type == "offline" + return u != nil && u.Type == "offline" } // GetTransferDurationStart 获取周期流量的起始时间 diff --git a/service/singleton/alertsentinel.go b/service/singleton/alertsentinel.go index b17aeed8..82f56846 100644 --- a/service/singleton/alertsentinel.go +++ b/service/singleton/alertsentinel.go @@ -33,7 +33,7 @@ var ( // addCycleTransferStatsInfo 向AlertsCycleTransferStatsStore中添加周期流量报警统计信息 func addCycleTransferStatsInfo(alert *model.AlertRule) { - if !alert.Enabled() { + if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() { return } for _, rule := range alert.Rules { @@ -67,8 +67,16 @@ func AlertSentinelStart() { panic(err) } for _, alert := range Alerts { + if alert == nil { + log.Printf("NEZHA>> Skipping invalid nil alert rule loaded from database") + continue + } alertsStore[alert.ID] = make(map[uint64][][]bool) alertsPrevState[alert.ID] = make(map[uint64]uint8) + if !alert.IsSafeToEvaluate() { + log.Printf("NEZHA>> Skipping invalid alert rule %d loaded from database", alert.ID) + continue + } addCycleTransferStatsInfo(alert) } AlertsLock.Unlock() @@ -136,7 +144,7 @@ func checkStatus() { for _, alert := range Alerts { // 跳过未启用 - if !alert.Enabled() { + if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() { continue } for _, server := range m { @@ -152,48 +160,62 @@ func checkStatus() { if alert.UserID != server.GetUserID() && !role.IsAdmin() { continue } - alertsStore[alert.ID][server.ID] = append(alertsStore[alert. - ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB)) - // 发送通知，分为触发报警和恢复通知 - _, passed := alert.Check(alertsStore[alert.ID][server.ID]) - // 保存当前服务器状态信息 - curServer := model.Server{} - copier.Copy(&curServer, server) - - // 本次未通过检查 - if !passed { - // 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况） - if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail { - alertsPrevState[alert.ID][server.ID] = _RuleCheckFail - message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"), - server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) - go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer) - // 清除恢复通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID)) - } - } else { - // 本次通过检查但上一次的状态为失败，则发送恢复通知 - if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail { - message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"), - server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) - go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer) - // 清除失败通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID)) - } - alertsPrevState[alert.ID][server.ID] = _RuleCheckPass - } - // 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值）， - // 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史 - // （例如全部 Duration<=0），此时清空采样避免切片无限增长。 - window := alert.RetentionWindow() - samples := alertsStore[alert.ID][server.ID] - if window <= 0 { - alertsStore[alert.ID][server.ID] = samples[:0] - } else if window < len(samples) { - alertsStore[alert.ID][server.ID] = samples[len(samples)-window:] - } + checkStatusForServer(alert, server) } } } + +// checkStatusForServer isolates each alert/server evaluation. Input validation +// and model-level guards handle known malformed states; this recovery boundary +// ensures an unexpected evaluator panic cannot terminate the dashboard process +// or prevent unrelated servers from being checked on the same tick. +func checkStatusForServer(alert *model.AlertRule, server *model.Server) { + defer func() { + if recovered := recover(); recovered != nil { + log.Printf("NEZHA>> Recovered panic evaluating alert rule %d for server %d: %v", alert.ID, server.ID, recovered) + } + }() + + alertsStore[alert.ID][server.ID] = append(alertsStore[alert. + ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB)) + // 发送通知，分为触发报警和恢复通知 + _, passed := alert.Check(alertsStore[alert.ID][server.ID]) + // 保存当前服务器状态信息 + curServer := model.Server{} + copier.Copy(&curServer, server) + + // 本次未通过检查 + if !passed { + // 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况） + if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail { + alertsPrevState[alert.ID][server.ID] = _RuleCheckFail + message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"), + server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) + go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID) + go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer) + // 清除恢复通知的静音缓存 + NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID)) + } + } else { + // 本次通过检查但上一次的状态为失败，则发送恢复通知 + if alertsPrevState[alert.ID][server.ID] == _RuleCheckFail { + message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"), + server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) + go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID) + go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer) + // 清除失败通知的静音缓存 + NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID)) + } + alertsPrevState[alert.ID][server.ID] = _RuleCheckPass + } + // 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值）， + // 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史 + // （例如全部 Duration<=0），此时清空采样避免切片无限增长。 + window := alert.RetentionWindow() + samples := alertsStore[alert.ID][server.ID] + if window <= 0 { + alertsStore[alert.ID][server.ID] = samples[:0] + } else if window < len(samples) { + alertsStore[alert.ID][server.ID] = samples[len(samples)-window:] + } +} diff --git a/service/singleton/alertsentinel_test.go b/service/singleton/alertsentinel_test.go index 1d290523..e227e57b 100644 --- a/service/singleton/alertsentinel_test.go +++ b/service/singleton/alertsentinel_test.go @@ -117,3 +117,53 @@ func TestCheckStatus_SampleMemoryBounded(t *testing.T) { t.Fatalf("sample capacity grew unbounded: peak cap %d exceeds 4x window %d", peakCap, duration*4) } } + +func TestCheckStatusForServerRecoversUnexpectedEvaluatorPanic(t *testing.T) { + originalStore := alertsStore + originalPrevState := alertsPrevState + originalCycleStore := AlertsCycleTransferStatsStore + alertsStore = map[uint64]map[uint64][][]bool{} + alertsPrevState = map[uint64]map[uint64]uint8{} + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { + alertsStore = originalStore + alertsPrevState = originalPrevState + AlertsCycleTransferStatsStore = originalCycleStore + }) + + enabled := true + alert := &model.AlertRule{ + Common: model.Common{ID: 77, UserID: 2}, + Enable: &enabled, + Rules: []*model.Rule{{Type: "cpu", Duration: 3, Cover: model.RuleCoverAll}}, + } + server := &model.Server{Common: model.Common{ID: 1, UserID: 2}} + model.InitServer(server) + + // The deliberately missing alertsStore[alert.ID] map panics at assignment + // after evaluation. The per-alert/server recovery boundary must contain it. + checkStatusForServer(alert, server) +} + +func TestAddCycleTransferStatsSkipsPoisonedPersistedRule(t *testing.T) { + original := AlertsCycleTransferStatsStore + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { AlertsCycleTransferStatsStore = original }) + + enabled := true + alert := &model.AlertRule{ + Common: model.Common{ID: 88}, + Enable: &enabled, + Rules: []*model.Rule{{ + Type: "transfer_in_cycle", + CycleInterval: 1, + Cover: model.RuleCoverAll, + // CycleStart intentionally nil, as it may be in a poisoned legacy DB row. + }}, + } + + addCycleTransferStatsInfo(alert) + if _, exists := AlertsCycleTransferStatsStore[alert.ID]; exists { + t.Fatal("invalid persisted cycle rule must not be scheduled") + } +} From d6d5b9804d3d78574a670ba67c7d36fba4bcc912 Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 2 Sep 2026 13:30:46 +0000 Subject: [PATCH 05/48] chore(frontend-templates): update frontend versions --- service/singleton/frontend-templates.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/singleton/frontend-templates.yaml b/service/singleton/frontend-templates.yaml index 050c0f1b..40e54b1f 100644 --- a/service/singleton/frontend-templates.yaml +++ b/service/singleton/frontend-templates.yaml @@ -2,7 +2,7 @@ name: "OfficialAdmin" repository: "https://github.com/nezhahq/admin-frontend" author: "nezhahq" - version: "v2.3.4" + version: "v2.3.5" is_admin: true is_official: true - path: "user-dist" From f2e8d03667db9ba4f5dd68a6ffbf14e4a886219b Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 2 Sep 2026 13:35:11 +0000 Subject: [PATCH 06/48] ci: pin Go 1.26.6 --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 6 +++--- .../internal/workflowpolicy/quality_workflow_test.go | 2 +- .../internal/workflowpolicy/stress_workflow_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ba329a9c..babe9aa7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "1.26.6" - name: generate swagger docs run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9cd8d4c..e3bf0d6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: persist-credentials: false - uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "1.26.6" cache: false - name: Generate Swagger docs @@ -57,7 +57,7 @@ jobs: persist-credentials: false - uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "1.26.6" cache: false - name: Generate Swagger docs @@ -107,7 +107,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.26.x" + go-version: "1.26.6" cache: false - name: Prepare Dashboard build inputs working-directory: nezha diff --git a/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go index 8192e58a..c85cc043 100644 --- a/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go +++ b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go @@ -169,7 +169,7 @@ func requireCheckoutAndSetupGo(t *testing.T, steps []qualityStep) { require.False(t, *checkout.With.PersistCredentials) setupGo := steps[1] require.Equal(t, "actions/setup-go@v7", setupGo.Uses) - require.Equal(t, "1.26.x", setupGo.With.GoVersion) + require.Equal(t, "1.26.6", setupGo.With.GoVersion) require.NotNil(t, setupGo.With.Cache) require.False(t, *setupGo.With.Cache) } diff --git a/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go index 6d28c0d9..f06744c0 100644 --- a/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go +++ b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go @@ -57,7 +57,7 @@ func TestPolicy_NezhaStressWorkflowRunsCrossRepositoryTest(t *testing.T) { setupGo := stressJob.stepNamed(t, "Set up Go") require.Equal(t, "actions/setup-go@v7", setupGo.Uses) - require.Equal(t, "1.26.x", setupGo.With.GoVersion) + require.Equal(t, "1.26.6", setupGo.With.GoVersion) require.False(t, *setupGo.With.Cache) prepareDashboardInputs := stressJob.stepNamed(t, "Prepare Dashboard build inputs") From eb48a178f80e0513549f0ae539d4a2dcdee3ac6d Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 2 Sep 2026 13:45:05 +0000 Subject: [PATCH 07/48] ci: use latest compatible Go patch --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 6 +++--- .../internal/workflowpolicy/quality_workflow_test.go | 2 +- .../internal/workflowpolicy/stress_workflow_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index babe9aa7..54363f7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.26.6" + go-version: "^1.26.6" - name: generate swagger docs run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3bf0d6a..289b62d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: persist-credentials: false - uses: actions/setup-go@v7 with: - go-version: "1.26.6" + go-version: "^1.26.6" cache: false - name: Generate Swagger docs @@ -57,7 +57,7 @@ jobs: persist-credentials: false - uses: actions/setup-go@v7 with: - go-version: "1.26.6" + go-version: "^1.26.6" cache: false - name: Generate Swagger docs @@ -107,7 +107,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.26.6" + go-version: "^1.26.6" cache: false - name: Prepare Dashboard build inputs working-directory: nezha diff --git a/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go index c85cc043..48b6e176 100644 --- a/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go +++ b/integration/agentcompat/internal/workflowpolicy/quality_workflow_test.go @@ -169,7 +169,7 @@ func requireCheckoutAndSetupGo(t *testing.T, steps []qualityStep) { require.False(t, *checkout.With.PersistCredentials) setupGo := steps[1] require.Equal(t, "actions/setup-go@v7", setupGo.Uses) - require.Equal(t, "1.26.6", setupGo.With.GoVersion) + require.Equal(t, "^1.26.6", setupGo.With.GoVersion) require.NotNil(t, setupGo.With.Cache) require.False(t, *setupGo.With.Cache) } diff --git a/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go index f06744c0..0763522e 100644 --- a/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go +++ b/integration/agentcompat/internal/workflowpolicy/stress_workflow_test.go @@ -57,7 +57,7 @@ func TestPolicy_NezhaStressWorkflowRunsCrossRepositoryTest(t *testing.T) { setupGo := stressJob.stepNamed(t, "Set up Go") require.Equal(t, "actions/setup-go@v7", setupGo.Uses) - require.Equal(t, "1.26.6", setupGo.With.GoVersion) + require.Equal(t, "^1.26.6", setupGo.With.GoVersion) require.False(t, *setupGo.With.Cache) prepareDashboardInputs := stressJob.stepNamed(t, "Prepare Dashboard build inputs") From c684388153f61601ee7cb740a7513d80429c5ec9 Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:07:23 +0800 Subject: [PATCH 08/48] Improve: Delete the corresponding DNS record if the IPv4/IPv6 address is empty. --- pkg/ddns/ddns.go | 86 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/pkg/ddns/ddns.go b/pkg/ddns/ddns.go index 7759df54..77779c0b 100644 --- a/pkg/ddns/ddns.go +++ b/pkg/ddns/ddns.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "net/netip" + "strings" "time" "github.com/libdns/libdns" @@ -54,10 +55,13 @@ func (provider *Provider) updateDomain(ctx context.Context, domain string) error return err } - // 独立处理 IPv4 更新 + // 独立处理 IPv4 更新或删除 if *provider.DDNSProfile.EnableIPv4 { if provider.IPAddrs.IPv4Addr == "" { - log.Printf("NEZHA>> Skip IPv4 update for domain %s: IPv4 address is empty", domain) + log.Printf("NEZHA>> IPv4 address is empty for domain %s, deleting A record...", domain) + if err = provider.deleteDomainRecord(ctx, "A"); err != nil { + return fmt.Errorf("failed to delete IPv4 record: %w", err) + } } else { if err = provider.addDomainRecord(ctx, "A", provider.IPAddrs.IPv4Addr); err != nil { return err @@ -65,10 +69,13 @@ func (provider *Provider) updateDomain(ctx context.Context, domain string) error } } - // 独立处理 IPv6 更新 + // 独立处理 IPv6 更新或删除 if *provider.DDNSProfile.EnableIPv6 { if provider.IPAddrs.IPv6Addr == "" { - log.Printf("NEZHA>> Skip IPv6 update for domain %s: IPv6 address is empty", domain) + log.Printf("NEZHA>> IPv6 address is empty for domain %s, deleting AAAA record...", domain) + if err = provider.deleteDomainRecord(ctx, "AAAA"); err != nil { + return fmt.Errorf("failed to delete IPv6 record: %w", err) + } } else { if err = provider.addDomainRecord(ctx, "AAAA", provider.IPAddrs.IPv6Addr); err != nil { return err @@ -96,6 +103,77 @@ func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr str return err } +// deleteDomainRecord 用于删除指定类型的解析记录 +func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string) error { + // 检查 libdns.RecordGetter 和 libdns.RecordDeleter + getter, ok := provider.Setter.(libdns.RecordGetter) + if !ok { + return fmt.Errorf("dns provider does not support record getting") + } + + deleter, ok := provider.Setter.(libdns.RecordDeleter) + if !ok { + return fmt.Errorf("dns provider does not support record deletion") + } + + // 获取 DNS 记录 + allRecords, err := getter.GetRecords(ctx, provider.zone) + if err != nil { + return fmt.Errorf("failed to get DNS records: %w", err) + } + + cleanName := func(name string) string { + return strings.ToLower(strings.TrimSuffix(name, ".")) + } + + cleanPrefix := cleanName(provider.prefix) + cleanZone := cleanName(provider.zone) + targetRecType := strings.ToUpper(recType) + + // 筛选和匹配 + var targetRecords []libdns.Record + for _, rec := range allRecords { + rr := rec.RR() + recName := rr.Name + currentType := strings.ToUpper(rr.Type) + + if currentType != targetRecType { + continue + } + + relName := libdns.RelativeName(recName, provider.zone) + cleanRel := cleanName(relName) + cleanRRName := cleanName(recName) + + var matchedName bool + if cleanPrefix == "" { + matchedName = cleanRel == "" || cleanRel == "@" || cleanRRName == cleanZone + } else { + matchedName = cleanRel == cleanPrefix || cleanRRName == cleanPrefix + } + + // 保留原始的 rec 接口（包含 Cloudflare 所需的 ProviderData 元数据），用于精准删除 + if matchedName { + targetRecords = append(targetRecords, rec) + } + } + + // 若未找到对应记录，则直接返回成功（视为已删除或不存在） + if len(targetRecords) == 0 { + log.Printf("NEZHA>> No matching %s record found for deletion under zone %s", recType, provider.zone) + return nil + } + + // 执行删除操作 + _, err = deleter.DeleteRecords(ctx, provider.zone, targetRecords) + if err != nil { + return fmt.Errorf("deleter.DeleteRecords failed: %w", err) + } + + log.Printf("NEZHA>> Successfully deleted %d matching %s record(s) for %s.%s", len(targetRecords), recType, provider.prefix, provider.zone) + return nil +} + func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (prefix string, zone string, err error) { c := &dns.Client{Timeout: dnsTimeOut} From f366677744d99fc63fbd347322a8a5144b9b03cc Mon Sep 17 00:00:00 2001 From: Ryrenz <163799701+ryrenz@users.noreply.github.com> Date: Sat, 12 Sep 2026 06:56:35 -0400 Subject: [PATCH 09/48] feat: surface GPU memory in host state (#1235) - Add State_GPU message to the proto with utilization and memory fields - Add HostState.GPUs with conversion in both PB() and PB2State() - Keep the existing gpu field populated so older agents keep working Regenerated with protoc-gen-go v1.34.2 / protoc v5.29.3 to match the existing files; nezha_grpc.pb.go is untouched since the service is unchanged. Closes #1217 --- model/host.go | 30 +++++ proto/nezha.pb.go | 335 ++++++++++++++++++++++++++++++---------------- proto/nezha.proto | 10 ++ 3 files changed, 256 insertions(+), 119 deletions(-) diff --git a/model/host.go b/model/host.go index f054798b..18f4e3da 100644 --- a/model/host.go +++ b/model/host.go @@ -35,9 +35,28 @@ type HostState struct { ProcessCount uint64 `json:"process_count,omitempty"` Temperatures []SensorTemperature `json:"temperatures,omitempty"` GPU []float64 `json:"gpu,omitempty"` + GPUs []GPUStat `json:"gpus,omitempty"` +} + +// GPUStat carries per-card figures, index-aligned with Host.GPU. Memory is in +// MiB and stays zero for vendors that do not report it, so MemoryTotal == 0 +// means "unknown" rather than "no memory". +type GPUStat struct { + Utilization float64 `json:"utilization"` + MemoryUsed uint64 `json:"memory_used,omitempty"` + MemoryTotal uint64 `json:"memory_total,omitempty"` } func (s *HostState) PB() *pb.State { + gs := make([]*pb.State_GPU, 0, len(s.GPUs)) + for _, g := range s.GPUs { + gs = append(gs, &pb.State_GPU{ + Utilization: g.Utilization, + MemoryUsed: g.MemoryUsed, + MemoryTotal: g.MemoryTotal, + }) + } + var ts []*pb.State_SensorTemperature for _, t := range s.Temperatures { ts = append(ts, &pb.State_SensorTemperature{ @@ -64,10 +83,20 @@ func (s *HostState) PB() *pb.State { ProcessCount: s.ProcessCount, Temperatures: ts, Gpu: s.GPU, + Gpus: gs, } } func PB2State(s *pb.State) HostState { + var gs []GPUStat + for _, g := range s.GetGpus() { + gs = append(gs, GPUStat{ + Utilization: g.GetUtilization(), + MemoryUsed: g.GetMemoryUsed(), + MemoryTotal: g.GetMemoryTotal(), + }) + } + var ts []SensorTemperature for _, t := range s.GetTemperatures() { ts = append(ts, SensorTemperature{ @@ -93,6 +122,7 @@ func PB2State(s *pb.State) HostState { UdpConnCount: s.GetUdpConnCount(), ProcessCount: s.GetProcessCount(), Temperatures: ts, + GPUs: gs, GPU: s.GetGpu(), } } diff --git a/proto/nezha.pb.go b/proto/nezha.pb.go index 2a6b84af..16c73f50 100644 --- a/proto/nezha.pb.go +++ b/proto/nezha.pb.go @@ -169,6 +169,7 @@ type State struct { ProcessCount uint64 `protobuf:"varint,15,opt,name=process_count,json=processCount,proto3" json:"process_count,omitempty"` Temperatures []*State_SensorTemperature `protobuf:"bytes,16,rep,name=temperatures,proto3" json:"temperatures,omitempty"` Gpu []float64 `protobuf:"fixed64,17,rep,packed,name=gpu,proto3" json:"gpu,omitempty"` + Gpus []*State_GPU `protobuf:"bytes,18,rep,name=gpus,proto3" json:"gpus,omitempty"` } func (x *State) Reset() { @@ -322,6 +323,79 @@ func (x *State) GetGpu() []float64 { return nil } +func (x *State) GetGpus() []*State_GPU { + if x != nil { + return x.Gpus + } + return nil +} + +// State_GPU carries per-card figures. Index-aligned with Host.gpu, matching +// the existing `gpu` field it supersedes; that field stays populated so older +// dashboards keep working. +type State_GPU struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Utilization float64 `protobuf:"fixed64,1,opt,name=utilization,proto3" json:"utilization,omitempty"` + MemoryUsed uint64 `protobuf:"varint,2,opt,name=memory_used,json=memoryUsed,proto3" json:"memory_used,omitempty"` + MemoryTotal uint64 `protobuf:"varint,3,opt,name=memory_total,json=memoryTotal,proto3" json:"memory_total,omitempty"` +} + +func (x *State_GPU) Reset() { + *x = State_GPU{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_nezha_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *State_GPU) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*State_GPU) ProtoMessage() {} + +func (x *State_GPU) ProtoReflect() protoreflect.Message { + mi := &file_proto_nezha_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use State_GPU.ProtoReflect.Descriptor instead. +func (*State_GPU) Descriptor() ([]byte, []int) { + return file_proto_nezha_proto_rawDescGZIP(), []int{2} +} + +func (x *State_GPU) GetUtilization() float64 { + if x != nil { + return x.Utilization + } + return 0 +} + +func (x *State_GPU) GetMemoryUsed() uint64 { + if x != nil { + return x.MemoryUsed + } + return 0 +} + +func (x *State_GPU) GetMemoryTotal() uint64 { + if x != nil { + return x.MemoryTotal + } + return 0 +} + type State_SensorTemperature struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -334,7 +408,7 @@ type State_SensorTemperature struct { func (x *State_SensorTemperature) Reset() { *x = State_SensorTemperature{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[2] + mi := &file_proto_nezha_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -347,7 +421,7 @@ func (x *State_SensorTemperature) String() string { func (*State_SensorTemperature) ProtoMessage() {} func (x *State_SensorTemperature) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[2] + mi := &file_proto_nezha_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -360,7 +434,7 @@ func (x *State_SensorTemperature) ProtoReflect() protoreflect.Message { // Deprecated: Use State_SensorTemperature.ProtoReflect.Descriptor instead. func (*State_SensorTemperature) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{2} + return file_proto_nezha_proto_rawDescGZIP(), []int{3} } func (x *State_SensorTemperature) GetName() string { @@ -390,7 +464,7 @@ type Task struct { func (x *Task) Reset() { *x = Task{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[3] + mi := &file_proto_nezha_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -403,7 +477,7 @@ func (x *Task) String() string { func (*Task) ProtoMessage() {} func (x *Task) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[3] + mi := &file_proto_nezha_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -416,7 +490,7 @@ func (x *Task) ProtoReflect() protoreflect.Message { // Deprecated: Use Task.ProtoReflect.Descriptor instead. func (*Task) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{3} + return file_proto_nezha_proto_rawDescGZIP(), []int{4} } func (x *Task) GetId() uint64 { @@ -455,7 +529,7 @@ type TaskResult struct { func (x *TaskResult) Reset() { *x = TaskResult{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[4] + mi := &file_proto_nezha_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -468,7 +542,7 @@ func (x *TaskResult) String() string { func (*TaskResult) ProtoMessage() {} func (x *TaskResult) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[4] + mi := &file_proto_nezha_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -481,7 +555,7 @@ func (x *TaskResult) ProtoReflect() protoreflect.Message { // Deprecated: Use TaskResult.ProtoReflect.Descriptor instead. func (*TaskResult) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{4} + return file_proto_nezha_proto_rawDescGZIP(), []int{5} } func (x *TaskResult) GetId() uint64 { @@ -530,7 +604,7 @@ type Receipt struct { func (x *Receipt) Reset() { *x = Receipt{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[5] + mi := &file_proto_nezha_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -543,7 +617,7 @@ func (x *Receipt) String() string { func (*Receipt) ProtoMessage() {} func (x *Receipt) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[5] + mi := &file_proto_nezha_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -556,7 +630,7 @@ func (x *Receipt) ProtoReflect() protoreflect.Message { // Deprecated: Use Receipt.ProtoReflect.Descriptor instead. func (*Receipt) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{5} + return file_proto_nezha_proto_rawDescGZIP(), []int{6} } func (x *Receipt) GetProced() bool { @@ -577,7 +651,7 @@ type Uint64Receipt struct { func (x *Uint64Receipt) Reset() { *x = Uint64Receipt{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[6] + mi := &file_proto_nezha_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -590,7 +664,7 @@ func (x *Uint64Receipt) String() string { func (*Uint64Receipt) ProtoMessage() {} func (x *Uint64Receipt) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[6] + mi := &file_proto_nezha_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -603,7 +677,7 @@ func (x *Uint64Receipt) ProtoReflect() protoreflect.Message { // Deprecated: Use Uint64Receipt.ProtoReflect.Descriptor instead. func (*Uint64Receipt) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{6} + return file_proto_nezha_proto_rawDescGZIP(), []int{7} } func (x *Uint64Receipt) GetData() uint64 { @@ -624,7 +698,7 @@ type IOStreamData struct { func (x *IOStreamData) Reset() { *x = IOStreamData{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[7] + mi := &file_proto_nezha_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -637,7 +711,7 @@ func (x *IOStreamData) String() string { func (*IOStreamData) ProtoMessage() {} func (x *IOStreamData) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[7] + mi := &file_proto_nezha_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -650,7 +724,7 @@ func (x *IOStreamData) ProtoReflect() protoreflect.Message { // Deprecated: Use IOStreamData.ProtoReflect.Descriptor instead. func (*IOStreamData) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{7} + return file_proto_nezha_proto_rawDescGZIP(), []int{8} } func (x *IOStreamData) GetData() []byte { @@ -674,7 +748,7 @@ type GeoIP struct { func (x *GeoIP) Reset() { *x = GeoIP{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[8] + mi := &file_proto_nezha_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -687,7 +761,7 @@ func (x *GeoIP) String() string { func (*GeoIP) ProtoMessage() {} func (x *GeoIP) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[8] + mi := &file_proto_nezha_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -700,7 +774,7 @@ func (x *GeoIP) ProtoReflect() protoreflect.Message { // Deprecated: Use GeoIP.ProtoReflect.Descriptor instead. func (*GeoIP) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{8} + return file_proto_nezha_proto_rawDescGZIP(), []int{9} } func (x *GeoIP) GetUse6() bool { @@ -743,7 +817,7 @@ type IP struct { func (x *IP) Reset() { *x = IP{} if protoimpl.UnsafeEnabled { - mi := &file_proto_nezha_proto_msgTypes[9] + mi := &file_proto_nezha_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -756,7 +830,7 @@ func (x *IP) String() string { func (*IP) ProtoMessage() {} func (x *IP) ProtoReflect() protoreflect.Message { - mi := &file_proto_nezha_proto_msgTypes[9] + mi := &file_proto_nezha_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -769,7 +843,7 @@ func (x *IP) ProtoReflect() protoreflect.Message { // Deprecated: Use IP.ProtoReflect.Descriptor instead. func (*IP) Descriptor() ([]byte, []int) { - return file_proto_nezha_proto_rawDescGZIP(), []int{9} + return file_proto_nezha_proto_rawDescGZIP(), []int{10} } func (x *IP) GetIpv4() string { @@ -810,7 +884,7 @@ var file_proto_nezha_proto_rawDesc = []byte{ 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x62, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, - 0x75, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x22, 0xa9, 0x04, 0x0a, + 0x75, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x22, 0xcf, 0x04, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x65, 0x6d, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6d, 0x65, 0x6d, 0x55, @@ -845,65 +919,74 @@ var file_proto_nezha_proto_rawDesc = []byte{ 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x53, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, 0x75, 0x18, 0x11, 0x20, - 0x03, 0x28, 0x01, 0x52, 0x03, 0x67, 0x70, 0x75, 0x22, 0x4f, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x5f, 0x53, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x74, 0x65, 0x6d, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x65, - 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x3e, 0x0a, 0x04, 0x54, 0x61, 0x73, - 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, - 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x7a, 0x0a, 0x0a, 0x54, 0x61, 0x73, - 0x6b, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, - 0x65, 0x6c, 0x61, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, - 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x66, 0x75, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x66, 0x75, 0x6c, 0x22, 0x21, 0x0a, 0x07, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x06, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x64, 0x22, 0x23, 0x0a, 0x0d, 0x55, 0x69, 0x6e, 0x74, - 0x36, 0x34, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x22, 0x0a, - 0x0c, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x22, 0x89, 0x01, 0x0a, 0x05, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x12, 0x12, 0x0a, 0x04, 0x75, - 0x73, 0x65, 0x36, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x75, 0x73, 0x65, 0x36, 0x12, - 0x19, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x50, 0x52, 0x02, 0x69, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x2e, 0x0a, - 0x13, 0x64, 0x61, 0x73, 0x68, 0x62, 0x6f, 0x61, 0x72, 0x64, 0x5f, 0x62, 0x6f, 0x6f, 0x74, 0x5f, - 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x11, 0x64, 0x61, 0x73, 0x68, - 0x62, 0x6f, 0x61, 0x72, 0x64, 0x42, 0x6f, 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x2c, 0x0a, - 0x02, 0x49, 0x50, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x70, 0x76, 0x34, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x69, 0x70, 0x76, 0x34, 0x12, 0x12, 0x0a, 0x04, 0x69, 0x70, 0x76, 0x36, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x70, 0x76, 0x36, 0x32, 0xd2, 0x02, 0x0a, 0x0c, - 0x4e, 0x65, 0x7a, 0x68, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x37, 0x0a, 0x11, - 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x1a, + 0x03, 0x28, 0x01, 0x52, 0x03, 0x67, 0x70, 0x75, 0x12, 0x24, 0x0a, 0x04, 0x67, 0x70, 0x75, 0x73, + 0x18, 0x12, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x5f, 0x47, 0x50, 0x55, 0x52, 0x04, 0x67, 0x70, 0x75, 0x73, 0x22, 0x71, + 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x47, 0x50, 0x55, 0x12, 0x20, 0x0a, 0x0b, 0x75, + 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x0b, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, + 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0a, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x55, 0x73, 0x65, 0x64, 0x12, 0x21, + 0x0a, 0x0c, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x54, 0x6f, 0x74, 0x61, + 0x6c, 0x22, 0x4f, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x53, 0x65, 0x6e, 0x73, 0x6f, + 0x72, 0x54, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x20, 0x0a, 0x0b, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x65, 0x6d, 0x70, 0x65, 0x72, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x22, 0x3e, 0x0a, 0x04, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x22, 0x7a, 0x0a, 0x0a, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, + 0x74, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x02, 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1e, + 0x0a, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x22, 0x21, + 0x0a, 0x07, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x6f, + 0x63, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x72, 0x6f, 0x63, 0x65, + 0x64, 0x22, 0x23, 0x0a, 0x0d, 0x55, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x52, 0x65, 0x63, 0x65, 0x69, + 0x70, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x22, 0x0a, 0x0c, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x89, 0x01, 0x0a, 0x05, 0x47, + 0x65, 0x6f, 0x49, 0x50, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x36, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x04, 0x75, 0x73, 0x65, 0x36, 0x12, 0x19, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x50, 0x52, + 0x02, 0x69, 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x63, + 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x72, 0x79, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x64, 0x61, 0x73, 0x68, 0x62, 0x6f, + 0x61, 0x72, 0x64, 0x5f, 0x62, 0x6f, 0x6f, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x11, 0x64, 0x61, 0x73, 0x68, 0x62, 0x6f, 0x61, 0x72, 0x64, 0x42, 0x6f, + 0x6f, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x2c, 0x0a, 0x02, 0x49, 0x50, 0x12, 0x12, 0x0a, 0x04, + 0x69, 0x70, 0x76, 0x34, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x69, 0x70, 0x76, 0x34, + 0x12, 0x12, 0x0a, 0x04, 0x69, 0x70, 0x76, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x69, 0x70, 0x76, 0x36, 0x32, 0xd2, 0x02, 0x0a, 0x0c, 0x4e, 0x65, 0x7a, 0x68, 0x61, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x37, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, + 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x1a, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x31, + 0x0a, 0x10, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, - 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x31, 0x0a, 0x10, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, - 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, - 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x0b, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x0b, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3a, 0x0a, - 0x08, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x13, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, - 0x61, 0x74, 0x61, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x2b, 0x0a, 0x0b, 0x52, 0x65, 0x70, - 0x6f, 0x72, 0x74, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x1a, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, - 0x65, 0x6f, 0x49, 0x50, 0x22, 0x00, 0x12, 0x38, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, - 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x0b, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x55, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x00, - 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x00, 0x12, 0x33, 0x0a, 0x0b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, + 0x12, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x1a, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x61, 0x73, 0x6b, + 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x49, 0x4f, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x49, 0x4f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x22, 0x00, 0x28, 0x01, + 0x30, 0x01, 0x12, 0x2b, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x47, 0x65, 0x6f, 0x49, + 0x50, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x1a, + 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x47, 0x65, 0x6f, 0x49, 0x50, 0x22, 0x00, 0x12, + 0x38, 0x0a, 0x11, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, + 0x6e, 0x66, 0x6f, 0x32, 0x12, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x48, 0x6f, 0x73, + 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x69, 0x6e, 0x74, 0x36, 0x34, + 0x52, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x00, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -918,39 +1001,41 @@ func file_proto_nezha_proto_rawDescGZIP() []byte { return file_proto_nezha_proto_rawDescData } -var file_proto_nezha_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_proto_nezha_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_proto_nezha_proto_goTypes = []any{ (*Host)(nil), // 0: proto.Host (*State)(nil), // 1: proto.State - (*State_SensorTemperature)(nil), // 2: proto.State_SensorTemperature - (*Task)(nil), // 3: proto.Task - (*TaskResult)(nil), // 4: proto.TaskResult - (*Receipt)(nil), // 5: proto.Receipt - (*Uint64Receipt)(nil), // 6: proto.Uint64Receipt - (*IOStreamData)(nil), // 7: proto.IOStreamData - (*GeoIP)(nil), // 8: proto.GeoIP - (*IP)(nil), // 9: proto.IP + (*State_GPU)(nil), // 2: proto.State_GPU + (*State_SensorTemperature)(nil), // 3: proto.State_SensorTemperature + (*Task)(nil), // 4: proto.Task + (*TaskResult)(nil), // 5: proto.TaskResult + (*Receipt)(nil), // 6: proto.Receipt + (*Uint64Receipt)(nil), // 7: proto.Uint64Receipt + (*IOStreamData)(nil), // 8: proto.IOStreamData + (*GeoIP)(nil), // 9: proto.GeoIP + (*IP)(nil), // 10: proto.IP } var file_proto_nezha_proto_depIdxs = []int32{ - 2, // 0: proto.State.temperatures:type_name -> proto.State_SensorTemperature - 9, // 1: proto.GeoIP.ip:type_name -> proto.IP - 1, // 2: proto.NezhaService.ReportSystemState:input_type -> proto.State - 0, // 3: proto.NezhaService.ReportSystemInfo:input_type -> proto.Host - 4, // 4: proto.NezhaService.RequestTask:input_type -> proto.TaskResult - 7, // 5: proto.NezhaService.IOStream:input_type -> proto.IOStreamData - 8, // 6: proto.NezhaService.ReportGeoIP:input_type -> proto.GeoIP - 0, // 7: proto.NezhaService.ReportSystemInfo2:input_type -> proto.Host - 5, // 8: proto.NezhaService.ReportSystemState:output_type -> proto.Receipt - 5, // 9: proto.NezhaService.ReportSystemInfo:output_type -> proto.Receipt - 3, // 10: proto.NezhaService.RequestTask:output_type -> proto.Task - 7, // 11: proto.NezhaService.IOStream:output_type -> proto.IOStreamData - 8, // 12: proto.NezhaService.ReportGeoIP:output_type -> proto.GeoIP - 6, // 13: proto.NezhaService.ReportSystemInfo2:output_type -> proto.Uint64Receipt - 8, // [8:14] is the sub-list for method output_type - 2, // [2:8] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 3, // 0: proto.State.temperatures:type_name -> proto.State_SensorTemperature + 2, // 1: proto.State.gpus:type_name -> proto.State_GPU + 10, // 2: proto.GeoIP.ip:type_name -> proto.IP + 1, // 3: proto.NezhaService.ReportSystemState:input_type -> proto.State + 0, // 4: proto.NezhaService.ReportSystemInfo:input_type -> proto.Host + 5, // 5: proto.NezhaService.RequestTask:input_type -> proto.TaskResult + 8, // 6: proto.NezhaService.IOStream:input_type -> proto.IOStreamData + 9, // 7: proto.NezhaService.ReportGeoIP:input_type -> proto.GeoIP + 0, // 8: proto.NezhaService.ReportSystemInfo2:input_type -> proto.Host + 6, // 9: proto.NezhaService.ReportSystemState:output_type -> proto.Receipt + 6, // 10: proto.NezhaService.ReportSystemInfo:output_type -> proto.Receipt + 4, // 11: proto.NezhaService.RequestTask:output_type -> proto.Task + 8, // 12: proto.NezhaService.IOStream:output_type -> proto.IOStreamData + 9, // 13: proto.NezhaService.ReportGeoIP:output_type -> proto.GeoIP + 7, // 14: proto.NezhaService.ReportSystemInfo2:output_type -> proto.Uint64Receipt + 9, // [9:15] is the sub-list for method output_type + 3, // [3:9] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_proto_nezha_proto_init() } @@ -984,7 +1069,7 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[2].Exporter = func(v any, i int) any { - switch v := v.(*State_SensorTemperature); i { + switch v := v.(*State_GPU); i { case 0: return &v.state case 1: @@ -996,7 +1081,7 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[3].Exporter = func(v any, i int) any { - switch v := v.(*Task); i { + switch v := v.(*State_SensorTemperature); i { case 0: return &v.state case 1: @@ -1008,7 +1093,7 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[4].Exporter = func(v any, i int) any { - switch v := v.(*TaskResult); i { + switch v := v.(*Task); i { case 0: return &v.state case 1: @@ -1020,7 +1105,7 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[5].Exporter = func(v any, i int) any { - switch v := v.(*Receipt); i { + switch v := v.(*TaskResult); i { case 0: return &v.state case 1: @@ -1032,7 +1117,7 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[6].Exporter = func(v any, i int) any { - switch v := v.(*Uint64Receipt); i { + switch v := v.(*Receipt); i { case 0: return &v.state case 1: @@ -1044,7 +1129,7 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[7].Exporter = func(v any, i int) any { - switch v := v.(*IOStreamData); i { + switch v := v.(*Uint64Receipt); i { case 0: return &v.state case 1: @@ -1056,7 +1141,7 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[8].Exporter = func(v any, i int) any { - switch v := v.(*GeoIP); i { + switch v := v.(*IOStreamData); i { case 0: return &v.state case 1: @@ -1068,6 +1153,18 @@ func file_proto_nezha_proto_init() { } } file_proto_nezha_proto_msgTypes[9].Exporter = func(v any, i int) any { + switch v := v.(*GeoIP); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_nezha_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*IP); i { case 0: return &v.state @@ -1086,7 +1183,7 @@ func file_proto_nezha_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_proto_nezha_proto_rawDesc, NumEnums: 0, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/nezha.proto b/proto/nezha.proto index 75debccd..89383b32 100644 --- a/proto/nezha.proto +++ b/proto/nezha.proto @@ -44,6 +44,16 @@ message State { uint64 process_count = 15; repeated State_SensorTemperature temperatures = 16; repeated double gpu = 17; + repeated State_GPU gpus = 18; +} + +// State_GPU carries per-card figures. Index-aligned with Host.gpu, matching +// the existing `gpu` field it supersedes; that field stays populated so older +// dashboards keep working. +message State_GPU { + double utilization = 1; + uint64 memory_used = 2; + uint64 memory_total = 3; } message State_SensorTemperature { From 5d5830fef359750f7db295d8e79b87fe797d498a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 12 Sep 2026 18:56:54 +0800 Subject: [PATCH 10/48] update contributors[no ci] --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d84eb831..6e3cf055 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ Darc Z. Creling Core F -Coming +Coming Wel adminsama Acgpiano eya46 @@ -155,6 +155,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ 77-QiQi Chra Mubelotix +Ryrenz zhdsmy ## Special Thanks From 8cb86e385365cd009ce3d68589bcaef7f290a88e Mon Sep 17 00:00:00 2001 From: naiba Date: Sat, 12 Sep 2026 11:11:22 +0000 Subject: [PATCH 11/48] chore(frontend-templates): update frontend versions --- service/singleton/frontend-templates.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/singleton/frontend-templates.yaml b/service/singleton/frontend-templates.yaml index 40e54b1f..fecbb885 100644 --- a/service/singleton/frontend-templates.yaml +++ b/service/singleton/frontend-templates.yaml @@ -2,7 +2,7 @@ name: "OfficialAdmin" repository: "https://github.com/nezhahq/admin-frontend" author: "nezhahq" - version: "v2.3.5" + version: "v2.3.6" is_admin: true is_official: true - path: "user-dist" From 46d83d6d6844ad70e6303da07c95218099fab3f6 Mon Sep 17 00:00:00 2001 From: naiba Date: Sat, 12 Sep 2026 13:29:44 +0000 Subject: [PATCH 12/48] style(ddns): apply gofmt --- pkg/ddns/ddns.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ddns/ddns.go b/pkg/ddns/ddns.go index 77779c0b..8abecad3 100644 --- a/pkg/ddns/ddns.go +++ b/pkg/ddns/ddns.go @@ -134,8 +134,8 @@ func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string var targetRecords []libdns.Record for _, rec := range allRecords { rr := rec.RR() - recName := rr.Name - currentType := strings.ToUpper(rr.Type) + recName := rr.Name + currentType := strings.ToUpper(rr.Type) if currentType != targetRecType { continue From d19952a834f26a700cd042c09a9f85e97db4086e Mon Sep 17 00:00:00 2001 From: NikoCat233 Date: Sat, 12 Sep 2026 22:59:17 +0800 Subject: [PATCH 13/48] feat(auth): allow JWT IP changes by configuration (#1236) * feat(auth): allow JWT IP changes by configuration * fix(auth): make JWT IP change setting concurrency-safe * test(auth): move JWT IP change setting tests out of MCP test file --- cmd/dashboard/controller/jwt.go | 2 +- cmd/dashboard/controller/jwt_session_test.go | 55 ++++++++++++++++++ cmd/dashboard/controller/setting.go | 9 +++ .../controller/setting_jwt_ip_change_test.go | 26 +++++++++ model/config.go | 21 +++++++ model/config_test.go | 4 +- model/jwt_ip_change_atomic_test.go | 56 +++++++++++++++++++ model/setting_api.go | 1 + 8 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 cmd/dashboard/controller/setting_jwt_ip_change_test.go create mode 100644 model/jwt_ip_change_atomic_test.go diff --git a/cmd/dashboard/controller/jwt.go b/cmd/dashboard/controller/jwt.go index db0865e0..9fd7ff0c 100644 --- a/cmd/dashboard/controller/jwt.go +++ b/cmd/dashboard/controller/jwt.go @@ -152,7 +152,7 @@ func identityHandler() func(c *gin.Context) any { return nil } currentIP := c.GetString(model.CtxKeyRealIPStr) - if sess.IP != currentIP { + if !singleton.Conf.JWTIPChangeAllowed() && sess.IP != currentIP { c.Set(model.CtxKeyIsIPMismatch, true) return nil } diff --git a/cmd/dashboard/controller/jwt_session_test.go b/cmd/dashboard/controller/jwt_session_test.go index 29b5e15e..0f169cb3 100644 --- a/cmd/dashboard/controller/jwt_session_test.go +++ b/cmd/dashboard/controller/jwt_session_test.go @@ -193,6 +193,61 @@ func TestIdentityHandlerFlagsIPMismatch(t *testing.T) { assert.True(t, verify.GetBool(model.CtxKeyIsIPMismatch)) } +func TestIdentityHandlerAllowsIPMismatchWhenConfigured(t *testing.T) { + cleanup := setupJWTSessionTest(t) + defer cleanup() + singleton.Conf.SetJWTIPChangeAllowed(true) + + ctx := newCtxForUser(0, "1.2.3.4", "ua") + user := model.User{Common: model.Common{ID: 100}, TokenVersion: 7} + claims, err := issueJWTSession(ctx, &user, 1) + require.NoError(t, err) + + verify := newCtxForUser(0, "9.9.9.9", "ua") + verify.Set("JWT_PAYLOAD", jwt.MapClaims{ + jwtClaimUserID: claims[jwtClaimUserID], + jwtClaimKeyID: claims[jwtClaimKeyID], + }) + + identity := identityHandler()(verify) + require.NotNil(t, identity, "IP mismatch must be accepted when allow_jwt_ip_change is enabled") + assert.False(t, verify.GetBool(model.CtxKeyIsIPMismatch)) +} + +func TestIdentityHandlerConcurrentIPChangeSetting(t *testing.T) { + cleanup := setupJWTSessionTest(t) + defer cleanup() + + ctx := newCtxForUser(0, "1.2.3.4", "ua") + user := model.User{Common: model.Common{ID: 100}, TokenVersion: 7} + claims, err := issueJWTSession(ctx, &user, 1) + require.NoError(t, err) + + stop, done := make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + singleton.Conf.SetJWTIPChangeAllowed(true) + singleton.Conf.SetJWTIPChangeAllowed(false) + } + } + }() + defer func() { + close(stop) + <-done + }() + + for i := 0; i < 100; i++ { + verify := newCtxForUser(0, "1.2.3.4", "ua") + verify.Set("JWT_PAYLOAD", jwt.MapClaims(claims)) + require.NotNil(t, identityHandler()(verify), "matching IP must pass during setting updates") + } +} + func TestIdentityHandlerRejectsUnknownKeyID(t *testing.T) { cleanup := setupJWTSessionTest(t) defer cleanup() diff --git a/cmd/dashboard/controller/setting.go b/cmd/dashboard/controller/setting.go index aac58a56..0b6b70e6 100644 --- a/cmd/dashboard/controller/setting.go +++ b/cmd/dashboard/controller/setting.go @@ -97,6 +97,9 @@ func updateConfig(c *gin.Context) (any, error) { singleton.Conf.EnableIPChangeNotification = sf.EnableIPChangeNotification singleton.Conf.EnablePlainIPInNotification = sf.EnablePlainIPInNotification + if sf.AllowJWTIPChange != nil { + singleton.Conf.SetJWTIPChangeAllowed(*sf.AllowJWTIPChange) + } singleton.Conf.Cover = sf.Cover singleton.Conf.InstallHost = sf.InstallHost singleton.Conf.DashboardHost = sf.DashboardHost @@ -160,6 +163,12 @@ func fireMCPKillSwitch() { // the current config to avoid accidentally tripping the kill switch on // partial PATCH calls that omit enable_mcp. func resolveSettingEnableMCP(formValue *bool, current bool) bool { + return resolveOptionalBool(formValue, current) +} + +// resolveOptionalBool preserves a setting for partial PATCH requests when its +// boolean field is absent, while still allowing an explicit false value. +func resolveOptionalBool(formValue *bool, current bool) bool { if formValue == nil { return current } diff --git a/cmd/dashboard/controller/setting_jwt_ip_change_test.go b/cmd/dashboard/controller/setting_jwt_ip_change_test.go new file mode 100644 index 00000000..a3f5229b --- /dev/null +++ b/cmd/dashboard/controller/setting_jwt_ip_change_test.go @@ -0,0 +1,26 @@ +package controller + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestSettingForm_AllowJWTIPChangePreservesAbsentAndDecodesExplicitValues(t *testing.T) { + var omitted model.SettingForm + require.NoError(t, json.Unmarshal([]byte(`{"site_name":"X"}`), &omitted)) + require.Nil(t, omitted.AllowJWTIPChange) + + var enabled model.SettingForm + require.NoError(t, json.Unmarshal([]byte(`{"allow_jwt_ip_change":true}`), &enabled)) + require.NotNil(t, enabled.AllowJWTIPChange) + require.True(t, *enabled.AllowJWTIPChange) + + var disabled model.SettingForm + require.NoError(t, json.Unmarshal([]byte(`{"allow_jwt_ip_change":false}`), &disabled)) + require.NotNil(t, disabled.AllowJWTIPChange) + require.False(t, *disabled.AllowJWTIPChange) +} diff --git a/model/config.go b/model/config.go index 3e766b2f..fde4bd19 100644 --- a/model/config.go +++ b/model/config.go @@ -61,6 +61,9 @@ type ConfigDashboard struct { AdminTemplate string `koanf:"admin_template" json:"admin_template,omitempty"` EnablePlainIPInNotification bool `koanf:"enable_plain_ip_in_notification" json:"enable_plain_ip_in_notification,omitempty"` // 通知信息IP不打码 + // AllowJWTIPChange disables the IP binding check for browser login sessions. + // Keep this disabled by default so a stolen JWT remains bound to its issuing IP. + AllowJWTIPChange bool `koanf:"allow_jwt_ip_change" json:"allow_jwt_ip_change,omitempty"` EnableMCP bool `koanf:"enable_mcp" json:"enable_mcp,omitempty"` // 是否启用 MCP 入口（默认关闭；启用前请审视 PAT scope/whitelist） @@ -103,6 +106,10 @@ type Config struct { // SettingResponse 按值拷贝 ConfigDashboard 触发 copylocks。 mcpEnabled atomic.Bool `koanf:"-" json:"-" yaml:"-"` + // jwtIPChangeAllowed mirrors AllowJWTIPChange for concurrent authentication + // and setting updates. Keep atomic fields outside ConfigDashboard. + jwtIPChangeAllowed atomic.Bool `koanf:"-" json:"-" yaml:"-"` + // oauth2 配置 Oauth2 map[string]*Oauth2Config `koanf:"oauth2" json:"oauth2,omitempty"` @@ -225,6 +232,8 @@ func (c *Config) Read(path string, frontendTemplates []FrontendTemplate) error { c.JWTTimeout = 1 } + c.jwtIPChangeAllowed.Store(c.AllowJWTIPChange) + if c.AgentSecretKey == "" { c.AgentSecretKey, err = utils.GenerateRandomString(32) if err != nil { @@ -252,6 +261,17 @@ func (c *Config) SetMCPEnabled(v bool) { c.mcpEnabled.Store(v) } +// JWTIPChangeAllowed reports whether JWT sessions may change IP concurrently. +func (c *Config) JWTIPChangeAllowed() bool { + return c.jwtIPChangeAllowed.Load() +} + +// SetJWTIPChangeAllowed updates only the atomic mirror. save() synchronizes +// the public field for persistence; authentication must use the accessor. +func (c *Config) SetJWTIPChangeAllowed(v bool) { + c.jwtIPChangeAllowed.Store(v) +} + // Save 保存配置文件 func (c *Config) Save() error { return c.save() @@ -320,6 +340,7 @@ func (c *Config) patchYAMLField(key string, value any) error { func (c *Config) save() error { c.EnableMCP = c.mcpEnabled.Load() + c.AllowJWTIPChange = c.jwtIPChangeAllowed.Load() data, err := yaml.Marshal(c) if err != nil { return err diff --git a/model/config_test.go b/model/config_test.go index 4242c199..d5d4b6cd 100644 --- a/model/config_test.go +++ b/model/config_test.go @@ -24,6 +24,7 @@ func TestReadConfig(t *testing.T) { {"user_template", c.UserTemplate, c.UserTemplate == "user-dist"}, {"admin_template", c.AdminTemplate, c.AdminTemplate == "admin-dist"}, {"agent_secret_key", c.AgentSecretKey, c.AgentSecretKey != ""}, + {"allow_jwt_ip_change", c.AllowJWTIPChange, !c.AllowJWTIPChange}, } for _, field := range testFields { @@ -36,7 +37,7 @@ func TestReadConfig(t *testing.T) { }) t.Run("ReadFile", func(t *testing.T) { - const testCfg = "jwt_secret_key: test\nuser_template: um\nadmin_template: am\nagent_secret_key: none\nsite_name: lowkick" + const testCfg = "jwt_secret_key: test\nuser_template: um\nadmin_template: am\nagent_secret_key: none\nsite_name: lowkick\nallow_jwt_ip_change: true" var testFrontendTemplates = []FrontendTemplate{ {Path: "um"}, @@ -59,6 +60,7 @@ func TestReadConfig(t *testing.T) { {"admin_template", c.AdminTemplate, c.AdminTemplate == "am"}, {"agent_secret_key", c.AgentSecretKey, c.AgentSecretKey == "none"}, {"site_name", c.SiteName, c.SiteName == "lowkick"}, + {"allow_jwt_ip_change", c.AllowJWTIPChange, c.AllowJWTIPChange}, } for _, field := range testFields { diff --git a/model/jwt_ip_change_atomic_test.go b/model/jwt_ip_change_atomic_test.go new file mode 100644 index 00000000..bbb2b343 --- /dev/null +++ b/model/jwt_ip_change_atomic_test.go @@ -0,0 +1,56 @@ +package model + +import ( + "path/filepath" + "sync" + "testing" +) + +func TestJWTIPChangeAllowedPersistence(t *testing.T) { + c := &Config{} + if c.JWTIPChangeAllowed() { + t.Fatal("zero-value Config must bind JWT sessions to their IP") + } + // An absent agent secret makes Read save the config during initialization. + path := newTempConfig(t, "jwt_secret_key: test\nallow_jwt_ip_change: true\n") + if err := c.Read(path, nil); err != nil { + t.Fatal(err) + } + if !c.JWTIPChangeAllowed() { + t.Fatal("Read must initialize the atomic mirror before saving") + } + for _, want := range []bool{true, false} { + c.SetJWTIPChangeAllowed(want) + if err := c.Save(); err != nil { + t.Fatal(err) + } + reloaded := &Config{} + if err := reloaded.Read(path, nil); err != nil { + t.Fatal(err) + } + if c.AllowJWTIPChange != want || reloaded.JWTIPChangeAllowed() != want { + t.Fatalf("saved and reloaded setting must be %v", want) + } + } +} + +func TestJWTIPChangeAllowedConcurrentAccess(t *testing.T) { + c := &Config{filePath: filepath.Join(t.TempDir(), "config.yaml")} + var wg sync.WaitGroup + wg.Go(func() { + for i := 0; i < 10000; i++ { + c.SetJWTIPChangeAllowed(i%2 == 0) + } + }) + wg.Go(func() { + for i := 0; i < 10000; i++ { + _ = c.JWTIPChangeAllowed() + } + }) + for i := 0; i < 20; i++ { + if err := c.Save(); err != nil { + t.Error(err) + } + } + wg.Wait() +} diff --git a/model/setting_api.go b/model/setting_api.go index 590afb9e..bd0510a0 100644 --- a/model/setting_api.go +++ b/model/setting_api.go @@ -19,6 +19,7 @@ type SettingForm struct { AgentTLS bool `json:"tls,omitempty" validate:"optional"` EnableIPChangeNotification bool `json:"enable_ip_change_notification,omitempty" validate:"optional"` EnablePlainIPInNotification bool `json:"enable_plain_ip_in_notification,omitempty" validate:"optional"` + AllowJWTIPChange *bool `json:"allow_jwt_ip_change,omitempty" validate:"optional"` EnableMCP *bool `json:"enable_mcp,omitempty" validate:"optional"` } From 1155c1757310c13eb9f5a7eaf1f6dc92ada86c58 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 12 Sep 2026 22:59:36 +0800 Subject: [PATCH 14/48] update contributors[no ci] --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e3cf055..9cdf081a 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ Darc Z. Creling Core F -Coming Wel +NikoCat233 adminsama Acgpiano eya46 @@ -136,6 +136,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ 凌 榆木 93 +Coming Wel :D Esdese GreenTeodoro839 @@ -145,7 +146,6 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ Martijn Lindeman Mingzhuo Yin Nanjing Hopefun Network Technology Co. Ltd. -NikoCat233 Spetrum Tao Chen Tater Li From 06d387779d0dcabf5464e09b4dbad408df526bb9 Mon Sep 17 00:00:00 2001 From: naiba Date: Sun, 13 Sep 2026 00:35:12 +0000 Subject: [PATCH 15/48] fix(tsdb): survive low disk space --- pkg/tsdb/disk_guard_test.go | 124 +++++++++++++++++++++ pkg/tsdb/maintenance.go | 5 +- pkg/tsdb/tsdb.go | 128 +++++++++++++++++++++- pkg/tsdb/writer.go | 23 +++- service/singleton/tsdb.go | 17 ++- service/singleton/tsdb_disk_guard_test.go | 58 ++++++++++ 6 files changed, 341 insertions(+), 14 deletions(-) create mode 100644 pkg/tsdb/disk_guard_test.go create mode 100644 service/singleton/tsdb_disk_guard_test.go diff --git a/pkg/tsdb/disk_guard_test.go b/pkg/tsdb/disk_guard_test.go new file mode 100644 index 00000000..2fa2109a --- /dev/null +++ b/pkg/tsdb/disk_guard_test.go @@ -0,0 +1,124 @@ +package tsdb + +import ( + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testReadOnlyChecker struct { + value atomic.Bool +} + +func (c *testReadOnlyChecker) IsReadOnly() bool { + return c.value.Load() +} + +func newDiskGuardTestDB(t *testing.T) *TSDB { + t.Helper() + db, err := Open(&Config{ + DataPath: filepath.Join(t.TempDir(), "tsdb"), + RetentionDays: 1, + MinFreeDiskSpaceGB: 1, + DedupInterval: time.Second, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, db.Close()) }) + return db +} + +func bufferedRows(w *bufferedWriter) int { + w.mu.Lock() + defer w.mu.Unlock() + return len(w.buffer) +} + +func TestReadOnlyStorageDropsWritesAndKeepsQueriesAvailable(t *testing.T) { + db := newDiskGuardTestDB(t) + checker := &testReadOnlyChecker{} + db.readOnly = checker + originalAddRows := db.addRowsFn + var addRowsCalls atomic.Int32 + db.addRowsFn = func(rows []storage.MetricRow, precisionBits uint8) { + addRowsCalls.Add(1) + originalAddRows(rows, precisionBits) + } + + firstTimestamp := time.Now().Add(-time.Minute) + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: firstTimestamp, + CPU: 10, + })) + db.Flush() + require.Equal(t, int32(1), addRowsCalls.Load()) + + before, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day) + require.NoError(t, err) + require.NotEmpty(t, before) + + checker.value.Store(true) + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: firstTimestamp.Add(2 * time.Second), + CPU: 20, + })) + assert.True(t, db.WritesPaused()) + assert.Zero(t, bufferedRows(db.writer), "read-only writes must not accumulate in memory") + assert.Equal(t, int32(1), addRowsCalls.Load(), "read-only samples must not reach VictoriaMetrics") + + afterDrop, err := db.QueryServerMetrics(1, MetricServerCPU, Period1Day) + require.NoError(t, err) + assert.Equal(t, before, afterDrop, "read-only mode must preserve existing query access") + + checker.value.Store(false) + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: firstTimestamp.Add(4 * time.Second), + CPU: 30, + })) + db.Flush() + assert.False(t, db.WritesPaused()) + assert.Equal(t, int32(2), addRowsCalls.Load(), "writes must resume after storage leaves read-only mode") +} + +func TestDiskFullAddRowsPanicPausesWrites(t *testing.T) { + db := newDiskGuardTestDB(t) + db.addRowsFn = func([]storage.MetricRow, uint8) { + panic("write data: no space left on device") + } + + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: time.Now(), + CPU: 10, + })) + assert.NotPanics(t, db.Flush) + assert.True(t, db.WritesPaused()) + + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: time.Now().Add(time.Second), + CPU: 20, + })) + assert.Zero(t, bufferedRows(db.writer)) +} + +func TestNonDiskStoragePanicIsNotHidden(t *testing.T) { + db := newDiskGuardTestDB(t) + db.addRowsFn = func([]storage.MetricRow, uint8) { + panic("storage invariant failed") + } + + require.NoError(t, db.WriteServerMetrics(&ServerMetrics{ + ServerID: 1, + Timestamp: time.Now(), + CPU: 10, + })) + assert.PanicsWithValue(t, "storage invariant failed", db.Flush) +} diff --git a/pkg/tsdb/maintenance.go b/pkg/tsdb/maintenance.go index fe9c73e1..4f2c4a66 100644 --- a/pkg/tsdb/maintenance.go +++ b/pkg/tsdb/maintenance.go @@ -10,8 +10,11 @@ func (db *TSDB) Maintenance() { if db.closed { return } + if !db.acceptsWrites() { + return + } log.Println("NEZHA>> TSDB starting maintenance (flush)...") - db.storage.DebugFlush() + db.debugFlushSafely() log.Println("NEZHA>> TSDB maintenance completed") } diff --git a/pkg/tsdb/tsdb.go b/pkg/tsdb/tsdb.go index d3627b61..9ae4a16b 100644 --- a/pkg/tsdb/tsdb.go +++ b/pkg/tsdb/tsdb.go @@ -1,15 +1,26 @@ package tsdb import ( + "errors" "fmt" "log" "path/filepath" + "strings" "sync" + "sync/atomic" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/storage" ) +// ErrDiskFull is returned when VictoriaMetrics cannot initialize because the +// filesystem containing the TSDB has run out of space. +var ErrDiskFull = errors.New("TSDB disk is full") + +type readOnlyChecker interface { + IsReadOnly() bool +} + // TSDB 封装 VictoriaMetrics 存储 type TSDB struct { storage *storage.Storage @@ -18,6 +29,13 @@ type TSDB struct { closed bool writer *bufferedWriter + + readOnly readOnlyChecker + addRowsFn func([]storage.MetricRow, uint8) + debugFlushFn func() + + readOnlyObserved atomic.Bool + diskFullPaused atomic.Bool } // InitGlobalSettings 初始化 VictoriaMetrics 包级别的全局设置。 @@ -35,7 +53,18 @@ func InitGlobalSettings(config *Config) { } // Open 打开或创建 TSDB 存储 -func Open(config *Config) (*TSDB, error) { +func Open(config *Config) (db *TSDB, err error) { + defer func() { + if recovered := recover(); recovered != nil { + if diskErr := diskFullPanicError(recovered); diskErr != nil { + db = nil + err = diskErr + return + } + panic(recovered) + } + }() + if config == nil { config = DefaultConfig() } @@ -59,9 +88,12 @@ func Open(config *Config) (*TSDB, error) { stor := storage.MustOpenStorage(dataPath, opts) - db := &TSDB{ - storage: stor, - config: config, + db = &TSDB{ + storage: stor, + config: config, + readOnly: stor, + addRowsFn: stor.AddRows, + debugFlushFn: stor.DebugFlush, } db.writer = newBufferedWriter(db, config.WriteBufferSize, config.WriteBufferFlushInterval) @@ -72,6 +104,86 @@ func Open(config *Config) (*TSDB, error) { return db, nil } +func diskFullPanicError(recovered any) error { + message := strings.ToLower(fmt.Sprint(recovered)) + for _, marker := range []string{ + "no space left on device", + "disk quota exceeded", + "not enough space on the disk", + } { + if strings.Contains(message, marker) { + return fmt.Errorf("%w: %v", ErrDiskFull, recovered) + } + } + return nil +} + +// WritesPaused reports whether new samples are currently being discarded. +// Queries remain available while VictoriaMetrics is in read-only mode. +func (db *TSDB) WritesPaused() bool { + if db.diskFullPaused.Load() { + return true + } + return db.readOnly != nil && db.readOnly.IsReadOnly() +} + +func (db *TSDB) acceptsWrites() bool { + if db.diskFullPaused.Load() { + return false + } + + readOnly := db.readOnly != nil && db.readOnly.IsReadOnly() + if readOnly { + if db.readOnlyObserved.CompareAndSwap(false, true) { + log.Println("NEZHA>> TSDB writes paused because free disk space is below the configured limit; dashboard, queries, and alerts remain available") + } + return false + } + + if db.readOnlyObserved.CompareAndSwap(true, false) { + log.Println("NEZHA>> TSDB writes resumed after free disk space recovered") + } + return true +} + +func (db *TSDB) pauseWritesAfterDiskFull(recovered any) { + if db.diskFullPaused.CompareAndSwap(false, true) { + log.Printf("NEZHA>> TSDB disk is full; disabling TSDB writes until dashboard restart while keeping dashboard, queries, and alerts running: %v", recovered) + } +} + +func (db *TSDB) addRowsSafely(rows []storage.MetricRow) { + if len(rows) == 0 || !db.acceptsWrites() { + return + } + + defer func() { + if recovered := recover(); recovered != nil { + if diskFullPanicError(recovered) == nil { + panic(recovered) + } + db.pauseWritesAfterDiskFull(recovered) + } + }() + db.addRowsFn(rows, 64) +} + +func (db *TSDB) debugFlushSafely() { + if !db.acceptsWrites() { + return + } + + defer func() { + if recovered := recover(); recovered != nil { + if diskFullPanicError(recovered) == nil { + panic(recovered) + } + db.pauseWritesAfterDiskFull(recovered) + } + }() + db.debugFlushFn() +} + // Close 关闭 TSDB 存储 func (db *TSDB) Close() error { db.mu.Lock() @@ -110,8 +222,14 @@ func (db *TSDB) IsClosed() bool { // Flush 强制刷盘（主要用于测试） func (db *TSDB) Flush() { + db.mu.RLock() + defer db.mu.RUnlock() + if db.closed { + return + } + if db.writer != nil { db.writer.flush() } - db.storage.DebugFlush() + db.debugFlushSafely() } diff --git a/pkg/tsdb/writer.go b/pkg/tsdb/writer.go index 7896c094..d6471e2e 100644 --- a/pkg/tsdb/writer.go +++ b/pkg/tsdb/writer.go @@ -47,18 +47,29 @@ func (w *bufferedWriter) flushLoop() { } func (w *bufferedWriter) write(rows []storage.MetricRow) { + if !w.db.acceptsWrites() { + w.discard() + return + } + w.mu.Lock() w.buffer = append(w.buffer, rows...) if len(w.buffer) >= w.maxSize { rows := w.buffer w.buffer = make([]storage.MetricRow, 0, w.maxSize) w.mu.Unlock() - w.db.storage.AddRows(rows, 64) + w.db.addRowsSafely(rows) return } w.mu.Unlock() } +func (w *bufferedWriter) discard() { + w.mu.Lock() + w.buffer = make([]storage.MetricRow, 0, w.maxSize) + w.mu.Unlock() +} + func (w *bufferedWriter) flush() { w.mu.Lock() if len(w.buffer) == 0 { @@ -69,7 +80,7 @@ func (w *bufferedWriter) flush() { w.buffer = make([]storage.MetricRow, 0, w.maxSize) w.mu.Unlock() - w.db.storage.AddRows(rows, 64) + w.db.addRowsSafely(rows) } func (w *bufferedWriter) stop() { @@ -171,7 +182,7 @@ func (db *TSDB) WriteServerMetrics(m *ServerMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } @@ -200,7 +211,7 @@ func (db *TSDB) WriteServiceMetrics(m *ServiceMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } @@ -265,7 +276,7 @@ func (db *TSDB) WriteBatchServerMetrics(metrics []*ServerMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } @@ -295,7 +306,7 @@ func (db *TSDB) WriteBatchServiceMetrics(metrics []*ServiceMetrics) error { if db.writer != nil { db.writer.write(rows) } else { - db.storage.AddRows(rows, 64) + db.addRowsSafely(rows) } return nil } diff --git a/service/singleton/tsdb.go b/service/singleton/tsdb.go index 745e83d6..e5311bb1 100644 --- a/service/singleton/tsdb.go +++ b/service/singleton/tsdb.go @@ -1,6 +1,7 @@ package singleton import ( + "errors" "log" "time" @@ -10,6 +11,8 @@ import ( var TSDBShared *tsdb.TSDB +var openTSDB = tsdb.Open + func InitTSDB() error { config := &tsdb.Config{ RetentionDays: 30, @@ -44,11 +47,21 @@ func InitTSDB() error { return nil } - var err error - TSDBShared, err = tsdb.Open(config) + TSDBShared = nil + db, err := openTSDB(config) if err != nil { + if errors.Is(err, tsdb.ErrDiskFull) { + log.Printf("NEZHA>> Warning: TSDB is unavailable because its disk is full; dashboard and alerts will continue without TSDB writes: %v", err) + if DB != nil { + if migrateErr := DB.AutoMigrate(model.ServiceHistory{}); migrateErr != nil { + log.Printf("NEZHA>> Warning: failed to prepare SQLite service history fallback: %v", migrateErr) + } + } + return nil + } return err } + TSDBShared = db log.Println("NEZHA>> TSDB initialized successfully") diff --git a/service/singleton/tsdb_disk_guard_test.go b/service/singleton/tsdb_disk_guard_test.go new file mode 100644 index 00000000..50f71f3f --- /dev/null +++ b/service/singleton/tsdb_disk_guard_test.go @@ -0,0 +1,58 @@ +package singleton + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/pkg/tsdb" +) + +func TestInitTSDBContinuesWhenDiskIsFull(t *testing.T) { + originalConf := Conf + originalDB := DB + originalShared := TSDBShared + originalOpen := openTSDB + t.Cleanup(func() { + Conf = originalConf + DB = originalDB + TSDBShared = originalShared + openTSDB = originalOpen + }) + + Conf = &ConfigClass{Config: &model.Config{ + TSDB: model.TSDBConf{DataPath: "/full/tsdb"}, + }} + DB = nil + TSDBShared = nil + openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) { + return nil, fmt.Errorf("open failed: %w", tsdb.ErrDiskFull) + } + + require.NoError(t, InitTSDB()) + require.Nil(t, TSDBShared) +} + +func TestInitTSDBStillReturnsNonDiskErrors(t *testing.T) { + originalConf := Conf + originalShared := TSDBShared + originalOpen := openTSDB + t.Cleanup(func() { + Conf = originalConf + TSDBShared = originalShared + openTSDB = originalOpen + }) + + Conf = &ConfigClass{Config: &model.Config{ + TSDB: model.TSDBConf{DataPath: "/broken/tsdb"}, + }} + TSDBShared = nil + openTSDB = func(*tsdb.Config) (*tsdb.TSDB, error) { + return nil, fmt.Errorf("permission denied") + } + + require.ErrorContains(t, InitTSDB(), "permission denied") + require.Nil(t, TSDBShared) +} From f2f10ea7bc079d8250a108a05ac549ba469a669e Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Sun, 13 Sep 2026 20:55:03 +0800 Subject: [PATCH 16/48] Update ddns.go --- pkg/ddns/ddns.go | 180 +++++++++++++++++++++++++++-------------------- 1 file changed, 103 insertions(+), 77 deletions(-) diff --git a/pkg/ddns/ddns.go b/pkg/ddns/ddns.go index 8abecad3..b0bde582 100644 --- a/pkg/ddns/ddns.go +++ b/pkg/ddns/ddns.go @@ -35,57 +35,62 @@ func (provider *Provider) GetProfileID() uint64 { } func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...string) { - for _, domain := range utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains) { - for retries := range int(provider.DDNSProfile.MaxRetries) { - log.Printf("NEZHA>> Updating DNS Record of domain %s: %d/%d", domain, retries+1, provider.DDNSProfile.MaxRetries) - if err := provider.updateDomain(ctx, domain); err != nil { - log.Printf("NEZHA>> Failed to update DNS record of domain %s: %v", domain, err) - } else { - log.Printf("NEZHA>> Update DNS record of domain %s succeeded", domain) - break + domains := utils.IfOr(len(overrideDomains) > 0, overrideDomains, provider.DDNSProfile.Domains) + maxRetries := int(provider.DDNSProfile.MaxRetries) + if maxRetries <= 0 { + maxRetries = 1 + } + + for _, domain := range domains { + var err error + provider.prefix, provider.zone, err = provider.splitDomainSOA(ctx, domain) + if err != nil { + log.Printf("NEZHA>> Failed to split domain SOA for %s: %v", domain, err) + continue + } + + // 独立处理 IPv4 更新或删除 + if provider.DDNSProfile.EnableIPv4 != nil && *provider.DDNSProfile.EnableIPv4 { + for retries := 0; retries < maxRetries; retries++ { + log.Printf("NEZHA>> Updating IPv4 record of domain %s: %d/%d", domain, retries+1, maxRetries) + var ipv4Err error + if provider.IPAddrs.IPv4Addr == "" { + ipv4Err = provider.deleteDomainRecord(ctx, "A") + } else { + ipv4Err = provider.addDomainRecord(ctx, "A", provider.IPAddrs.IPv4Addr) + } + + if ipv4Err != nil { + log.Printf("NEZHA>> Failed to update IPv4 record of domain %s: %v", domain, ipv4Err) + } else { + log.Printf("NEZHA>> Update IPv4 record of domain %s succeeded", domain) + break + } + } + } + + // 独立处理 IPv6 更新或删除 + if provider.DDNSProfile.EnableIPv6 != nil && *provider.DDNSProfile.EnableIPv6 { + for retries := 0; retries < maxRetries; retries++ { + log.Printf("NEZHA>> Updating IPv6 record of domain %s: %d/%d", domain, retries+1, maxRetries) + var ipv6Err error + if provider.IPAddrs.IPv6Addr == "" { + ipv6Err = provider.deleteDomainRecord(ctx, "AAAA") + } else { + ipv6Err = provider.addDomainRecord(ctx, "AAAA", provider.IPAddrs.IPv6Addr) + } + + if ipv6Err != nil { + log.Printf("NEZHA>> Failed to update IPv6 record of domain %s: %v", domain, ipv6Err) + } else { + log.Printf("NEZHA>> Update IPv6 record of domain %s succeeded", domain) + break + } } } } } -func (provider *Provider) updateDomain(ctx context.Context, domain string) error { - var err error - provider.prefix, provider.zone, err = provider.splitDomainSOA(ctx, domain) - if err != nil { - return err - } - - // 独立处理 IPv4 更新或删除 - if *provider.DDNSProfile.EnableIPv4 { - if provider.IPAddrs.IPv4Addr == "" { - log.Printf("NEZHA>> IPv4 address is empty for domain %s, deleting A record...", domain) - if err = provider.deleteDomainRecord(ctx, "A"); err != nil { - return fmt.Errorf("failed to delete IPv4 record: %w", err) - } - } else { - if err = provider.addDomainRecord(ctx, "A", provider.IPAddrs.IPv4Addr); err != nil { - return err - } - } - } - - // 独立处理 IPv6 更新或删除 - if *provider.DDNSProfile.EnableIPv6 { - if provider.IPAddrs.IPv6Addr == "" { - log.Printf("NEZHA>> IPv6 address is empty for domain %s, deleting AAAA record...", domain) - if err = provider.deleteDomainRecord(ctx, "AAAA"); err != nil { - return fmt.Errorf("failed to delete IPv6 record: %w", err) - } - } else { - if err = provider.addDomainRecord(ctx, "AAAA", provider.IPAddrs.IPv6Addr); err != nil { - return err - } - } - } - - return nil -} - func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr string) error { netipAddr, err := netip.ParseAddr(addr) if err != nil { @@ -103,68 +108,90 @@ func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr str return err } -// deleteDomainRecord 用于删除指定类型的解析记录 +// deleteDomainRecord 用于安全删除指定类型的解析记录 func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string) error { - // 检查 libdns.RecordGetter 和 libdns.RecordDeleter - getter, ok := provider.Setter.(libdns.RecordGetter) - if !ok { - return fmt.Errorf("dns provider does not support record getting") - } + // 同时断言 RecordGetter 与 RecordDeleter，防止提供商因缺少接口导致断言错误 + getter, okGetter := provider.Setter.(libdns.RecordGetter) + deleter, okDeleter := provider.Setter.(libdns.RecordDeleter) - deleter, ok := provider.Setter.(libdns.RecordDeleter) - if !ok { - return fmt.Errorf("dns provider does not support record deletion") - } - - // 获取 DNS 记录 - allRecords, err := getter.GetRecords(ctx, provider.zone) - if err != nil { - return fmt.Errorf("failed to get DNS records: %w", err) + if !okGetter || !okDeleter { + log.Printf("NEZHA>> DNS provider does not support record getting or deletion, safely skipping deletion for %s", recType) + return nil } cleanName := func(name string) string { return strings.ToLower(strings.TrimSuffix(name, ".")) } - cleanPrefix := cleanName(provider.prefix) - cleanZone := cleanName(provider.zone) targetRecType := strings.ToUpper(recType) - // 筛选和匹配 + // 通过特征匹配识别 Hurricane Electric (HE) 等特殊供应商 + providerType := strings.ToLower(fmt.Sprintf("%T", provider.Setter)) + isHeProvider := strings.Contains(providerType, "he") || + strings.Contains(providerType, "hurricane") || + strings.Contains(providerType, "dns.he") + + // 针对 HE 供应商：仅返回根域（@），无法枚举子域名 + if isHeProvider { + log.Printf("NEZHA>> Provider identified as Hurricane Electric (HE) variant, using direct set deletion path for %s.%s", provider.prefix, provider.zone) + var dummyIP netip.Addr + if targetRecType == "A" { + dummyIP = netip.MustParseAddr("0.0.0.0") + } else { + dummyIP = netip.MustParseAddr("::") + } + _, err := provider.Setter.SetRecords(ctx, provider.zone, + []libdns.Record{ + libdns.Address{ + Name: provider.prefix, + IP: dummyIP, + TTL: time.Minute, + }, + }) + return err + } + + // 获取 DNS 记录（针对标准提供商，如 Cloudflare） + allRecords, err := getter.GetRecords(ctx, provider.zone) + if err != nil { + return fmt.Errorf("failed to get DNS records: %w", err) + } + + cleanZone := cleanName(provider.zone) + + // 筛选和匹配目标记录 var targetRecords []libdns.Record for _, rec := range allRecords { rr := rec.RR() - recName := rr.Name - currentType := strings.ToUpper(rr.Type) - - if currentType != targetRecType { + if strings.ToUpper(rr.Type) != targetRecType { continue } - relName := libdns.RelativeName(recName, provider.zone) + relName := libdns.RelativeName(rr.Name, provider.zone) cleanRel := cleanName(relName) - cleanRRName := cleanName(recName) + cleanRRName := cleanName(rr.Name) + + isApex := cleanRel == "" || cleanRel == "@" || cleanRRName == cleanZone var matchedName bool if cleanPrefix == "" { - matchedName = cleanRel == "" || cleanRel == "@" || cleanRRName == cleanZone + matchedName = isApex } else { matchedName = cleanRel == cleanPrefix || cleanRRName == cleanPrefix } - // 保留原始的 rec 接口（包含 Cloudflare 所需的 ProviderData 元数据），用于精准删除 if matchedName { targetRecords = append(targetRecords, rec) } } - // 若未找到对应记录，则直接返回成功（视为已删除或不存在） + // 若未找到对应记录（例如已经成功删除），则直接返回成功 if len(targetRecords) == 0 { - log.Printf("NEZHA>> No matching %s record found for deletion under zone %s", recType, provider.zone) + log.Printf("NEZHA>> No matching %s record found for deletion under zone %s, already clean", recType, provider.zone) return nil } - // 执行删除操作 + // 执行实际删除操作 _, err = deleter.DeleteRecords(ctx, provider.zone, targetRecords) if err != nil { return fmt.Errorf("deleter.DeleteRecords failed: %w", err) @@ -174,7 +201,7 @@ func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string return nil } -func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (prefix string, zone string, err error) { +func (provider *Provider) splitDomainSOA(ctx context.Context, domain string, overrideServers ...string) (prefix string, zone string, err error) { c := &dns.Client{Timeout: dnsTimeOut} domain += "." @@ -201,7 +228,6 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (pr if soa, ok := r.Answer[0].(*dns.SOA); ok { zone := soa.Hdr.Name prefix := libdns.RelativeName(domain, zone) - // Convert "@" to empty string for zone apex if prefix == "@" { prefix = "" } From cbc583dd3a5f8dd78d69d8e0934f2b117a04d07d Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:39:22 +0800 Subject: [PATCH 17/48] Update ddns.go --- pkg/ddns/ddns.go | 115 ++++++++++++++++++++++------------------------- 1 file changed, 54 insertions(+), 61 deletions(-) diff --git a/pkg/ddns/ddns.go b/pkg/ddns/ddns.go index b0bde582..b26ee47d 100644 --- a/pkg/ddns/ddns.go +++ b/pkg/ddns/ddns.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/libdns/he" "github.com/libdns/libdns" "github.com/miekg/dns" @@ -22,9 +23,6 @@ const ( ) type Provider struct { - prefix string - zone string - DDNSProfile *model.DDNSProfile IPAddrs *model.IP Setter libdns.RecordSetter @@ -42,10 +40,19 @@ func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...s } for _, domain := range domains { - var err error - provider.prefix, provider.zone, err = provider.splitDomainSOA(ctx, domain) - if err != nil { - log.Printf("NEZHA>> Failed to split domain SOA for %s: %v", domain, err) + var prefix, zone string + var soaErr error + + for retries := 0; retries < maxRetries; retries++ { + prefix, zone, soaErr = provider.splitDomainSOA(ctx, domain) + if soaErr == nil { + break + } + log.Printf("NEZHA>> Failed to split domain SOA for %s (attempt %d/%d): %v", domain, retries+1, maxRetries, soaErr) + } + + if soaErr != nil { + log.Printf("NEZHA>> Failed to split domain SOA for %s after %d retries, skipping domain", domain, maxRetries) continue } @@ -55,9 +62,9 @@ func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...s log.Printf("NEZHA>> Updating IPv4 record of domain %s: %d/%d", domain, retries+1, maxRetries) var ipv4Err error if provider.IPAddrs.IPv4Addr == "" { - ipv4Err = provider.deleteDomainRecord(ctx, "A") + ipv4Err = provider.deleteDomainRecord(ctx, prefix, zone, "A") } else { - ipv4Err = provider.addDomainRecord(ctx, "A", provider.IPAddrs.IPv4Addr) + ipv4Err = provider.addDomainRecord(ctx, prefix, zone, "A", provider.IPAddrs.IPv4Addr) } if ipv4Err != nil { @@ -75,9 +82,9 @@ func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...s log.Printf("NEZHA>> Updating IPv6 record of domain %s: %d/%d", domain, retries+1, maxRetries) var ipv6Err error if provider.IPAddrs.IPv6Addr == "" { - ipv6Err = provider.deleteDomainRecord(ctx, "AAAA") + ipv6Err = provider.deleteDomainRecord(ctx, prefix, zone, "AAAA") } else { - ipv6Err = provider.addDomainRecord(ctx, "AAAA", provider.IPAddrs.IPv6Addr) + ipv6Err = provider.addDomainRecord(ctx, prefix, zone, "AAAA", provider.IPAddrs.IPv6Addr) } if ipv6Err != nil { @@ -91,16 +98,16 @@ func (provider *Provider) UpdateDomain(ctx context.Context, overrideDomains ...s } } -func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr string) error { +func (provider *Provider) addDomainRecord(ctx context.Context, prefix, zone, recType, addr string) error { netipAddr, err := netip.ParseAddr(addr) if err != nil { return fmt.Errorf("parse error: %v", err) } - _, err = provider.Setter.SetRecords(ctx, provider.zone, + _, err = provider.Setter.SetRecords(ctx, zone, []libdns.Record{ libdns.Address{ - Name: provider.prefix, + Name: prefix, IP: netipAddr, TTL: time.Minute, }, @@ -109,8 +116,25 @@ func (provider *Provider) addDomainRecord(ctx context.Context, recType, addr str } // deleteDomainRecord 用于安全删除指定类型的解析记录 -func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string) error { - // 同时断言 RecordGetter 与 RecordDeleter，防止提供商因缺少接口导致断言错误 +func (provider *Provider) deleteDomainRecord(ctx context.Context, prefix, zone, recType string) error { + targetRecType := strings.ToUpper(recType) + + // 针对 HE 服务商使用标准的 libdns.RR + if _, ok := provider.Setter.(*he.Provider); ok { + deleter, okDeleter := provider.Setter.(libdns.RecordDeleter) + if !okDeleter { + return fmt.Errorf("he provider does not implement RecordDeleter") + } + _, err := deleter.DeleteRecords(ctx, zone, []libdns.Record{ + libdns.RR{ + Name: prefix, + Type: targetRecType, + }, + }) + return err + } + + // 检查接口 getter, okGetter := provider.Setter.(libdns.RecordGetter) deleter, okDeleter := provider.Setter.(libdns.RecordDeleter) @@ -122,44 +146,15 @@ func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string cleanName := func(name string) string { return strings.ToLower(strings.TrimSuffix(name, ".")) } - cleanPrefix := cleanName(provider.prefix) - targetRecType := strings.ToUpper(recType) - - // 通过特征匹配识别 Hurricane Electric (HE) 等特殊供应商 - providerType := strings.ToLower(fmt.Sprintf("%T", provider.Setter)) - isHeProvider := strings.Contains(providerType, "he") || - strings.Contains(providerType, "hurricane") || - strings.Contains(providerType, "dns.he") - - // 针对 HE 供应商：仅返回根域（@），无法枚举子域名 - if isHeProvider { - log.Printf("NEZHA>> Provider identified as Hurricane Electric (HE) variant, using direct set deletion path for %s.%s", provider.prefix, provider.zone) - var dummyIP netip.Addr - if targetRecType == "A" { - dummyIP = netip.MustParseAddr("0.0.0.0") - } else { - dummyIP = netip.MustParseAddr("::") - } - _, err := provider.Setter.SetRecords(ctx, provider.zone, - []libdns.Record{ - libdns.Address{ - Name: provider.prefix, - IP: dummyIP, - TTL: time.Minute, - }, - }) - return err - } + cleanPrefix := cleanName(prefix) // 获取 DNS 记录（针对标准提供商，如 Cloudflare） - allRecords, err := getter.GetRecords(ctx, provider.zone) + allRecords, err := getter.GetRecords(ctx, zone) if err != nil { return fmt.Errorf("failed to get DNS records: %w", err) } - cleanZone := cleanName(provider.zone) - - // 筛选和匹配目标记录 + cleanZone := cleanName(zone) var targetRecords []libdns.Record for _, rec := range allRecords { rr := rec.RR() @@ -167,7 +162,7 @@ func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string continue } - relName := libdns.RelativeName(rr.Name, provider.zone) + relName := libdns.RelativeName(rr.Name, zone) cleanRel := cleanName(relName) cleanRRName := cleanName(rr.Name) @@ -187,28 +182,26 @@ func (provider *Provider) deleteDomainRecord(ctx context.Context, recType string // 若未找到对应记录（例如已经成功删除），则直接返回成功 if len(targetRecords) == 0 { - log.Printf("NEZHA>> No matching %s record found for deletion under zone %s, already clean", recType, provider.zone) + log.Printf("NEZHA>> No matching %s record found for deletion under zone %s, already clean", recType, zone) return nil } - // 执行实际删除操作 - _, err = deleter.DeleteRecords(ctx, provider.zone, targetRecords) + _, err = deleter.DeleteRecords(ctx, zone, targetRecords) if err != nil { return fmt.Errorf("deleter.DeleteRecords failed: %w", err) } - log.Printf("NEZHA>> Successfully deleted %d matching %s record(s) for %s.%s", len(targetRecords), recType, provider.prefix, provider.zone) + log.Printf("NEZHA>> Successfully deleted %d matching %s record(s) for %s.%s", len(targetRecords), recType, prefix, zone) return nil } -func (provider *Provider) splitDomainSOA(ctx context.Context, domain string, overrideServers ...string) (prefix string, zone string, err error) { +func (provider *Provider) splitDomainSOA(ctx context.Context, domain string) (prefix string, zone string, err error) { c := &dns.Client{Timeout: dnsTimeOut} domain += "." indexes := dns.Split(domain) servers := utils.DNSServers - customDNSServers, _ := ctx.Value(DNSServerKey{}).([]string) if len(customDNSServers) > 0 { servers = customDNSServers @@ -224,14 +217,14 @@ func (provider *Provider) splitDomainSOA(ctx context.Context, domain string, ove continue } - if len(r.Answer) > 0 { + if r != nil && len(r.Answer) > 0 { if soa, ok := r.Answer[0].(*dns.SOA); ok { - zone := soa.Hdr.Name - prefix := libdns.RelativeName(domain, zone) - if prefix == "@" { - prefix = "" + zoneName := soa.Hdr.Name + pfx := libdns.RelativeName(domain, zoneName) + if pfx == "@" { + pfx = "" } - return prefix, zone, nil + return pfx, zoneName, nil } } } From 892619c95ced7b5e560bb6f978ead4c6bf756bb4 Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:39:52 +0800 Subject: [PATCH 18/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 135 +++++++++++++++++++++++++++++++----------- 1 file changed, 101 insertions(+), 34 deletions(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 70d7cb9c..c2d3d327 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -3,45 +3,112 @@ package ddns import ( "context" "testing" + "time" + + "github.com/libdns/he" + "github.com/libdns/libdns" + "github.com/nezhahq/nezha/model" ) -type testSt struct { - domain string - zone string - prefix string +type MockSetter struct { + Records []libdns.Record + SetCalled bool + GetCalled bool + DelCalled bool + DeleteError error } -func TestSplitDomainSOA(t *testing.T) { - cases := []testSt{ - { - domain: "www.example.co.uk", - zone: "example.co.uk.", - prefix: "www", - }, - { - domain: "abc.example.com", - zone: "example.com.", - prefix: "abc", - }, - { - domain: "example.com", - zone: "example.com.", - prefix: "", - }, +func (m *MockSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + m.SetCalled = true + m.Records = append(m.Records, records...) + return records, nil +} + +func (m *MockSetter) GetRecords(ctx context.Context, zone string) ([]libdns.Record, error) { + m.GetCalled = true + return m.Records, nil +} + +func (m *MockSetter) DeleteRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + m.DelCalled = true + if m.DeleteError != nil { + return nil, m.DeleteError + } + m.Records = nil + return records, nil +} + +type MockUnsupportedSetter struct{} + +func (m *MockUnsupportedSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + return records, nil +} + +func TestUnsupportedGetterDeleterSkip(t *testing.T) { + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{MaxRetries: 1}, + IPAddrs: &model.IP{IPv4Addr: ""}, + Setter: &MockUnsupportedSetter{}, } - ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{"1.1.1.1:53"}) - provider := &Provider{} - for _, c := range cases { - prefix, zone, err := provider.splitDomainSOA(ctx, c.domain) - if err != nil { - t.Fatalf("Error: %s", err) - } - if prefix != c.prefix { - t.Fatalf("Expected prefix %s, but got %s", c.prefix, prefix) - } - if zone != c.zone { - t.Fatalf("Expected zone %s, but got %s", c.zone, zone) - } + err := provider.deleteDomainRecord(context.Background(), "sub", "example.com", "A") + if err != nil { + t.Fatalf("expected no error for unsupported setter deletion skip, got %v", err) + } +} + +func TestIndependentDualStackProcessing(t *testing.T) { + enableIPv4 := true + enableIPv6 := true + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{ + MaxRetries: 1, + EnableIPv4: &enableIPv4, + EnableIPv6: &enableIPv6, + }, + IPAddrs: &model.IP{ + IPv4Addr: "1.1.1.1", + IPv6Addr: "", + }, + Setter: &MockSetter{}, + } + + if provider.DDNSProfile.EnableIPv4 == nil || !*provider.DDNSProfile.EnableIPv4 { + t.Fatalf("IPv4 should be enabled") + } +} + +func TestHEDirectDeletionPath(t *testing.T) { + heProvider := &he.Provider{APIKey: "test-key"} + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{MaxRetries: 1}, + IPAddrs: &model.IP{IPv4Addr: ""}, + Setter: heProvider, + } + + err := provider.deleteDomainRecord(context.Background(), "test", "example.com", "A") + if err == nil { + t.Fatalf("expected an error due to invalid test API key in HE direct path, but got nil") + } +} + +func TestTransientSOAFailureAndRetry(t *testing.T) { + maxRetries := 3 + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{ + MaxRetries: int32(maxRetries), + Domains: []string{"notfound.invalid"}, + }, + IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, + Setter: &MockSetter{}, + } + + ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{"192.0.2.1:53"}) + start := time.Now() + provider.UpdateDomain(ctx) + duration := time.Since(start) + + if duration > 35*time.Second { + t.Fatalf("SOA retry loop took too long: %v", duration) } } From dd0b598dc3eca902f219715b4fc13a6044e9241c Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:50:08 +0800 Subject: [PATCH 19/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index c2d3d327..adbe169b 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -96,7 +96,7 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { maxRetries := 3 provider := &Provider{ DDNSProfile: &model.DDNSProfile{ - MaxRetries: int32(maxRetries), + MaxRetries: int64(maxRetries), Domains: []string{"notfound.invalid"}, }, IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, From 0a71cbdb6379e2f4920463190a3b15752965bd1f Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 08:56:04 +0800 Subject: [PATCH 20/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index adbe169b..06145e5f 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -96,7 +96,7 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { maxRetries := 3 provider := &Provider{ DDNSProfile: &model.DDNSProfile{ - MaxRetries: int64(maxRetries), + MaxRetries: uint64(maxRetries), Domains: []string{"notfound.invalid"}, }, IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, From 02fe54bb803a841f488ab695d6b87b643f1ebcd2 Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:10:17 +0800 Subject: [PATCH 21/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 06145e5f..64dc1f56 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -108,7 +108,7 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { provider.UpdateDomain(ctx) duration := time.Since(start) - if duration > 35*time.Second { + if duration > 180*time.Second { t.Fatalf("SOA retry loop took too long: %v", duration) } } From 19d959c60df5d8208aed473354cb05b536a59a8d Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:02:03 +0800 Subject: [PATCH 22/48] Update ddns.go --- pkg/ddns/ddns.go | 79 ++++++------------------------------------------ 1 file changed, 10 insertions(+), 69 deletions(-) diff --git a/pkg/ddns/ddns.go b/pkg/ddns/ddns.go index b26ee47d..ef93b846 100644 --- a/pkg/ddns/ddns.go +++ b/pkg/ddns/ddns.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/libdns/he" "github.com/libdns/libdns" "github.com/miekg/dns" @@ -115,83 +114,25 @@ func (provider *Provider) addDomainRecord(ctx context.Context, prefix, zone, rec return err } -// deleteDomainRecord 用于安全删除指定类型的解析记录 func (provider *Provider) deleteDomainRecord(ctx context.Context, prefix, zone, recType string) error { - targetRecType := strings.ToUpper(recType) - - // 针对 HE 服务商使用标准的 libdns.RR - if _, ok := provider.Setter.(*he.Provider); ok { - deleter, okDeleter := provider.Setter.(libdns.RecordDeleter) - if !okDeleter { - return fmt.Errorf("he provider does not implement RecordDeleter") - } - _, err := deleter.DeleteRecords(ctx, zone, []libdns.Record{ - libdns.RR{ - Name: prefix, - Type: targetRecType, - }, - }) - return err - } - - // 检查接口 - getter, okGetter := provider.Setter.(libdns.RecordGetter) deleter, okDeleter := provider.Setter.(libdns.RecordDeleter) - - if !okGetter || !okDeleter { - log.Printf("NEZHA>> DNS provider does not support record getting or deletion, safely skipping deletion for %s", recType) + if !okDeleter { + log.Printf("NEZHA>> DNS provider does not support RecordDeleter, safely skipping deletion for %s", recType) return nil } - cleanName := func(name string) string { - return strings.ToLower(strings.TrimSuffix(name, ".")) - } - cleanPrefix := cleanName(prefix) - - // 获取 DNS 记录（针对标准提供商，如 Cloudflare） - allRecords, err := getter.GetRecords(ctx, zone) - if err != nil { - return fmt.Errorf("failed to get DNS records: %w", err) - } - - cleanZone := cleanName(zone) - var targetRecords []libdns.Record - for _, rec := range allRecords { - rr := rec.RR() - if strings.ToUpper(rr.Type) != targetRecType { - continue - } - - relName := libdns.RelativeName(rr.Name, zone) - cleanRel := cleanName(relName) - cleanRRName := cleanName(rr.Name) - - isApex := cleanRel == "" || cleanRel == "@" || cleanRRName == cleanZone - - var matchedName bool - if cleanPrefix == "" { - matchedName = isApex - } else { - matchedName = cleanRel == cleanPrefix || cleanRRName == cleanPrefix - } - - if matchedName { - targetRecords = append(targetRecords, rec) - } - } - - // 若未找到对应记录（例如已经成功删除），则直接返回成功 - if len(targetRecords) == 0 { - log.Printf("NEZHA>> No matching %s record found for deletion under zone %s, already clean", recType, zone) - return nil - } - - _, err = deleter.DeleteRecords(ctx, zone, targetRecords) + targetRecType := strings.ToUpper(recType) + _, err := deleter.DeleteRecords(ctx, zone, []libdns.Record{ + libdns.RR{ + Name: prefix, + Type: targetRecType, + }, + }) if err != nil { return fmt.Errorf("deleter.DeleteRecords failed: %w", err) } - log.Printf("NEZHA>> Successfully deleted %d matching %s record(s) for %s.%s", len(targetRecords), recType, prefix, zone) + log.Printf("NEZHA>> Successfully deleted %s record for %s.%s", recType, prefix, zone) return nil } From f611f0a551bbfd7812b92e38928be1bdd15734a5 Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:04:38 +0800 Subject: [PATCH 23/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 215 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 178 insertions(+), 37 deletions(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 64dc1f56..6ac786ef 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -2,20 +2,97 @@ package ddns import ( "context" + "net" "testing" "time" - "github.com/libdns/he" "github.com/libdns/libdns" + "github.com/miekg/dns" + "github.com/nezhahq/nezha/model" ) +type testSt struct { + domain string + zone string + prefix string +} + +func TestSplitDomainSOA(t *testing.T) { + handler := dns.NewServeMux() + handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { + m := new(dns.Msg) + m.SetReply(r) + if len(r.Question) > 0 { + qname := r.Question[0].Name + soa := &dns.SOA{ + Hdr: dns.RR_Header{Name: qname, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Ns: "ns.example.com.", + Mbox: "admin.example.com.", + Serial: 1, + Refresh: 3600, + Retry: 600, + Expire: 86400, + Minttl: 60, + } + m.Answer = append(m.Answer, soa) + } + _ = w.WriteMsg(m) + }) + + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer pc.Close() + + server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler} + go func() { + _ = server.ActivateAndServe() + }() + defer server.Shutdown() + + addr := pc.LocalAddr().String() + + cases := []testSt{ + { + domain: "www.example.co.uk", + zone: "example.co.uk.", + prefix: "www", + }, + { + domain: "abc.example.com", + zone: "example.com.", + prefix: "abc", + }, + { + domain: "example.com", + zone: "example.com.", + prefix: "", + }, + } + + ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{addr}) + provider := &Provider{} + for _, c := range cases { + prefix, zone, err := provider.splitDomainSOA(ctx, c.domain) + if err != nil { + t.Fatalf("Error for domain %s: %s", c.domain, err) + } + if prefix != c.prefix { + t.Fatalf("Expected prefix %s, but got %s", c.prefix, prefix) + } + if zone != c.zone { + t.Fatalf("Expected zone %s, but got %s", c.zone, zone) + } + } +} + type MockSetter struct { - Records []libdns.Record - SetCalled bool - GetCalled bool - DelCalled bool - DeleteError error + Records []libdns.Record + SetCalled bool + DelCalled bool + DeletedRecords []libdns.Record } func (m *MockSetter) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { @@ -24,17 +101,9 @@ func (m *MockSetter) SetRecords(ctx context.Context, zone string, records []libd return records, nil } -func (m *MockSetter) GetRecords(ctx context.Context, zone string) ([]libdns.Record, error) { - m.GetCalled = true - return m.Records, nil -} - func (m *MockSetter) DeleteRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { m.DelCalled = true - if m.DeleteError != nil { - return nil, m.DeleteError - } - m.Records = nil + m.DeletedRecords = append(m.DeletedRecords, records...) return records, nil } @@ -53,62 +122,134 @@ func TestUnsupportedGetterDeleterSkip(t *testing.T) { err := provider.deleteDomainRecord(context.Background(), "sub", "example.com", "A") if err != nil { - t.Fatalf("expected no error for unsupported setter deletion skip, got %v", err) + t.Fatalf("expected no error for unsupported deleter skip, got %v", err) } } func TestIndependentDualStackProcessing(t *testing.T) { enableIPv4 := true enableIPv6 := true + mockSetter := &MockSetter{} + provider := &Provider{ DDNSProfile: &model.DDNSProfile{ MaxRetries: 1, + Domains: []string{"example.com"}, EnableIPv4: &enableIPv4, EnableIPv6: &enableIPv6, }, IPAddrs: &model.IP{ IPv4Addr: "1.1.1.1", - IPv6Addr: "", + IPv6Addr: "2001:db8::1", }, - Setter: &MockSetter{}, + Setter: mockSetter, } - if provider.DDNSProfile.EnableIPv4 == nil || !*provider.DDNSProfile.EnableIPv4 { - t.Fatalf("IPv4 should be enabled") - } -} + handler := dns.NewServeMux() + handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { + m := new(dns.Msg) + m.SetReply(r) + if len(r.Question) > 0 { + qname := r.Question[0].Name + soa := &dns.SOA{ + Hdr: dns.RR_Header{Name: qname, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + } + m.Answer = append(m.Answer, soa) + } + _ = w.WriteMsg(m) + }) -func TestHEDirectDeletionPath(t *testing.T) { - heProvider := &he.Provider{APIKey: "test-key"} - provider := &Provider{ - DDNSProfile: &model.DDNSProfile{MaxRetries: 1}, - IPAddrs: &model.IP{IPv4Addr: ""}, - Setter: heProvider, + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer pc.Close() + + server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler} + go func() { + _ = server.ActivateAndServe() + }() + defer server.Shutdown() + + ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()}) + provider.UpdateDomain(ctx) + + if !mockSetter.SetCalled { + t.Fatalf("expected SetRecords to have been called for dual stack updates") } - err := provider.deleteDomainRecord(context.Background(), "test", "example.com", "A") - if err == nil { - t.Fatalf("expected an error due to invalid test API key in HE direct path, but got nil") + if len(mockSetter.Records) != 2 { + t.Fatalf("expected 2 records (IPv4 and IPv6) to be set, got %d", len(mockSetter.Records)) } } func TestTransientSOAFailureAndRetry(t *testing.T) { + attempts := 0 + handler := dns.NewServeMux() + handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { + attempts++ + m := new(dns.Msg) + m.SetReply(r) + _ = w.WriteMsg(m) + }) + + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer pc.Close() + + server := &dns.Server{PacketConn: pc, Net: "udp", Handler: handler} + go func() { + _ = server.ActivateAndServe() + }() + defer server.Shutdown() + maxRetries := 3 provider := &Provider{ DDNSProfile: &model.DDNSProfile{ MaxRetries: uint64(maxRetries), - Domains: []string{"notfound.invalid"}, + Domains: []string{"fail.invalid"}, }, IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, Setter: &MockSetter{}, } - ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{"192.0.2.1:53"}) - start := time.Now() + ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()}) provider.UpdateDomain(ctx) - duration := time.Since(start) - if duration > 180*time.Second { - t.Fatalf("SOA retry loop took too long: %v", duration) + if attempts != maxRetries { + t.Fatalf("expected exact attempt count of %d, got %d", maxRetries, attempts) + } +} + +func TestGenericRecordDeleterContract(t *testing.T) { + mockSetter := &MockSetter{} + provider := &Provider{ + DDNSProfile: &model.DDNSProfile{MaxRetries: 1}, + IPAddrs: &model.IP{IPv4Addr: ""}, + Setter: mockSetter, + } + + err := provider.deleteDomainRecord(context.Background(), "sub", "example.com", "A") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !mockSetter.DelCalled { + t.Fatalf("expected DeleteRecords to be called") + } + + if len(mockSetter.DeletedRecords) != 1 { + t.Fatalf("expected 1 record sent for deletion, got %d", len(mockSetter.DeletedRecords)) + } + + rr, ok := mockSetter.DeletedRecords[0].(libdns.RR) + if !ok { + t.Fatalf("expected deletion payload to be libdns.RR") + } + + if rr.Name != "sub" || rr.Type != "A" { + t.Fatalf("unexpected RR fields: Name=%s, Type=%s", rr.Name, rr.Type) } } From ed9f21e4ea56f2e1734a35890937e486197e515d Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:13:10 +0800 Subject: [PATCH 24/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 6ac786ef..379e0df7 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -4,7 +4,6 @@ import ( "context" "net" "testing" - "time" "github.com/libdns/libdns" "github.com/miekg/dns" From 09db8b4361e2c5ccf76c27b9ddd619f0925624aa Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:29:08 +0800 Subject: [PATCH 25/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 47 ++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 379e0df7..5dc430ae 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -3,6 +3,7 @@ package ddns import ( "context" "net" + "strings" "testing" "github.com/libdns/libdns" @@ -24,17 +25,15 @@ func TestSplitDomainSOA(t *testing.T) { m.SetReply(r) if len(r.Question) > 0 { qname := r.Question[0].Name - soa := &dns.SOA{ - Hdr: dns.RR_Header{Name: qname, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, - Ns: "ns.example.com.", - Mbox: "admin.example.com.", - Serial: 1, - Refresh: 3600, - Retry: 600, - Expire: 86400, - Minttl: 60, + if strings.HasSuffix(qname, "example.co.uk.") || strings.HasSuffix(qname, "example.com.") { + soa := &dns.SOA{ + Hdr: dns.RR_Header{Name: qname, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Ns: "ns.example.com.", + Mbox: "admin.example.com.", + Serial: 1, + } + m.Answer = append(m.Answer, soa) } - m.Answer = append(m.Answer, soa) } _ = w.WriteMsg(m) }) @@ -149,9 +148,8 @@ func TestIndependentDualStackProcessing(t *testing.T) { m := new(dns.Msg) m.SetReply(r) if len(r.Question) > 0 { - qname := r.Question[0].Name soa := &dns.SOA{ - Hdr: dns.RR_Header{Name: qname, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Hdr: dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, } m.Answer = append(m.Answer, soa) } @@ -177,8 +175,21 @@ func TestIndependentDualStackProcessing(t *testing.T) { t.Fatalf("expected SetRecords to have been called for dual stack updates") } - if len(mockSetter.Records) != 2 { - t.Fatalf("expected 2 records (IPv4 and IPv6) to be set, got %d", len(mockSetter.Records)) + foundIPv4 := false + foundIPv6 := false + for _, rec := range mockSetter.Records { + if addrRec, ok := rec.(libdns.Address); ok { + if addrRec.IP.String() == "1.1.1.1" { + foundIPv4 = true + } + if addrRec.IP.String() == "2001:db8::1" { + foundIPv6 = true + } + } + } + + if !foundIPv4 || !foundIPv6 { + t.Fatalf("expected both IPv4 and IPv6 records to be processed independently, got records: %v", mockSetter.Records) } } @@ -204,11 +215,11 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { }() defer server.Shutdown() - maxRetries := 3 + var maxRetries uint64 = 3 provider := &Provider{ DDNSProfile: &model.DDNSProfile{ - MaxRetries: uint64(maxRetries), - Domains: []string{"fail.invalid"}, + MaxRetries: maxRetries, + Domains: []string{"fail"}, }, IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, Setter: &MockSetter{}, @@ -217,7 +228,7 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()}) provider.UpdateDomain(ctx) - if attempts != maxRetries { + if uint64(attempts) != maxRetries { t.Fatalf("expected exact attempt count of %d, got %d", maxRetries, attempts) } } From e9ddd46655c735eb118265785ff714e8541958e8 Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:36:04 +0800 Subject: [PATCH 26/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 5dc430ae..46b7ceb7 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -25,9 +25,16 @@ func TestSplitDomainSOA(t *testing.T) { m.SetReply(r) if len(r.Question) > 0 { qname := r.Question[0].Name - if strings.HasSuffix(qname, "example.co.uk.") || strings.HasSuffix(qname, "example.com.") { + var zoneName string + if strings.HasSuffix(qname, "example.co.uk.") { + zoneName = "example.co.uk." + } else if strings.HasSuffix(qname, "example.com.") { + zoneName = "example.com." + } + + if zoneName != "" { soa := &dns.SOA{ - Hdr: dns.RR_Header{Name: qname, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Hdr: dns.RR_Header{Name: zoneName, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, Ns: "ns.example.com.", Mbox: "admin.example.com.", Serial: 1, @@ -219,7 +226,7 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { provider := &Provider{ DDNSProfile: &model.DDNSProfile{ MaxRetries: maxRetries, - Domains: []string{"fail"}, + Domains: []string{"example.com"}, }, IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, Setter: &MockSetter{}, From db78c34b91fa234fe0fd1b9bf6294a902c3997a9 Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:43:36 +0800 Subject: [PATCH 27/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 46b7ceb7..78bb18d9 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -226,7 +226,9 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { provider := &Provider{ DDNSProfile: &model.DDNSProfile{ MaxRetries: maxRetries, - Domains: []string{"example.com"}, + Domains: []string{"sub.example.com"}, + EnableIPv4: &[]bool{true}[0], + EnableIPv6: &[]bool{false}[0], }, IPAddrs: &model.IP{IPv4Addr: "1.1.1.1"}, Setter: &MockSetter{}, From c19e8eb252d36ce1bb0791a0bff8bdac765feee1 Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:51:01 +0800 Subject: [PATCH 28/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 78bb18d9..4bbf31a0 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -23,6 +23,7 @@ func TestSplitDomainSOA(t *testing.T) { handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { m := new(dns.Msg) m.SetReply(r) + m.Authoritative = true if len(r.Question) > 0 { qname := r.Question[0].Name var zoneName string @@ -40,6 +41,7 @@ func TestSplitDomainSOA(t *testing.T) { Serial: 1, } m.Answer = append(m.Answer, soa) + m.Ns = append(m.Ns, soa) } } _ = w.WriteMsg(m) @@ -154,11 +156,16 @@ func TestIndependentDualStackProcessing(t *testing.T) { handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { m := new(dns.Msg) m.SetReply(r) + m.Authoritative = true if len(r.Question) > 0 { soa := &dns.SOA{ - Hdr: dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Hdr: dns.RR_Header{Name: r.Question[0].Name, Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: 300}, + Ns: "ns.example.com.", + Mbox: "admin.example.com.", + Serial: 1, } m.Answer = append(m.Answer, soa) + m.Ns = append(m.Ns, soa) } _ = w.WriteMsg(m) }) @@ -226,7 +233,7 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { provider := &Provider{ DDNSProfile: &model.DDNSProfile{ MaxRetries: maxRetries, - Domains: []string{"sub.example.com"}, + Domains: []string{"example.com"}, EnableIPv4: &[]bool{true}[0], EnableIPv6: &[]bool{false}[0], }, @@ -250,7 +257,8 @@ func TestGenericRecordDeleterContract(t *testing.T) { Setter: mockSetter, } - err := provider.deleteDomainRecord(context.Background(), "sub", "example.com", "A") + ctx := context.Background() + err := provider.deleteDomainRecord(ctx, "sub", "example.com", "A") if err != nil { t.Fatalf("unexpected error: %v", err) } From 39e1c6d1bef7087a3cad8ea87542accb279f6dcd Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:57:37 +0800 Subject: [PATCH 29/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index 4bbf31a0..bad35e75 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -244,8 +244,9 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()}) provider.UpdateDomain(ctx) - if uint64(attempts) != maxRetries { - t.Fatalf("expected exact attempt count of %d, got %d", maxRetries, attempts) + expectedAttempts := int(maxRetries) * 2 + if attempts != expectedAttempts { + t.Fatalf("expected exact attempt count of %d, got %d", expectedAttempts, attempts) } } From fa611ebd9e597cc579017ded4a716131e890b0da Mon Sep 17 00:00:00 2001 From: 77-QiQi <87052253+77-QiQi@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:06:44 +0800 Subject: [PATCH 30/48] Update ddns_test.go --- pkg/ddns/ddns_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/ddns/ddns_test.go b/pkg/ddns/ddns_test.go index bad35e75..dfb0c86f 100644 --- a/pkg/ddns/ddns_test.go +++ b/pkg/ddns/ddns_test.go @@ -4,6 +4,7 @@ import ( "context" "net" "strings" + "sync/atomic" "testing" "github.com/libdns/libdns" @@ -208,10 +209,10 @@ func TestIndependentDualStackProcessing(t *testing.T) { } func TestTransientSOAFailureAndRetry(t *testing.T) { - attempts := 0 + var attempts int64 handler := dns.NewServeMux() handler.HandleFunc(".", func(w dns.ResponseWriter, r *dns.Msg) { - attempts++ + atomic.AddInt64(&attempts, 1) m := new(dns.Msg) m.SetReply(r) _ = w.WriteMsg(m) @@ -244,9 +245,10 @@ func TestTransientSOAFailureAndRetry(t *testing.T) { ctx := context.WithValue(context.Background(), DNSServerKey{}, []string{pc.LocalAddr().String()}) provider.UpdateDomain(ctx) - expectedAttempts := int(maxRetries) * 2 - if attempts != expectedAttempts { - t.Fatalf("expected exact attempt count of %d, got %d", expectedAttempts, attempts) + actualAttempts := atomic.LoadInt64(&attempts) + expectedAttempts := int64(maxRetries) * 2 + if actualAttempts != expectedAttempts { + t.Fatalf("expected exact attempt count of %d, got %d", expectedAttempts, actualAttempts) } } From 589f01f562ad3ce6a80325fc5acabb1a3d1b5443 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 14 Sep 2026 11:23:19 +0800 Subject: [PATCH 31/48] update contributors[no ci] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9cdf081a..b9b4a3b2 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ Weblate (bot) MikoyChinese JackieSung4ev +77-QiQi Kris Lemoe spiritlhl @@ -152,7 +153,6 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ TomyJan Tony ace-consultoria -77-QiQi Chra Mubelotix Ryrenz From 9dab6a5039b5043d5e5127aa08d9614d59cbbcc2 Mon Sep 17 00:00:00 2001 From: naiba Date: Sat, 19 Sep 2026 10:44:17 +0000 Subject: [PATCH 32/48] fix: harden notification cache synchronization --- cmd/dashboard/controller/notification.go | 6 + .../notification_body_limit_test.go | 25 ++++ service/singleton/notification.go | 123 +++++++++-------- service/singleton/notification_test.go | 126 ++++++++++++++++++ 4 files changed, 225 insertions(+), 55 deletions(-) create mode 100644 cmd/dashboard/controller/notification_body_limit_test.go create mode 100644 service/singleton/notification_test.go diff --git a/cmd/dashboard/controller/notification.go b/cmd/dashboard/controller/notification.go index 43403e34..0264b339 100644 --- a/cmd/dashboard/controller/notification.go +++ b/cmd/dashboard/controller/notification.go @@ -1,6 +1,7 @@ package controller import ( + "net/http" "slices" "strconv" @@ -12,6 +13,8 @@ import ( "github.com/nezhahq/nezha/service/singleton" ) +const notificationBatchDeleteMaxBodyBytes = 1 << 20 + // List notification // @Summary List notification // @Security BearerAuth @@ -174,6 +177,9 @@ func updateNotification(c *gin.Context) (any, error) { // @Router /batch-delete/notification [post] func batchDeleteNotification(c *gin.Context) (any, error) { var n []uint64 + if c.Request != nil && c.Request.Body != nil { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, notificationBatchDeleteMaxBodyBytes) + } if err := c.ShouldBindJSON(&n); err != nil { return nil, err } diff --git a/cmd/dashboard/controller/notification_body_limit_test.go b/cmd/dashboard/controller/notification_body_limit_test.go new file mode 100644 index 00000000..4501cee9 --- /dev/null +++ b/cmd/dashboard/controller/notification_body_limit_test.go @@ -0,0 +1,25 @@ +package controller + +import ( + "bytes" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestBatchDeleteNotificationCapsRequestBody(t *testing.T) { + body := append(bytes.Repeat([]byte{' '}, notificationBatchDeleteMaxBodyBytes+1), '[', ']') + req := httptest.NewRequest(http.MethodPost, "/api/v1/batch-delete/notification", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = req + + _, err := batchDeleteNotification(c) + if err == nil || !strings.Contains(err.Error(), "request body too large") { + t.Fatalf("oversized request body was not rejected by MaxBytesReader: %v", err) + } +} diff --git a/service/singleton/notification.go b/service/singleton/notification.go index 0b358ab1..31bf3fe2 100644 --- a/service/singleton/notification.go +++ b/service/singleton/notification.go @@ -81,20 +81,29 @@ func NewNotificationClass() *NotificationClass { } func (c *NotificationClass) Update(n *model.Notification) { - c.listMu.Lock() + func() { + c.listMu.Lock() + defer c.listMu.Unlock() - _, ok := c.list[n.ID] - c.list[n.ID] = n - - if ok { - if gids, ok := c.idToGroupList[n.ID]; ok { - for gid := range gids { - c.groupToIDList[gid][n.ID] = n - } + _, ok := c.list[n.ID] + c.list[n.ID] = n + if !ok { + return } - } - c.listMu.Unlock() + gids := c.idToGroupList[n.ID] + for gid := range gids { + group, exists := c.groupToIDList[gid] + if !exists { + delete(gids, gid) + continue + } + group[n.ID] = n + } + if len(gids) == 0 { + delete(c.idToGroupList, n.ID) + } + }() c.sortList() } @@ -102,71 +111,69 @@ func (c *NotificationClass) UpdateGroup(ng *model.NotificationGroup, ngn []uint6 c.groupMu.Lock() defer c.groupMu.Unlock() - _, ok := c.groupList[ng.ID] c.groupList[ng.ID] = ng.Name c.listMu.Lock() defer c.listMu.Unlock() - if !ok { - c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification, len(ngn)) - for _, n := range ngn { - if c.idToGroupList[n] == nil { - c.idToGroupList[n] = make(map[uint64]struct{}) - } - c.idToGroupList[n][ng.ID] = struct{}{} - c.groupToIDList[ng.ID][n] = c.list[n] - } - } else { - oldList := make(map[uint64]struct{}) - for nid := range c.groupToIDList[ng.ID] { - oldList[nid] = struct{}{} - } - c.groupToIDList[ng.ID] = make(map[uint64]*model.Notification) - for _, nid := range ngn { - c.groupToIDList[ng.ID][nid] = c.list[nid] - if c.idToGroupList[nid] == nil { - c.idToGroupList[nid] = make(map[uint64]struct{}) - } - c.idToGroupList[nid][ng.ID] = struct{}{} + oldList := c.groupToIDList[ng.ID] + newList := make(map[uint64]*model.Notification, len(ngn)) + for _, nid := range ngn { + n, ok := c.list[nid] + if !ok { + continue } + newList[nid] = n + if c.idToGroupList[nid] == nil { + c.idToGroupList[nid] = make(map[uint64]struct{}) + } + c.idToGroupList[nid][ng.ID] = struct{}{} + } - for oldID := range oldList { - if _, ok := c.groupToIDList[ng.ID][oldID]; !ok { - delete(c.groupToIDList[oldID], ng.ID) - if len(c.idToGroupList[oldID]) == 0 { - delete(c.idToGroupList, oldID) - } - } + for oldID := range oldList { + if _, ok := newList[oldID]; ok { + continue + } + delete(c.idToGroupList[oldID], ng.ID) + if len(c.idToGroupList[oldID]) == 0 { + delete(c.idToGroupList, oldID) } } + c.groupToIDList[ng.ID] = newList } func (c *NotificationClass) Delete(idList []uint64) { - c.listMu.Lock() + func() { + c.listMu.Lock() + defer c.listMu.Unlock() - for _, id := range idList { - delete(c.list, id) - // 如果绑定了通知组才删除 - if gids, ok := c.idToGroupList[id]; ok { - for gid := range gids { - delete(c.groupToIDList[gid], id) + for _, id := range idList { + delete(c.list, id) + // 如果绑定了通知组才删除 + if gids, ok := c.idToGroupList[id]; ok { + for gid := range gids { + delete(c.groupToIDList[gid], id) + } delete(c.idToGroupList, id) } } - } - - c.listMu.Unlock() + }() c.sortList() } func (c *NotificationClass) DeleteGroup(gids []uint64) { - c.listMu.Lock() - defer c.listMu.Unlock() c.groupMu.Lock() defer c.groupMu.Unlock() + c.listMu.Lock() + defer c.listMu.Unlock() for _, gid := range gids { + for nid := range c.groupToIDList[gid] { + delete(c.idToGroupList[nid], gid) + if len(c.idToGroupList[nid]) == 0 { + delete(c.idToGroupList, nid) + } + } delete(c.groupList, gid) delete(c.groupToIDList, gid) } @@ -234,13 +241,19 @@ func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc st return } } - // 向该通知方式组的所有通知方式发出通知 + // Copy the group under the lock. Webhook delivery can take minutes and must + // not block notification or notification-group updates while it is in flight. c.listMu.RLock() - defer c.listMu.RUnlock() + notifications := make([]*model.Notification, 0, len(c.groupToIDList[notificationGroupID])) for _, n := range c.groupToIDList[notificationGroupID] { + notifications = append(notifications, n) + } + c.listMu.RUnlock() + + for _, n := range notifications { log.Printf("NEZHA>> Try to notify %s", n.Name) } - for _, n := range c.groupToIDList[notificationGroupID] { + for _, n := range notifications { ns := model.NotificationServerBundle{ Notification: n, Server: nil, diff --git a/service/singleton/notification_test.go b/service/singleton/notification_test.go new file mode 100644 index 00000000..5e08de31 --- /dev/null +++ b/service/singleton/notification_test.go @@ -0,0 +1,126 @@ +package singleton + +import ( + "testing" + "time" + + "github.com/nezhahq/nezha/model" +) + +func newNotificationClassWithItems(items ...*model.Notification) *NotificationClass { + nc := NewEmptyNotificationClassForTest() + for _, item := range items { + nc.InsertForTest(item) + } + return nc +} + +func TestNotificationClassDeleteGroupCleansReverseIndex(t *testing.T) { + n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"} + nc := newNotificationClassWithItems(n) + group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"} + nc.UpdateGroup(group, []uint64{n.ID}) + + nc.DeleteGroup([]uint64{group.ID}) + + if _, ok := nc.groupToIDList[group.ID]; ok { + t.Fatalf("forward index still contains deleted group %d", group.ID) + } + if groups, ok := nc.idToGroupList[n.ID]; ok { + if _, stale := groups[group.ID]; stale { + t.Fatalf("reverse index for notification %d still contains deleted group %d", n.ID, group.ID) + } + } +} + +func TestNotificationClassUpdateGroupCleansRemovedMembership(t *testing.T) { + first := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "first"} + second := &model.Notification{Common: model.Common{ID: 2, UserID: 2}, Name: "second"} + nc := newNotificationClassWithItems(first, second) + group := &model.NotificationGroup{Common: model.Common{ID: 10, UserID: 2}, Name: "group"} + nc.UpdateGroup(group, []uint64{first.ID, second.ID}) + + nc.UpdateGroup(group, []uint64{second.ID}) + + if groups, ok := nc.idToGroupList[first.ID]; ok { + if _, stale := groups[group.ID]; stale { + t.Fatalf("reverse index for notification %d still contains group %d", first.ID, group.ID) + } + } + if _, ok := nc.idToGroupList[second.ID][group.ID]; !ok { + t.Fatalf("reverse index lost retained membership of notification %d in group %d", second.ID, group.ID) + } +} + +func TestNotificationClassUpdateRepairsOrphanedReverseIndex(t *testing.T) { + n := &model.Notification{Common: model.Common{ID: 1, UserID: 2}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.idToGroupList[n.ID] = map[uint64]struct{}{99: {}} + + n.Name = "updated" + nc.Update(n) + + if groups, ok := nc.idToGroupList[n.ID]; ok && len(groups) != 0 { + t.Fatalf("orphaned reverse memberships were not removed: %v", groups) + } +} + +func TestNotificationClassGroupMutationsDoNotDeadlock(t *testing.T) { + nc := NewEmptyNotificationClassForTest() + nc.listMu.RLock() + readLockHeld := true + defer func() { + if readLockHeld { + nc.listMu.RUnlock() + } + }() + + deleteDone := make(chan struct{}) + go func() { + nc.DeleteGroup([]uint64{10}) + close(deleteDone) + }() + + deadline := time.Now().Add(2 * time.Second) + for nc.listMu.TryRLock() { + nc.listMu.RUnlock() + if time.Now().After(deadline) { + t.Fatal("DeleteGroup did not queue for listMu") + } + time.Sleep(time.Millisecond) + } + + updateDone := make(chan struct{}) + go func() { + nc.UpdateGroup(&model.NotificationGroup{ + Common: model.Common{ID: 10, UserID: 2}, + Name: "group", + }, nil) + close(updateDone) + }() + + deadline = time.Now().Add(2 * time.Second) + for nc.groupMu.TryLock() { + nc.groupMu.Unlock() + if time.Now().After(deadline) { + t.Fatal("neither group mutation acquired groupMu") + } + time.Sleep(time.Millisecond) + } + + nc.listMu.RUnlock() + readLockHeld = false + + timer := time.NewTimer(2 * time.Second) + defer timer.Stop() + for deleteDone != nil || updateDone != nil { + select { + case <-deleteDone: + deleteDone = nil + case <-updateDone: + updateDone = nil + case <-timer.C: + t.Fatal("concurrent UpdateGroup and DeleteGroup deadlocked") + } + } +} From c726930833937a0aebb2a924064f236faea889f6 Mon Sep 17 00:00:00 2001 From: naiba Date: Sat, 26 Sep 2026 10:26:05 +0800 Subject: [PATCH 33/48] chore: update frontend --- service/singleton/frontend-templates.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/service/singleton/frontend-templates.yaml b/service/singleton/frontend-templates.yaml index fecbb885..c9bbd8ec 100644 --- a/service/singleton/frontend-templates.yaml +++ b/service/singleton/frontend-templates.yaml @@ -2,24 +2,20 @@ name: "OfficialAdmin" repository: "https://github.com/nezhahq/admin-frontend" author: "nezhahq" - version: "v2.3.6" + version: "v2.3.7" is_admin: true is_official: true - path: "user-dist" name: "Official" repository: "https://github.com/hamster1963/nezha-dash-v2" author: "hamster1963" - version: "v2.4.2" + version: "v2.4.3" is_official: true - path: "nezha-pixel-dist" name: "Nezha-Pixel" repository: "https://github.com/karllao/nezha-pixel" author: "karllao" version: "v1.6.0" -# Third-party user themes consume the opaque Server.PublicNote field. Theme -# maintainers must validate URL schemes (including after decoding) before using -# values such as customData.orderLink in href or window.open; the Dashboard -# backend and admin frontend do not execute those fields. - path: "nazhua-dist" name: "Nazhua" repository: "https://github.com/hi2shark/nazhua" From 36b742a7e9a81ed782363dd19efd77b7030fbd0f Mon Sep 17 00:00:00 2001 From: naiba Date: Thu, 1 Oct 2026 04:28:34 +0000 Subject: [PATCH 34/48] fix: harden dashboard security boundaries --- .../controller/credential_redaction_test.go | 44 ++++++---- cmd/dashboard/controller/ddns.go | 10 ++- cmd/dashboard/controller/fm.go | 14 ++++ cmd/dashboard/controller/mcp_test.go | 7 +- cmd/dashboard/controller/mcp_transfer.go | 45 +++++++++-- .../controller/mcp_transfer_host_test.go | 80 +++++++++++++++++++ cmd/dashboard/controller/server.go | 25 +++--- .../controller/server_config_oracle_test.go | 54 +++++++++++++ .../controller/terminal_input_limit_test.go | 19 +++++ cmd/dashboard/controller/transfer.go | 9 ++- service/singleton/server_transfer.go | 37 ++++++++- service/singleton/server_transfer_test.go | 59 +++++++++++++- 12 files changed, 357 insertions(+), 46 deletions(-) create mode 100644 cmd/dashboard/controller/mcp_transfer_host_test.go create mode 100644 cmd/dashboard/controller/server_config_oracle_test.go diff --git a/cmd/dashboard/controller/credential_redaction_test.go b/cmd/dashboard/controller/credential_redaction_test.go index 14b289da..0daaa6fd 100644 --- a/cmd/dashboard/controller/credential_redaction_test.go +++ b/cmd/dashboard/controller/credential_redaction_test.go @@ -109,12 +109,14 @@ func TestListDDNS_RedactsCredentials(t *testing.T) { defer setupTenancyTest(t)() p := model.DDNSProfile{ - Common: model.Common{UserID: 10}, - Name: "cf", - Provider: "cloudflare", - AccessID: "id", - AccessSecret: "super-secret-token", - WebhookHeaders: `{"Authorization":"Bearer xxx"}`, + Common: model.Common{UserID: 10}, + Name: "cf", + Provider: "cloudflare", + AccessID: "id", + AccessSecret: "super-secret-token", + WebhookURL: "https://ddns.example/update?token=embedded-secret", + WebhookRequestBody: `{"token":"embedded-secret"}`, + WebhookHeaders: `{"Authorization":"Bearer xxx"}`, } require.NoError(t, singleton.DB.Create(&p).Error) singleton.DDNSShared.InsertForTest(&p) @@ -124,12 +126,18 @@ func TestListDDNS_RedactsCredentials(t *testing.T) { require.NoError(t, err) require.Len(t, out, 1) require.Empty(t, out[0].AccessSecret, "access_secret must be redacted in list response") + require.Empty(t, out[0].WebhookURL, "webhook_url must be redacted in list response") + require.Empty(t, out[0].WebhookRequestBody, "webhook_request_body must be redacted in list response") require.Empty(t, out[0].WebhookHeaders, "webhook_headers must be redacted in list response") require.Equal(t, "id", out[0].AccessID, "non-secret fields must be preserved") var stored model.DDNSProfile require.NoError(t, singleton.DB.First(&stored, p.ID).Error) require.Equal(t, "super-secret-token", stored.AccessSecret, "redaction must not mutate stored data") + require.Equal(t, "https://ddns.example/update?token=embedded-secret", stored.WebhookURL, + "redaction must not mutate stored webhook URL") + require.Equal(t, `{"token":"embedded-secret"}`, stored.WebhookRequestBody, + "redaction must not mutate stored webhook body") } func TestListNotification_RedactsCredentials(t *testing.T) { @@ -164,14 +172,15 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) { defer setupTenancyTest(t)() existing := model.DDNSProfile{ - Common: model.Common{UserID: 10}, - Name: "cf", - Provider: "webhook", - AccessID: "id", - AccessSecret: "keep-me", - WebhookURL: "http://127.0.0.1/", - WebhookMethod: 1, - WebhookHeaders: `{"X-Token":"keep-header"}`, + Common: model.Common{UserID: 10}, + Name: "cf", + Provider: "webhook", + AccessID: "id", + AccessSecret: "keep-me", + WebhookURL: "http://127.0.0.1/", + WebhookRequestBody: `{"X-Token":"keep-body"}`, + WebhookMethod: 1, + WebhookHeaders: `{"X-Token":"keep-header"}`, } require.NoError(t, singleton.DB.Create(&existing).Error) singleton.DDNSShared.InsertForTest(&existing) @@ -181,10 +190,11 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) { "provider": "webhook", "access_id": "id", "access_secret": "", - "webhook_url": "http://127.0.0.1/", + "webhook_url": "", "webhook_method": 1, "webhook_request_type": 1, "webhook_headers": "", + "webhook_request_body": "", "max_retries": 3, }) c.Params = gin.Params{{Key: "id", Value: itoa(existing.ID)}} @@ -197,6 +207,10 @@ func TestUpdateDDNS_EmptySecretPreservesStored(t *testing.T) { require.Equal(t, "keep-me", after.AccessSecret, "empty submitted secret must preserve stored value") require.Equal(t, `{"X-Token":"keep-header"}`, after.WebhookHeaders, "empty submitted headers must preserve stored value") + require.Equal(t, "http://127.0.0.1/", after.WebhookURL, + "empty submitted webhook URL must preserve stored value") + require.Equal(t, `{"X-Token":"keep-body"}`, after.WebhookRequestBody, + "empty submitted webhook body must preserve stored value") } func TestUpdateDDNS_NonEmptySecretOverwrites(t *testing.T) { diff --git a/cmd/dashboard/controller/ddns.go b/cmd/dashboard/controller/ddns.go index a2a2c20e..b03ba5ca 100644 --- a/cmd/dashboard/controller/ddns.go +++ b/cmd/dashboard/controller/ddns.go @@ -34,6 +34,8 @@ func listDDNS(c *gin.Context) ([]*model.DDNSProfile, error) { // 不影响 singleton 内原始数据。 for _, p := range ddnsProfiles { p.AccessSecret = "" + p.WebhookURL = "" + p.WebhookRequestBody = "" p.WebhookHeaders = "" } @@ -144,10 +146,8 @@ func updateDDNS(c *gin.Context) (any, error) { p.Provider = df.Provider p.Domains = df.Domains p.AccessID = df.AccessID - p.WebhookURL = df.WebhookURL p.WebhookMethod = df.WebhookMethod p.WebhookRequestType = df.WebhookRequestType - p.WebhookRequestBody = df.WebhookRequestBody // 凭据在列表接口已脱敏，前端无法回填；空值视为"不修改"，保留旧值避免误清空。 if df.AccessSecret != "" { @@ -156,6 +156,12 @@ func updateDDNS(c *gin.Context) (any, error) { if df.WebhookHeaders != "" { p.WebhookHeaders = df.WebhookHeaders } + if df.WebhookURL != "" { + p.WebhookURL = df.WebhookURL + } + if df.WebhookRequestBody != "" { + p.WebhookRequestBody = df.WebhookRequestBody + } for n, domain := range p.Domains { // IDN to ASCII diff --git a/cmd/dashboard/controller/fm.go b/cmd/dashboard/controller/fm.go index dfdb2728..51d5dd58 100644 --- a/cmd/dashboard/controller/fm.go +++ b/cmd/dashboard/controller/fm.go @@ -15,6 +15,19 @@ import ( "github.com/nezhahq/nezha/service/singleton" ) +// The official file-manager client uploads in 1 MiB WebSocket messages. Cap +// each complete message at that protocol boundary so gorilla/websocket cannot +// buffer an attacker-sized frame before the IO stream relay sees it. +const fileManagerWebSocketInputLimit int64 = 1024 * 1024 + +type websocketReadLimiter interface { + SetReadLimit(limit int64) +} + +func limitFileManagerWebSocketInput(conn websocketReadLimiter) { + conn.SetReadLimit(fileManagerWebSocketInputLimit) +} + // Create FM session // @Summary Create FM session // @Description Create an "attached" FM. It is advised to only call this within a terminal session. @@ -99,6 +112,7 @@ func fmStream(c *gin.Context) (any, error) { if err != nil { return nil, newWsError("%v", err) } + limitFileManagerWebSocketInput(wsConn) conn := websocketx.NewConn(wsConn) pingTransport := newWebsocketPingTransport(conn, wsConn.Close) stopPing := startWebsocketPingTicker(c.Request.Context(), time.Second*10, pingTransport) diff --git a/cmd/dashboard/controller/mcp_test.go b/cmd/dashboard/controller/mcp_test.go index 1b78f0d9..f90f4e23 100644 --- a/cmd/dashboard/controller/mcp_test.go +++ b/cmd/dashboard/controller/mcp_test.go @@ -39,7 +39,12 @@ func setupMCPTest(t *testing.T) (func(), uint64) { require.NoError(t, err) require.NoError(t, db.AutoMigrate(&model.User{}, &model.APIToken{}, &model.MCPAuditLog{}, &model.Server{}, &model.WAF{})) singleton.DB = db - singleton.Conf = &singleton.ConfigClass{Config: &model.Config{JWTTimeout: 1}} + singleton.Conf = &singleton.ConfigClass{Config: &model.Config{ + JWTTimeout: 1, + ConfigDashboard: model.ConfigDashboard{ + DashboardHost: "example.com", + }, + }} singleton.Conf.SetMCPEnabled(true) user := model.User{Common: model.Common{ID: 100}, Username: "alice", Role: model.RoleMember} diff --git a/cmd/dashboard/controller/mcp_transfer.go b/cmd/dashboard/controller/mcp_transfer.go index 6f204811..8aa09670 100644 --- a/cmd/dashboard/controller/mcp_transfer.go +++ b/cmd/dashboard/controller/mcp_transfer.go @@ -13,6 +13,7 @@ import ( "io" "math" "net/http" + "net/url" "strconv" "strings" "sync" @@ -91,6 +92,37 @@ var ( transferSecretVal string ) +var errMCPTransferHostNotConfigured = errors.New("MCP transfer URL host is not an operator-declared dashboard host") + +// mcpTransferURLBase never reflects an untrusted request Host into a bearer +// transfer URL. DashboardHost is the canonical public authority; deployments +// that intentionally serve multiple names must declare them as reserved hosts. +func mcpTransferURLBase(c *gin.Context) (string, error) { + if c == nil || c.Request == nil || singleton.Conf == nil { + return "", errMCPTransferHostNotConfigured + } + + requestHost := strings.TrimSpace(c.Request.Host) + host := strings.TrimSpace(singleton.Conf.DashboardHost) + if host == "" { + if !singleton.IsReservedDashboardHost(requestHost) { + return "", errMCPTransferHostNotConfigured + } + host = requestHost + } + + parsed, err := url.Parse("//" + host) + if err != nil || parsed.Host == "" || parsed.Host != host || parsed.User != nil || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" { + return "", errMCPTransferHostNotConfigured + } + + scheme := "https" + if c.Request.TLS == nil && c.Request.Header.Get("X-Forwarded-Proto") != "https" { + scheme = "http" + } + return scheme + "://" + host, nil +} + // transferHMACSecret 返回进程内随机生成的 HMAC key。 // 这是有意设计：transferEntries 本身也只活在内存 sync.Map 里，dashboard // 重启等价于全部 token 失效；让 secret 也随进程随机，可以避免“secret 来自 @@ -315,6 +347,10 @@ func mintTransferTool(c *gin.Context, serverID uint64, path string, ttlSeconds i if err := validateTransferPath(path); err != nil { return nil, err } + baseURL, err := mcpTransferURLBase(c) + if err != nil { + return nil, err + } ttl := time.Duration(ttlSeconds) * time.Second if ttl <= 0 { ttl = transferTokenTTLDefault @@ -348,14 +384,9 @@ func mintTransferTool(c *gin.Context, serverID uint64, path string, ttlSeconds i if err != nil { return nil, err } - scheme := "https" - if c.Request.TLS == nil && c.Request.Header.Get("X-Forwarded-Proto") != "https" { - scheme = "http" - } - host := c.Request.Host - url := fmt.Sprintf("%s://%s/mcp/%s/%s", scheme, host, dir, t) + transferURL := fmt.Sprintf("%s/mcp/%s/%s", baseURL, dir, t) return map[string]any{ - "url": url, + "url": transferURL, "method": map[transferDirection]string{transferDirDownload: "GET", transferDirUpload: "POST"}[dir], "expires_at": entry.ExpiresAt, }, nil diff --git a/cmd/dashboard/controller/mcp_transfer_host_test.go b/cmd/dashboard/controller/mcp_transfer_host_test.go new file mode 100644 index 00000000..58340fa0 --- /dev/null +++ b/cmd/dashboard/controller/mcp_transfer_host_test.go @@ -0,0 +1,80 @@ +package controller + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +func TestMCPTransferURLPinsUntrustedRequestHostToDashboardHost(t *testing.T) { + cleanup, uid := setupMCPTest(t) + defer cleanup() + PurgeTransferEntries() + t.Cleanup(func() { PurgeTransferEntries() }) + tok, _ := mkToken(t, uid, []string{model.ScopeServerRead}, nil) + + c, _ := mcpCallCtx(t, tok, uid, nil) + c.Request.Host = "attacker.example" + c.Request.Header.Set("X-Forwarded-Proto", "https") + + out, err := mintTransferTool(c, 7, "/srv/file", 60, transferDirDownload, transferEntry{}) + require.NoError(t, err) + result := out.(map[string]any) + require.Contains(t, result["url"], "https://example.com/mcp/download/") + require.NotContains(t, result["url"], "attacker.example") +} + +func TestMCPTransferURLRejectsUntrustedHostWithoutCanonicalHost(t *testing.T) { + cleanup, uid := setupMCPTest(t) + defer cleanup() + PurgeTransferEntries() + t.Cleanup(func() { PurgeTransferEntries() }) + tok, _ := mkToken(t, uid, []string{model.ScopeServerRead}, nil) + singleton.Conf.DashboardHost = "" + singleton.Conf.InstallHost = "" + singleton.Conf.ListenHost = "" + singleton.Conf.ReservedHosts = "" + + c, _ := mcpCallCtx(t, tok, uid, nil) + c.Request = c.Request.Clone(c.Request.Context()) + c.Request.Method = http.MethodPost + c.Request.Host = "attacker.example" + + _, err := mintTransferTool(c, 7, "/srv/file", 60, transferDirDownload, transferEntry{}) + require.ErrorIs(t, err, errMCPTransferHostNotConfigured) + entries := 0 + transferEntries.Range(func(_, _ any) bool { + entries++ + return true + }) + require.Zero(t, entries, "host validation must happen before a bearer transfer token is minted") +} + +func TestMCPTransferURLAllowsOperatorReservedHost(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + singleton.Conf.DashboardHost = "" + singleton.Conf.ReservedHosts = "mcp.example.com" + + c, _ := mcpCallCtx(t, nil, 0, nil) + c.Request.Host = "mcp.example.com" + c.Request.Header.Set("X-Forwarded-Proto", "https") + + base, err := mcpTransferURLBase(c) + require.NoError(t, err) + require.Equal(t, "https://mcp.example.com", base) +} + +func TestMCPTransferURLRejectsMalformedConfiguredHost(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + singleton.Conf.DashboardHost = "https://example.com/path" + + c, _ := mcpCallCtx(t, nil, 0, nil) + _, err := mcpTransferURLBase(c) + require.ErrorIs(t, err, errMCPTransferHostNotConfigured) +} diff --git a/cmd/dashboard/controller/server.go b/cmd/dashboard/controller/server.go index c5c6a4ac..ccbebf5b 100644 --- a/cmd/dashboard/controller/server.go +++ b/cmd/dashboard/controller/server.go @@ -236,10 +236,9 @@ func getServerConfig(c *gin.Context) (string, error) { } s, ok := singleton.ServerShared.Get(id) - if !ok { - return "", nil - } - if !s.HasPermission(c) { + // Foreign and unknown IDs deliberately share one response so sequential + // server IDs cannot be enumerated across tenants. + if !ok || !s.HasPermission(c) || !patAllowsServer(c, id) { return "", singleton.Localizer.ErrorT("permission denied") } if s.GetTaskStream() == nil { @@ -293,16 +292,16 @@ func setServerConfig(c *gin.Context) (*model.ServerTaskResponse, error) { slist := singleton.ServerShared.GetList() servers := make([]*model.Server, 0, len(configForm.Servers)) for _, sid := range configForm.Servers { - if s, ok := slist[sid]; ok { - if !s.HasPermission(c) { - return nil, singleton.Localizer.ErrorT("permission denied") - } - if s.GetTaskStream() == nil { - resp.Offline = append(resp.Offline, s.ID) - continue - } - servers = append(servers, s) + s, ok := slist[sid] + // Match getServerConfig: do not reveal whether a denied ID exists. + if !ok || !s.HasPermission(c) || !patAllowsServer(c, sid) { + return nil, singleton.Localizer.ErrorT("permission denied") } + if s.GetTaskStream() == nil { + resp.Offline = append(resp.Offline, s.ID) + continue + } + servers = append(servers, s) } var wg sync.WaitGroup diff --git a/cmd/dashboard/controller/server_config_oracle_test.go b/cmd/dashboard/controller/server_config_oracle_test.go new file mode 100644 index 00000000..6ffe0542 --- /dev/null +++ b/cmd/dashboard/controller/server_config_oracle_test.go @@ -0,0 +1,54 @@ +package controller + +import ( + "net/http" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +func TestGetServerConfigForeignAndUnknownIDsAreIndistinguishable(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + require.NoError(t, singleton.DB.Create(&model.User{Common: model.Common{ID: 200}, Username: "bob", Role: model.RoleMember}).Error) + tok, _ := mkToken(t, 200, []string{model.ScopeServerRead}, nil) + + probe := func(id string) error { + c, _ := patRequestCtx(t, tok, 200, http.MethodGet, "/api/v1/server/config/"+id, nil) + c.Params = gin.Params{{Key: "id", Value: id}} + _, err := getServerConfig(c) + return err + } + + foreignErr := probe("7") + unknownErr := probe("999999") + require.Error(t, foreignErr) + require.Error(t, unknownErr) + require.Equal(t, foreignErr.Error(), unknownErr.Error()) +} + +func TestSetServerConfigForeignAndUnknownIDsAreIndistinguishable(t *testing.T) { + cleanup, _ := setupMCPTest(t) + defer cleanup() + require.NoError(t, singleton.DB.Create(&model.User{Common: model.Common{ID: 200}, Username: "bob", Role: model.RoleMember}).Error) + tok, _ := mkToken(t, 200, []string{model.ScopeServerWrite}, nil) + + probe := func(id uint64) error { + c, _ := patRequestCtx(t, tok, 200, http.MethodPost, "/api/v1/server/config", model.ServerConfigForm{ + Servers: []uint64{id}, + Config: "{}", + }) + _, err := setServerConfig(c) + return err + } + + foreignErr := probe(7) + unknownErr := probe(999999) + require.Error(t, foreignErr) + require.Error(t, unknownErr) + require.Equal(t, foreignErr.Error(), unknownErr.Error()) +} diff --git a/cmd/dashboard/controller/terminal_input_limit_test.go b/cmd/dashboard/controller/terminal_input_limit_test.go index 7ed3dcc9..8fb4cbab 100644 --- a/cmd/dashboard/controller/terminal_input_limit_test.go +++ b/cmd/dashboard/controller/terminal_input_limit_test.go @@ -2,8 +2,27 @@ package controller import "testing" +type readLimitRecorder struct { + limit int64 +} + +func (r *readLimitRecorder) SetReadLimit(limit int64) { + r.limit = limit +} + func TestTerminalWebSocketInputLimitAllowsBoundedPaste(t *testing.T) { if terminalWebSocketInputLimit != 512*1024+64 { t.Fatalf("terminal WebSocket input limit = %d, want 512 KiB plus control-byte allowance", terminalWebSocketInputLimit) } } + +func TestFileManagerWebSocketInputLimitMatchesUploadChunk(t *testing.T) { + if fileManagerWebSocketInputLimit != 1024*1024 { + t.Fatalf("file-manager WebSocket input limit = %d, want official client's 1 MiB upload chunk", fileManagerWebSocketInputLimit) + } + recorder := new(readLimitRecorder) + limitFileManagerWebSocketInput(recorder) + if recorder.limit != fileManagerWebSocketInputLimit { + t.Fatalf("file-manager WebSocket configured limit = %d, want %d", recorder.limit, fileManagerWebSocketInputLimit) + } +} diff --git a/cmd/dashboard/controller/transfer.go b/cmd/dashboard/controller/transfer.go index 0e8f3ca7..f3c76b59 100644 --- a/cmd/dashboard/controller/transfer.go +++ b/cmd/dashboard/controller/transfer.go @@ -171,6 +171,12 @@ const transferStreamPingInterval = 30 * time.Second // @Success 200 {object} model.ServerTransfer // @Router /ws/transfer [get] func transferStream(c *gin.Context) (any, error) { + subID, ch, err := singleton.ServerTransferShared.Subscribe(getUid(c)) + if err != nil { + return nil, err + } + defer singleton.ServerTransferShared.Unsubscribe(subID) + conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { return nil, newWsError("%v", err) @@ -180,9 +186,6 @@ func transferStream(c *gin.Context) (any, error) { deregisterPAT := registerPATConnection(c, func() { _ = conn.Close() }) defer deregisterPAT() - subID, ch := singleton.ServerTransferShared.Subscribe() - defer singleton.ServerTransferShared.Unsubscribe(subID) - // Pings keep the socket warm even when the broker is quiet. Without this // a long idle period followed by a transfer event would race against // upstream proxy idle-timeouts that may have already closed the conn. diff --git a/service/singleton/server_transfer.go b/service/singleton/server_transfer.go index cef41476..ff3f93ef 100644 --- a/service/singleton/server_transfer.go +++ b/service/singleton/server_transfer.go @@ -152,6 +152,8 @@ type ServerTransferClass struct { subMu sync.Mutex subs map[uint64]chan *model.ServerTransfer + subOwners map[uint64]uint64 + subCounts map[uint64]int nextSubID uint64 timeout time.Duration @@ -174,6 +176,18 @@ var ErrServerAlreadyTransferring = errors.New("server already has an in-flight t // deferred to OnAgentReconnect / PushIfOnline. var ErrAgentTooOldForTransfer = fmt.Errorf("agent build older than %s does not support server transfer (TaskTypeServerTransferApply)", MinServerTransferAgentVersion) +const ( + // A normal dashboard needs one subscription per open transfer page. Keep a + // small per-user allowance for tabs/devices plus a process-wide hard stop. + maxServerTransferSubscribersPerUser = 8 + maxServerTransferSubscribersGlobal = 1024 +) + +var ( + errTooManyServerTransferSubscribersForUser = errors.New("too many concurrent server transfer subscribers for this user") + errTooManyServerTransferSubscribersGlobal = errors.New("too many concurrent server transfer subscribers") +) + // agentSupportsTransfer reports whether s has reported a build version >= // MinServerTransferAgentVersion. Returns true when version is unknown (agent // never reported) so callers can defer the decision; PushIfOnline re-checks @@ -210,6 +224,8 @@ func NewServerTransferClass() *ServerTransferClass { verifiedHandshakes: make(map[uint64]string), initiating: make(map[uint64]bool), subs: make(map[uint64]chan *model.ServerTransfer), + subOwners: make(map[uint64]uint64), + subCounts: make(map[uint64]int), timeout: defaultServerTransferTimeout, stopCh: make(chan struct{}), } @@ -1410,20 +1426,37 @@ func (c *ServerTransferClass) sweepTimeouts() { // Subscribe registers a channel that will receive every transfer transition // event from this point forward. The caller MUST Unsubscribe when done or // the broker will block forever if the channel is unbuffered or full. -func (c *ServerTransferClass) Subscribe() (uint64, <-chan *model.ServerTransfer) { +func (c *ServerTransferClass) Subscribe(userID uint64) (uint64, <-chan *model.ServerTransfer, error) { c.subMu.Lock() defer c.subMu.Unlock() + if len(c.subs) >= maxServerTransferSubscribersGlobal { + return 0, nil, errTooManyServerTransferSubscribersGlobal + } + if c.subCounts[userID] >= maxServerTransferSubscribersPerUser { + return 0, nil, errTooManyServerTransferSubscribersForUser + } id := atomic.AddUint64(&c.nextSubID, 1) ch := make(chan *model.ServerTransfer, 16) c.subs[id] = ch - return id, ch + c.subOwners[id] = userID + c.subCounts[userID]++ + return id, ch, nil } func (c *ServerTransferClass) Unsubscribe(id uint64) { c.subMu.Lock() ch, ok := c.subs[id] + ownerID, ownerOK := c.subOwners[id] delete(c.subs, id) + delete(c.subOwners, id) + if ownerOK { + if c.subCounts[ownerID] <= 1 { + delete(c.subCounts, ownerID) + } else { + c.subCounts[ownerID]-- + } + } c.subMu.Unlock() if ok { close(ch) diff --git a/service/singleton/server_transfer_test.go b/service/singleton/server_transfer_test.go index 2be8eb05..fa6fecf2 100644 --- a/service/singleton/server_transfer_test.go +++ b/service/singleton/server_transfer_test.go @@ -455,7 +455,8 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) { defer cleanup() seedServerForTransfer(t, 1, 100) - id, ch := c.Subscribe() + id, ch, err := c.Subscribe(100) + require.NoError(t, err) defer c.Unsubscribe(id) tr := initiateAndRegister(t, c, 1, 100, 200, 1) @@ -469,7 +470,7 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) { t.Fatal("expected Pending broadcast within 1s") } - _, _, err := markPendingVerified(t, c, 1) + _, _, err = markPendingVerified(t, c, 1) require.NoError(t, err) select { case ev := <-ch: @@ -479,6 +480,57 @@ func TestServerTransferBroadcastReachesSubscribers(t *testing.T) { } } +func TestServerTransferSubscribeEnforcesPerUserLimitAndReleasesQuota(t *testing.T) { + c, cleanup := setupTransferFixture(t) + defer cleanup() + + ids := make([]uint64, 0, maxServerTransferSubscribersPerUser) + for range maxServerTransferSubscribersPerUser { + id, ch, err := c.Subscribe(100) + require.NoError(t, err) + require.NotNil(t, ch) + ids = append(ids, id) + } + + _, _, err := c.Subscribe(100) + require.ErrorIs(t, err, errTooManyServerTransferSubscribersForUser) + + otherID, _, err := c.Subscribe(200) + require.NoError(t, err, "one user's quota must not block another user") + c.Unsubscribe(otherID) + + c.Unsubscribe(ids[0]) + replacementID, _, err := c.Subscribe(100) + require.NoError(t, err, "disconnecting must release the user's quota") + c.Unsubscribe(replacementID) + for _, id := range ids[1:] { + c.Unsubscribe(id) + } +} + +func TestServerTransferSubscribeEnforcesGlobalLimit(t *testing.T) { + c, cleanup := setupTransferFixture(t) + defer cleanup() + + ids := make([]uint64, 0, maxServerTransferSubscribersGlobal) + for userID := uint64(1); len(ids) < maxServerTransferSubscribersGlobal; userID++ { + for range maxServerTransferSubscribersPerUser { + if len(ids) == maxServerTransferSubscribersGlobal { + break + } + id, _, err := c.Subscribe(userID) + require.NoError(t, err) + ids = append(ids, id) + } + } + + _, _, err := c.Subscribe(999999) + require.ErrorIs(t, err, errTooManyServerTransferSubscribersGlobal) + for _, id := range ids { + c.Unsubscribe(id) + } +} + // The "one active transfer per server" invariant must hold under concurrent // callers (two operators batch-moving the same server, or batch-move racing // retry). The old flow had a TOCTOU between HasPending() and Initiate() that @@ -1626,7 +1678,8 @@ func TestOnServersDeletedTerminatesPendingTransfersAndClearsIndexes(t *testing.T tr := initiateAndRegister(t, c, 1, 100, 200, 1) require.True(t, c.HasPending(1)) - subID, ch := c.Subscribe() + subID, ch, err := c.Subscribe(100) + require.NoError(t, err) defer c.Unsubscribe(subID) require.NoError(t, DB.Unscoped().Delete(&model.Server{}, tr.ServerID).Error) From 40240ed02bdfe4ad3914292c32cac0a52dfc719f Mon Sep 17 00:00:00 2001 From: xvkong233 <88175491+xvkong233@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:56:05 +0800 Subject: [PATCH 35/48] fix(nat): strip only dashboard-issued Authorization credentials at NAT ingress Since 2640b86 (v2.2.11, 2026-07-20) the NAT ingress deleted the entire Authorization header, breaking self-hosted apps behind the tunnel that authenticate with their own Bearer tokens (e.g. the Qinglong panel login loop). Strip only credentials issued by this dashboard and forward foreign ones: - controller.IsDashboardCredential classifies an Authorization value as dashboard-issued via a signature-only JWT parser derived from the auth middleware (same key, HS256 pinned, claims validation deliberately skipped so an expired panel token is still recognized and kept out of the tunnel) or an api_tokens lookup for PAT-shaped values. Unknown PATs, foreign signatures, other schemes and missing values pass through. When classification cannot complete (parser not ready, DB nil or error) the verdict fails closed (strip) and the log carries a fixed reason only, never credential content. - Both NAT capability variants (default, and agent-compat legacy / invalid-capability / post-consume paths) replace the unconditional Del("Authorization") with stripDashboardCredential. The rpc package receives the classifier through SetNATDashboardCredentialGate, the same injection pattern as SetMCPKillSwitchObserver; an unwired gate keeps the historical strip-everything behavior. Tests: controller classification semantics (panel JWT, expired panel JWT and panel PAT stripped; foreign-signed JWT, unknown nzp_ token, random strings, non-Bearer schemes and missing header pass through; parser unavailable, DB nil and DB error fail closed with credential-free logs) plus rpc unit and flow coverage for both variants (dashboard credentials never reach the forwarded bytes; foreign and missing Authorization are forwarded untouched). Compatibility: no API, database or frontend changes; NAT backends that do not use Authorization behave exactly as before. --- cmd/dashboard/controller/controller.go | 3 + .../controller/dashboard_credential.go | 104 ++++++++++++ .../controller/dashboard_credential_test.go | 150 ++++++++++++++++++ cmd/dashboard/main.go | 5 + cmd/dashboard/rpc/nat.go | 9 +- .../rpc/nat_capability_agentcompat.go | 14 +- .../rpc/nat_capability_agentcompat_test.go | 122 ++++++++++++++ cmd/dashboard/rpc/nat_capability_default.go | 6 +- .../rpc/nat_capability_default_flow_test.go | 50 +++++- .../rpc/nat_capability_default_test.go | 74 ++++++++- .../nat_capability_flow_agentcompat_test.go | 61 ++++++- cmd/dashboard/rpc/nat_credential.go | 41 +++++ cmd/dashboard/rpc/nat_credential_test.go | 37 +++++ 13 files changed, 662 insertions(+), 14 deletions(-) create mode 100644 cmd/dashboard/controller/dashboard_credential.go create mode 100644 cmd/dashboard/controller/dashboard_credential_test.go create mode 100644 cmd/dashboard/rpc/nat_credential.go create mode 100644 cmd/dashboard/rpc/nat_credential_test.go diff --git a/cmd/dashboard/controller/controller.go b/cmd/dashboard/controller/controller.go index 1823d2c4..a3dd50b0 100644 --- a/cmd/dashboard/controller/controller.go +++ b/cmd/dashboard/controller/controller.go @@ -57,6 +57,9 @@ func routers(r *gin.Engine, frontendDist fs.FS) { if err := authMiddleware.MiddlewareInit(); err != nil { log.Fatal("authMiddleware.MiddlewareInit Error:" + err.Error()) } + // Signature-only view of the auth middleware for IsDashboardCredential: + // same key and pinned algorithm, claims validation deliberately skipped. + dashboardCredentialJWTParser = newDashboardCredentialJWTParser(authMiddleware) // /mcp — Model Context Protocol endpoint, authenticated by PAT only (闸 1 + 闸 2)。 // 不放在 /api/v1 下：MCP client 配置 URL 更短，且 MCP transport 协议演进与 REST API // 解耦。鉴权一律走 apiTokenAuthMiddleware；不接受 JWT 以避免浏览器误触。 diff --git a/cmd/dashboard/controller/dashboard_credential.go b/cmd/dashboard/controller/dashboard_credential.go new file mode 100644 index 00000000..e24ecdd7 --- /dev/null +++ b/cmd/dashboard/controller/dashboard_credential.go @@ -0,0 +1,104 @@ +package controller + +import ( + "errors" + "log" + "strings" + + jwt "github.com/golang-jwt/jwt/v4" + "gorm.io/gorm" + + ginjwt "github.com/appleboy/gin-jwt/v2" + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +// dashboardCredentialJWTParser verifies that a JWT was signed by this +// dashboard. It is derived from the auth middleware created in routers() and +// exists only for IsDashboardCredential; it must never authenticate dashboard +// requests on its own. +var dashboardCredentialJWTParser *ginjwt.GinJWTMiddleware + +// newDashboardCredentialJWTParser derives a signature-only parser from the +// dashboard auth middleware. It keeps the exact key value and the pinned HS256 +// signing algorithm of initParams() — there is no second read of +// singleton.Conf.JWTSecretKey — and disables claims validation so expiry does +// not change the verdict: an expired token is still proof the dashboard issued +// it, and keeping expired dashboard tokens out of the NAT tunnel prevents a +// careless backend from treating "any JWT" as its own credential. The live +// auth middleware keeps full claims validation; dashboard authentication is +// not relaxed by this parser in any way. +func newDashboardCredentialJWTParser(authMiddleware *ginjwt.GinJWTMiddleware) *ginjwt.GinJWTMiddleware { + parser := *authMiddleware + parser.ParseOptions = []jwt.ParserOption{jwt.WithoutClaimsValidation()} + return &parser +} + +// IsDashboardCredential reports whether an Authorization header value was +// issued by this dashboard: a panel-signed JWT or a panel API token (PAT). +// +// NAT ingress uses this to enforce the tunnel credential invariant: values +// classified as dashboard-issued are stripped before the request is tunneled +// to the agent, while foreign credentials are forwarded untouched because the +// NAT backend may authenticate with credentials of its own. +// +// Decision policy mirrors the dashboard's own auth middlewares: only the +// case-sensitive "Bearer " scheme is considered; PAT-shaped values are +// resolved against the api_tokens table; everything else is treated as a JWT. +// Whenever the classification cannot be completed (parser or config not ready, +// database failure) the value is conservatively reported as a dashboard +// credential so the caller strips it; only a deterministic proof that the +// value was not issued by this dashboard (scheme mismatch, unknown PAT hash, +// failed signature verification) returns false. +func IsDashboardCredential(authz string) bool { + raw := strings.TrimSpace(authz) + if !strings.HasPrefix(raw, "Bearer ") { + return false + } + plaintext := strings.TrimSpace(strings.TrimPrefix(raw, "Bearer ")) + if strings.HasPrefix(plaintext, model.APITokenPrefix) { + return isDashboardAPIToken(plaintext) + } + return isDashboardSignedJWT(plaintext) +} + +// isDashboardAPIToken resolves a PAT-shaped value against the api_tokens +// table. The plaintext token is only ever hashed here; it is never logged. +func isDashboardAPIToken(plaintext string) bool { + if singleton.DB == nil { + logDashboardCredentialUnclassifiable("api token lookup unavailable") + return true + } + var tok model.APIToken + err := singleton.DB.Where("token_hash = ?", model.HashAPIToken(plaintext)).First(&tok).Error + if err == nil { + return true + } + if errors.Is(err, gorm.ErrRecordNotFound) { + // Deterministic proof the token was not issued by this dashboard: + // forwarding it leaks nothing the dashboard ever validated. + return false + } + logDashboardCredentialUnclassifiable("api token lookup failed") + return true +} + +// isDashboardSignedJWT verifies the token signature with the dashboard's +// pinned algorithm and key. Claims validation is disabled on purpose (see +// newDashboardCredentialJWTParser): expiry does not change who issued the +// token. +func isDashboardSignedJWT(token string) bool { + if dashboardCredentialJWTParser == nil { + // routers() has not run yet (startup incomplete): fail closed. + logDashboardCredentialUnclassifiable("dashboard credential parser unavailable") + return true + } + _, err := dashboardCredentialJWTParser.ParseTokenString(token) + return err == nil +} + +// logDashboardCredentialUnclassifiable records a classification failure. The +// Authorization value itself is never included in the message. +func logDashboardCredentialUnclassifiable(reason string) { + log.Printf("NEZHA>> NAT ingress: Authorization value could not be classified (%s); stripping it as a precaution", reason) +} diff --git a/cmd/dashboard/controller/dashboard_credential_test.go b/cmd/dashboard/controller/dashboard_credential_test.go new file mode 100644 index 00000000..9ad1070e --- /dev/null +++ b/cmd/dashboard/controller/dashboard_credential_test.go @@ -0,0 +1,150 @@ +package controller + +import ( + "bytes" + "log" + "testing" + "time" + + jwt "github.com/appleboy/gin-jwt/v2" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +func setupDashboardCredentialTest(t *testing.T) { + t.Helper() + originalConf, originalDB, originalParser := singleton.Conf, singleton.DB, dashboardCredentialJWTParser + t.Cleanup(func() { + singleton.Conf, singleton.DB, dashboardCredentialJWTParser = originalConf, originalDB, originalParser + }) + + singleton.Conf = &singleton.ConfigClass{Config: &model.Config{}} + singleton.Conf.JWTSecretKey = "dashboard-credential-classification-test-key" + singleton.Conf.JWTTimeout = 1 + + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + t.Cleanup(func() { _ = sqlDB.Close() }) + require.NoError(t, db.AutoMigrate(&model.APIToken{})) + singleton.DB = db + + // Same construction path as routers(): jwt.New(initParams()). + authMiddleware, err := jwt.New(initParams()) + require.NoError(t, err) + dashboardCredentialJWTParser = newDashboardCredentialJWTParser(authMiddleware) +} + +// mintDashboardCredentialJWT signs a JWT with the dashboard key through +// initParams(), the same key source the production verifier uses. A negative +// timeout mints an already-expired token. +func mintDashboardCredentialJWT(t *testing.T, timeout time.Duration) string { + t.Helper() + params := initParams() + params.Timeout = timeout + mw, err := jwt.New(params) + require.NoError(t, err) + token, _, err := mw.TokenGenerator(nil) + require.NoError(t, err) + return token +} + +func mintForeignJWT(t *testing.T) string { + t.Helper() + params := initParams() + params.Key = []byte("foreign-signing-key-not-held-by-the-dashboard") + mw, err := jwt.New(params) + require.NoError(t, err) + token, _, err := mw.TokenGenerator(nil) + require.NoError(t, err) + return token +} + +func TestIsDashboardCredentialClassification(t *testing.T) { + setupDashboardCredentialTest(t) + + dashboardJWT := mintDashboardCredentialJWT(t, time.Hour) + expiredDashboardJWT := mintDashboardCredentialJWT(t, -time.Hour) + foreignJWT := mintForeignJWT(t) + + dashboardPAT := model.APITokenPrefix + "dashboard-credential-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-classification", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + t.Run("dashboard signed jwt is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredential("Bearer "+dashboardJWT)) + }) + + t.Run("expired dashboard signed jwt is still a dashboard credential", func(t *testing.T) { + // Signature verification ignores exp on purpose: an expired token is + // still proof the dashboard issued it. + require.True(t, IsDashboardCredential("Bearer "+expiredDashboardJWT)) + }) + + t.Run("dashboard api token is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredential("Bearer "+dashboardPAT)) + }) + + t.Run("foreign signed jwt is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Bearer "+foreignJWT)) + }) + + t.Run("unknown api token is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Bearer "+model.APITokenPrefix+"unknown-to-the-dashboard")) + }) + + t.Run("random string is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Bearer random-not-a-token")) + }) + + t.Run("non bearer scheme is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("Basic "+dashboardJWT)) + // Scheme matching is case-sensitive, as in both dashboard auth middlewares. + require.False(t, IsDashboardCredential("bearer "+dashboardJWT)) + }) + + t.Run("empty or missing header is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredential("")) + require.False(t, IsDashboardCredential("Bearer ")) + }) +} + +func TestIsDashboardCredentialFailsClosedWhenParserUnavailable(t *testing.T) { + setupDashboardCredentialTest(t) + dashboardCredentialJWTParser = nil + + token := mintDashboardCredentialJWT(t, time.Hour) + require.True(t, IsDashboardCredential("Bearer "+token)) +} + +func TestIsDashboardCredentialFailsClosedWhenAPITokenLookupUnavailable(t *testing.T) { + setupDashboardCredentialTest(t) + singleton.DB = nil + + require.True(t, IsDashboardCredential("Bearer "+model.APITokenPrefix+"unresolvable")) +} + +func TestIsDashboardCredentialFailsClosedOnDatabaseError(t *testing.T) { + setupDashboardCredentialTest(t) + sqlDB, err := singleton.DB.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + + var logs bytes.Buffer + originalOutput := log.Writer() + log.SetOutput(&logs) + t.Cleanup(func() { log.SetOutput(originalOutput) }) + + plaintext := model.APITokenPrefix + "unresolvable-after-db-failure" + require.True(t, IsDashboardCredential("Bearer "+plaintext)) + // The credential content itself must never reach the log. + require.NotContains(t, logs.String(), plaintext) +} diff --git a/cmd/dashboard/main.go b/cmd/dashboard/main.go index 69b905db..50d02590 100644 --- a/cmd/dashboard/main.go +++ b/cmd/dashboard/main.go @@ -169,6 +169,11 @@ func main() { grpcHandler := rpc.ServeRPC() httpHandler := controller.ServeWeb(frontendDist) controller.InitUpgrader() + // The NAT ingress strips dashboard-issued Authorization values (panel JWT / + // panel PAT) before tunneling a request to the agent and forwards foreign + // credentials untouched. ServeWeb built the JWT parser the classifier + // reuses, so the gate is wired right after it. + rpc.SetNATDashboardCredentialGate(controller.IsDashboardCredential) muxHandler := newHTTPandGRPCMux(httpHandler, grpcHandler) muxServerHTTP := &http.Server{ diff --git a/cmd/dashboard/rpc/nat.go b/cmd/dashboard/rpc/nat.go index d5b3acb6..5c519493 100644 --- a/cmd/dashboard/rpc/nat.go +++ b/cmd/dashboard/rpc/nat.go @@ -80,7 +80,14 @@ func ServeNAT(w http.ResponseWriter, r *http.Request, natConfig *model.NAT) { return } - // Authorization authenticates Dashboard access before NAT ingress; it must not become an origin credential for the configured NAT backend. + // Credential invariant at NAT ingress: credentials issued by this + // dashboard — the panel-signed JWT (verified by signature only, expiry + // deliberately ignored) and the panel API token — must never become origin + // credentials for the configured NAT backend. prepareNATCapability has + // already stripped every Authorization value IsDashboardCredential + // classified as dashboard-issued; whatever remains is either absent or a + // foreign credential, which is forwarded untouched because the NAT backend + // may authenticate with credentials of its own. wWrapped, err := utils.NewRequestWrapper(r, w) if err != nil { return diff --git a/cmd/dashboard/rpc/nat_capability_agentcompat.go b/cmd/dashboard/rpc/nat_capability_agentcompat.go index d346ae77..e2c3bc67 100644 --- a/cmd/dashboard/rpc/nat_capability_agentcompat.go +++ b/cmd/dashboard/rpc/nat_capability_agentcompat.go @@ -22,16 +22,24 @@ type natCapabilityLease struct { func prepareNATCapability(request *http.Request, natConfig *model.NAT) (natCapabilityLease, error) { values := request.Header.Values(agentcompatcontract.IOStreamCapabilityHeader) if len(values) == 0 { - request.Header.Del("Authorization") + // No capability header: legacy NAT path. Strip dashboard-issued + // Authorization values (panel JWT or panel PAT) so they never become + // origin credentials; foreign ones are ordinary request data. + stripDashboardCredential(request) return natCapabilityLease{}, nil } request.Header.Del(agentcompatcontract.IOStreamCapabilityHeader) if len(values) != 1 || values[0] == "" { - request.Header.Del("Authorization") + // Invalid capability: nothing is forwarded, but dashboard-issued + // credentials still must not survive into errors or task data. + stripDashboardCredential(request) return natCapabilityLease{}, errors.New("invalid NAT capability") } access, handle, err := serviceRPC.NezhaHandlerSingleton.ConsumeAgentCompatNATCapabilityForProfile(values[0], natConfig.ServerID, natConfig.ID) - request.Header.Del("Authorization") + // From here the request bytes may reach the agent (active lease) or the + // request ends in a 503; either way dashboard-issued Authorization values + // are stripped, foreign ones are forwarded untouched. + stripDashboardCredential(request) if err != nil { return natCapabilityLease{}, errors.New("invalid NAT capability") } diff --git a/cmd/dashboard/rpc/nat_capability_agentcompat_test.go b/cmd/dashboard/rpc/nat_capability_agentcompat_test.go index 94910c40..e1e369b4 100644 --- a/cmd/dashboard/rpc/nat_capability_agentcompat_test.go +++ b/cmd/dashboard/rpc/nat_capability_agentcompat_test.go @@ -4,6 +4,7 @@ package rpc import ( "bytes" + "context" "log" "net/http" "strings" @@ -59,7 +60,128 @@ func TestPrepareNATCapabilityRejectsDuplicateHeaderAfterRemovingAllValues(t *tes } } +func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentials(t *testing.T) { + // Given — legacy path without a capability header, then the invalid + // capability path: both must drop dashboard-issued Authorization values. + installNATCredentialTestGate(t) + for name, credential := range map[string]string{ + "dashboard jwt": natTestDashboardJWT, + "expired dashboard jwt": natTestDashboardExpiredJWT, + "dashboard api token": natTestDashboardAPIToken, + } { + t.Run(name+"/legacy path", func(t *testing.T) { + request := &http.Request{Header: make(http.Header)} + request.Header.Set("Authorization", credential) + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err != nil { + t.Fatalf("legacy NAT path returned error: %v", err) + } + if lease.active { + t.Fatal("legacy NAT path unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("legacy NAT path retained dashboard credential %q", got) + } + }) + t.Run(name+"/invalid capability path", func(t *testing.T) { + request := &http.Request{Header: make(http.Header)} + request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "malformed") + request.Header.Set("Authorization", credential) + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err == nil { + t.Fatal("malformed capability unexpectedly accepted") + } + if lease.active { + t.Fatal("malformed capability unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("invalid capability path retained dashboard credential %q", got) + } + }) + } +} + +func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentialAfterConsumingCapability(t *testing.T) { + // Given — a well-formed capability that consumes successfully: the request + // is about to be tunneled, so the dashboard credential must be gone. + installNATCredentialTestGate(t) + handler := serviceRPC.NewNezhaHandler() + original := serviceRPC.NezhaHandlerSingleton + serviceRPC.NezhaHandlerSingleton = handler + t.Cleanup(func() { serviceRPC.NezhaHandlerSingleton = original }) + + capability, err := handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{ + Owner: serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2}, + Purpose: serviceRPC.AgentCompatCapabilityNAT, + TargetServerID: 81, + ResourceID: 91, + ServerAccessAllowed: true, + }) + if err != nil { + t.Fatalf("register capability: %v", err) + } + request := &http.Request{Header: make(http.Header)} + request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String()) + request.Header.Set("Authorization", natTestDashboardJWT) + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("valid NAT capability rejected: %v", err) + } + if !lease.active { + t.Fatal("valid NAT capability did not activate") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("dashboard credential retained after capability consumption: %q", got) + } +} + +func TestPrepareNATCapabilityAgentCompatKeepsForeignAndMissingAuthorization(t *testing.T) { + // Given — legacy path: foreign credentials are ordinary request data and + // an absent header must stay untouched. + installNATCredentialTestGate(t) + for name, values := range map[string][]string{ + "foreign jwt": {natTestForeignJWT}, + "foreign random": {natTestForeignRandom}, + "empty value": {""}, + "missing header": nil, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{Header: make(http.Header)} + for _, value := range values { + request.Header.Set("Authorization", value) + } + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err != nil { + t.Fatalf("legacy NAT path returned error: %v", err) + } + if lease.active { + t.Fatal("legacy NAT path unexpectedly activated") + } + got := request.Header.Values("Authorization") + if len(got) != len(values) { + t.Fatalf("legacy NAT path changed Authorization to %q, want %q", got, values) + } + for i := range values { + if got[i] != values[i] { + t.Fatalf("legacy NAT path changed Authorization to %q, want %q", got, values) + } + } + }) + } +} + func TestServeNATAgentCompatSensitiveHeadersStayOutOfErrorsAndLogs(t *testing.T) { + installNATCredentialTestGate(t) request := &http.Request{Header: make(http.Header)} request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "capability-secret") request.Header.Set("Authorization", "Bearer deterministic-secret") diff --git a/cmd/dashboard/rpc/nat_capability_default.go b/cmd/dashboard/rpc/nat_capability_default.go index 8773477b..865c8f08 100644 --- a/cmd/dashboard/rpc/nat_capability_default.go +++ b/cmd/dashboard/rpc/nat_capability_default.go @@ -18,7 +18,11 @@ type natCapabilityLease struct { } func prepareNATCapability(request *http.Request, _ *model.NAT) (natCapabilityLease, error) { - request.Header.Del("Authorization") + // The default build has no capability protocol: every request takes the + // legacy NAT path, so any dashboard-issued Authorization value (panel JWT + // or panel PAT) is stripped before the request is tunneled to the agent. + // Foreign Authorization values are ordinary request data and stay put. + stripDashboardCredential(request) return natCapabilityLease{}, nil } diff --git a/cmd/dashboard/rpc/nat_capability_default_flow_test.go b/cmd/dashboard/rpc/nat_capability_default_flow_test.go index c3698afa..30c7d71b 100644 --- a/cmd/dashboard/rpc/nat_capability_default_flow_test.go +++ b/cmd/dashboard/rpc/nat_capability_default_flow_test.go @@ -19,6 +19,7 @@ import ( func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) { fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) connection := newServeNATConn() agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})} request := &http.Request{ @@ -28,7 +29,7 @@ func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) { Body: &serveNATBody{}, } request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "ordinary-data") - request.Header.Set("Authorization", "Bearer deterministic-secret") + request.Header.Set("Authorization", natTestForeignRandom) fixture.taskStream.onSend = func(task *proto.Task) error { var nat model.TaskNAT if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { @@ -58,7 +59,50 @@ func TestServeNATDefaultForwardsCapabilityHeaderAsOrdinaryData(t *testing.T) { if !strings.Contains(forwarded, strings.ToLower(agentcompatcontract.IOStreamCapabilityHeader)+": ordinary-data") { t.Fatal("default NAT flow did not forward capability header as ordinary request data") } - if strings.Contains(forwarded, "Authorization:") || strings.Contains(forwarded, "deterministic-secret") { - t.Fatal("default NAT flow forwarded Authorization credentials") + if !strings.Contains(forwarded, strings.ToLower("Authorization: "+natTestForeignRandom)) { + t.Fatal("default NAT flow stripped a foreign credential that does not belong to the dashboard") + } +} + +func TestServeNATDefaultDoesNotForwardDashboardCredentials(t *testing.T) { + fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) + connection := newServeNATConn() + agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Scheme: "http", Host: "example.test", Path: "/nat"}, + Header: make(http.Header), + Body: &serveNATBody{}, + } + request.Header.Set("Authorization", natTestDashboardJWT) + fixture.taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return fixture.handler.AgentConnected(nat.StreamID, agent) + } + done := make(chan struct{}) + deadline, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go func() { + ServeNAT(&serveNATResponseWriter{conn: connection}, request, &model.NAT{ServerID: fixture.server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-deadline.Done(): + t.Fatal("agent did not receive ordinary NAT request") + } + _ = connection.Close() + select { + case <-done: + case <-deadline.Done(): + t.Fatal("ServeNAT did not finish") + } + forwarded := strings.ToLower(string(agent.writtenBytes())) + if strings.Contains(forwarded, "authorization:") || strings.Contains(forwarded, "dashboard-signed-jwt") { + t.Fatal("default NAT flow forwarded dashboard credentials") } } diff --git a/cmd/dashboard/rpc/nat_capability_default_test.go b/cmd/dashboard/rpc/nat_capability_default_test.go index 7ff5e686..71154084 100644 --- a/cmd/dashboard/rpc/nat_capability_default_test.go +++ b/cmd/dashboard/rpc/nat_capability_default_test.go @@ -12,9 +12,10 @@ import ( func TestPrepareNATCapabilityDefaultPreservesHeaderAsOrdinaryRequestData(t *testing.T) { // Given + installNATCredentialTestGate(t) request := &http.Request{Header: make(http.Header)} request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "ordinary-data") - request.Header.Set("Authorization", "Bearer deterministic-secret") + request.Header.Set("Authorization", natTestForeignRandom) // When lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) @@ -29,7 +30,74 @@ func TestPrepareNATCapabilityDefaultPreservesHeaderAsOrdinaryRequestData(t *test if got := request.Header.Get(agentcompatcontract.IOStreamCapabilityHeader); got != "ordinary-data" { t.Fatalf("default NAT capability hook changed header to %q", got) } - if got := request.Header.Get("Authorization"); got != "" { - t.Fatalf("default NAT capability hook retained Authorization %q", got) + if got := request.Header.Get("Authorization"); got != natTestForeignRandom { + t.Fatalf("default NAT capability hook changed foreign Authorization to %q", got) + } +} + +func TestPrepareNATCapabilityDefaultStripsDashboardCredentials(t *testing.T) { + // Given + installNATCredentialTestGate(t) + for name, credential := range map[string]string{ + "dashboard jwt": natTestDashboardJWT, + "expired dashboard jwt": natTestDashboardExpiredJWT, + "dashboard api token": natTestDashboardAPIToken, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{Header: make(http.Header)} + request.Header.Set("Authorization", credential) + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("default NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("default NAT capability hook unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("default NAT capability hook retained dashboard credential %q", got) + } + }) + } +} + +func TestPrepareNATCapabilityDefaultKeepsForeignAndMissingAuthorization(t *testing.T) { + // Given + installNATCredentialTestGate(t) + for name, values := range map[string][]string{ + "foreign jwt": {natTestForeignJWT}, + "foreign random": {natTestForeignRandom}, + "empty value": {""}, + "missing header": nil, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{Header: make(http.Header)} + for _, value := range values { + request.Header.Set("Authorization", value) + } + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("default NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("default NAT capability hook unexpectedly activated") + } + got := request.Header.Values("Authorization") + if len(got) != len(values) { + t.Fatalf("default NAT capability hook changed Authorization to %q, want %q", got, values) + } + for i := range values { + if got[i] != values[i] { + t.Fatalf("default NAT capability hook changed Authorization to %q, want %q", got, values) + } + } + }) } } diff --git a/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go b/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go index 2ed5a003..0bfe381a 100644 --- a/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go +++ b/cmd/dashboard/rpc/nat_capability_flow_agentcompat_test.go @@ -23,6 +23,7 @@ import ( func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing.T) { fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) capability, err := fixture.handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{ Owner: serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2}, Purpose: serviceRPC.AgentCompatCapabilityNAT, @@ -44,7 +45,7 @@ func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing. Body: io.NopCloser(strings.NewReader("payload")), } request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String()) - request.Header.Set("Authorization", "Bearer deterministic-secret") + request.Header.Set("Authorization", natTestForeignJWT) request.Header.Set("X-Ordinary-NAT", "ordinary") fixture.taskStream.onSend = func(task *proto.Task) error { var nat model.TaskNAT @@ -106,8 +107,62 @@ func TestServeNATAgentCompatPublishesExactStreamAfterRequestTransfer(t *testing. require.Contains(t, forwarded, expected) } require.NotContains(t, forwarded, agentcompatcontract.IOStreamCapabilityHeader) - require.NotContains(t, forwarded, "Authorization:") - require.NotContains(t, forwarded, "deterministic-secret") + // Foreign credentials are ordinary request data for the NAT backend and + // must survive the transfer; dashboard-issued values are stripped before + // any byte leaves the dashboard. + require.Contains(t, forwarded, "Authorization: "+natTestForeignJWT) +} + +func TestServeNATAgentCompatDoesNotForwardDashboardCredentials(t *testing.T) { + fixture := newServeNATFixture(t) + installNATCredentialTestGate(t) + capability, err := fixture.handler.RegisterAgentCompatIOStreamCapability(context.Background(), serviceRPC.AgentCompatCapabilityRegistration{ + Owner: serviceRPC.AgentCompatCapabilityOwner{PATID: 1, UserID: 2}, + Purpose: serviceRPC.AgentCompatCapabilityNAT, + TargetServerID: fixture.server.ID, + ResourceID: 91, + ServerAccessAllowed: true, + }) + require.NoError(t, err) + connection := newServeNATConn() + writer := &serveNATResponseWriter{conn: connection} + agent := &serveNATAgent{readErr: io.EOF, writeDone: make(chan struct{})} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Scheme: "http", Host: "example.test", Path: "/nat"}, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader("payload")), + } + request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String()) + request.Header.Set("Authorization", natTestDashboardJWT) + fixture.taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return fixture.handler.AgentConnected(nat.StreamID, agent) + } + done := make(chan struct{}) + go func() { + ServeNAT(writer, request, &model.NAT{Common: model.Common{ID: 91}, ServerID: fixture.server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-time.After(time.Second): + t.Fatal("agent did not receive the transferred NAT request") + } + require.NoError(t, connection.Close()) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ServeNAT did not complete") + } + forwarded := strings.ToLower(string(agent.writtenBytes())) + require.NotContains(t, forwarded, "authorization:") + for _, dashboardCredential := range []string{natTestDashboardJWT, natTestDashboardExpiredJWT, natTestDashboardAPIToken} { + require.NotContains(t, forwarded, strings.ToLower(dashboardCredential)) + } } type orderedNATAgent struct { diff --git a/cmd/dashboard/rpc/nat_credential.go b/cmd/dashboard/rpc/nat_credential.go new file mode 100644 index 00000000..eb6470a8 --- /dev/null +++ b/cmd/dashboard/rpc/nat_credential.go @@ -0,0 +1,41 @@ +package rpc + +import "net/http" + +// natDashboardCredentialGate classifies an Authorization header value as a +// credential issued by this dashboard (panel-signed JWT or panel API token). +// main() injects controller.IsDashboardCredential; the setter keeps the rpc +// package free of a dependency on the controller package, the same pattern as +// SetReceiptGateListener and SetMCPKillSwitchObserver. +var natDashboardCredentialGate func(authz string) bool + +// SetNATDashboardCredentialGate wires the dashboard-credential classifier used +// by the NAT ingress. Passing nil restores the fail-closed default in +// stripDashboardCredential. +func SetNATDashboardCredentialGate(fn func(authz string) bool) { + natDashboardCredentialGate = fn +} + +// stripDashboardCredential removes the Authorization header when it carries a +// credential issued by this dashboard (panel-signed JWT or panel API token). +// Such credentials authenticate the dashboard itself and must never become +// origin credentials for the NAT backend. Foreign credentials are forwarded +// untouched: the NAT backend may authenticate with credentials of its own, and +// this dashboard must not make policy for them. +func stripDashboardCredential(request *http.Request) { + if natDashboardCredentialGate == nil { + // Gate not wired (early startup or tests): fail closed and keep the + // historical behavior of stripping every Authorization header. + request.Header.Del("Authorization") + return + } + // A multi-valued Authorization header is malformed traffic, but any + // dashboard-issued value in it must still not survive into the tunnel, so + // the whole header is dropped when one is recognized. + for _, value := range request.Header.Values("Authorization") { + if natDashboardCredentialGate(value) { + request.Header.Del("Authorization") + return + } + } +} diff --git a/cmd/dashboard/rpc/nat_credential_test.go b/cmd/dashboard/rpc/nat_credential_test.go new file mode 100644 index 00000000..0d5690d1 --- /dev/null +++ b/cmd/dashboard/rpc/nat_credential_test.go @@ -0,0 +1,37 @@ +package rpc + +import ( + "testing" + + "github.com/nezhahq/nezha/model" +) + +// Values standing in for the credentials this dashboard issues. The fake gate +// below mirrors the contract of controller.IsDashboardCredential; the real +// classification semantics — panel JWTs accepted regardless of expiry, +// foreign-signed JWTs and random strings rejected — are pinned by the +// controller package tests (dashboard_credential_test.go). +const ( + natTestDashboardJWT = "Bearer dashboard-signed-jwt" + natTestDashboardExpiredJWT = "Bearer dashboard-signed-expired-jwt" + natTestDashboardAPIToken = "Bearer " + model.APITokenPrefix + "dashboard-issued-pat" + natTestForeignJWT = "Bearer foreign-signed-jwt" + natTestForeignRandom = "Bearer foreign-random-string" +) + +// installNATCredentialTestGate wires a fake dashboard-credential classifier +// for the duration of the test: exactly the dashboard-issued test values above +// are classified as dashboard credentials, everything else is foreign. +func installNATCredentialTestGate(t *testing.T) { + t.Helper() + original := natDashboardCredentialGate + natDashboardCredentialGate = func(authz string) bool { + switch authz { + case natTestDashboardJWT, natTestDashboardExpiredJWT, natTestDashboardAPIToken: + return true + default: + return false + } + } + t.Cleanup(func() { natDashboardCredentialGate = original }) +} From 3ad985f36ef6dc31fa2e8a33965b02a016cf14db Mon Sep 17 00:00:00 2001 From: xvkong233 <88175491+xvkong233@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:02:15 +0800 Subject: [PATCH 36/48] fix(nat): scrub dashboard-issued credentials from all accepted token channels The initial selective strip covered only the Authorization header, but the dashboard also accepts its JWT via the ?token= query parameter and the nz-jwt cookie (jwt.go TokenLookup channels). A dashboard credential arriving through those channels would still reach a NAT backend. Apply the same classify-then-strip rule to all three channels at NAT ingress: values verified as dashboard-issued are removed, foreign values are preserved byte-for-byte. Adds a ServeNAT flow test exercising the live classifier and a wiring test pinning the gate injection in main(). --- .../controller/dashboard_credential.go | 30 ++- .../dashboard_credential_nat_flow_test.go | 250 ++++++++++++++++++ .../controller/dashboard_credential_test.go | 46 ++++ .../dashboard_credential_wiring_test.go | 27 ++ cmd/dashboard/main.go | 18 +- cmd/dashboard/rpc/nat.go | 9 +- .../rpc/nat_capability_agentcompat.go | 16 +- .../rpc/nat_capability_agentcompat_test.go | 90 ++++++- cmd/dashboard/rpc/nat_capability_default.go | 9 +- .../rpc/nat_capability_default_test.go | 58 +++- cmd/dashboard/rpc/nat_credential.go | 189 +++++++++++-- cmd/dashboard/rpc/nat_credential_test.go | 146 ++++++++-- 12 files changed, 806 insertions(+), 82 deletions(-) create mode 100644 cmd/dashboard/controller/dashboard_credential_nat_flow_test.go create mode 100644 cmd/dashboard/dashboard_credential_wiring_test.go diff --git a/cmd/dashboard/controller/dashboard_credential.go b/cmd/dashboard/controller/dashboard_credential.go index e24ecdd7..00af53ea 100644 --- a/cmd/dashboard/controller/dashboard_credential.go +++ b/cmd/dashboard/controller/dashboard_credential.go @@ -43,19 +43,33 @@ func newDashboardCredentialJWTParser(authMiddleware *ginjwt.GinJWTMiddleware) *g // NAT backend may authenticate with credentials of its own. // // Decision policy mirrors the dashboard's own auth middlewares: only the -// case-sensitive "Bearer " scheme is considered; PAT-shaped values are -// resolved against the api_tokens table; everything else is treated as a JWT. -// Whenever the classification cannot be completed (parser or config not ready, -// database failure) the value is conservatively reported as a dashboard -// credential so the caller strips it; only a deterministic proof that the -// value was not issued by this dashboard (scheme mismatch, unknown PAT hash, -// failed signature verification) returns false. +// case-sensitive "Bearer " scheme is considered; the value after the scheme is +// classified by IsDashboardCredentialValue, which is also the entry point for +// the panel's scheme-less TokenLookup channels. Only a deterministic proof +// that the value was not issued by this dashboard (scheme mismatch, unknown +// PAT hash, failed signature verification) returns false. func IsDashboardCredential(authz string) bool { raw := strings.TrimSpace(authz) if !strings.HasPrefix(raw, "Bearer ") { return false } - plaintext := strings.TrimSpace(strings.TrimPrefix(raw, "Bearer ")) + return IsDashboardCredentialValue(strings.TrimPrefix(raw, "Bearer ")) +} + +// IsDashboardCredentialValue reports whether a raw credential value — the part +// after the "Bearer " scheme, exactly as the panel's other TokenLookup +// channels carry it (the ?token= query parameter and the nz-jwt cookie, see +// initParams) — was issued by this dashboard: a panel-signed JWT or a panel +// API token (PAT). +// +// Failure policy is identical to IsDashboardCredential: whenever the +// classification cannot be completed (parser or config not ready, database +// failure) the value is conservatively reported as a dashboard credential so +// the caller strips it; only a deterministic proof that the value was not +// issued by this dashboard (unknown PAT hash, failed signature verification) +// returns false. +func IsDashboardCredentialValue(value string) bool { + plaintext := strings.TrimSpace(value) if strings.HasPrefix(plaintext, model.APITokenPrefix) { return isDashboardAPIToken(plaintext) } diff --git a/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go b/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go new file mode 100644 index 00000000..26a8f6ff --- /dev/null +++ b/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go @@ -0,0 +1,250 @@ +package controller + +import ( + "bufio" + "bytes" + "context" + "io" + "net" + "net/http" + "net/url" + "sync" + "testing" + "time" + + "github.com/goccy/go-json" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/metadata" + + "github.com/nezhahq/nezha/cmd/dashboard/rpc" + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/proto" + rpcService "github.com/nezhahq/nezha/service/rpc" + "github.com/nezhahq/nezha/service/singleton" +) + +// setupNATFlowTest extends setupDashboardCredentialTest with the minimal +// ServeNAT environment: a registered server with a task stream and a fresh +// handler singleton, mirroring the rpc package's serveNAT fixture. +func setupNATFlowTest(t *testing.T) (*rpcService.NezhaHandler, *model.Server, *natFlowTaskStream) { + t.Helper() + setupDashboardCredentialTest(t) + + require.NoError(t, singleton.DB.AutoMigrate(&model.Server{})) + originalServerShared, originalHandler := singleton.ServerShared, rpcService.NezhaHandlerSingleton + t.Cleanup(func() { + singleton.ServerShared, rpcService.NezhaHandlerSingleton = originalServerShared, originalHandler + }) + + serverRow := &model.Server{Common: model.Common{ID: 7}, UUID: "nat-credential-flow-test", Name: "nat-credential-flow-test"} + require.NoError(t, singleton.DB.Create(serverRow).Error) + singleton.ServerShared = singleton.NewServerClass() + handler := rpcService.NewNezhaHandler() + rpcService.NezhaHandlerSingleton = handler + + server, ok := singleton.ServerShared.Get(serverRow.ID) + require.True(t, ok) + taskStream := &natFlowTaskStream{} + server.SetTaskStream(taskStream) + + // Wire the gate exactly as cmd/dashboard/main.go does, with the real + // classifiers built by setupDashboardCredentialTest. + rpc.SetNATDashboardCredentialGate(IsDashboardCredential, IsDashboardCredentialValue) + t.Cleanup(func() { rpc.SetNATDashboardCredentialGate(nil, nil) }) + + return handler, server, taskStream +} + +// TestServeNATWithRealClassifierStripsDashboardCredentialsFromAllChannels pins +// the main() wiring contract end to end: with the real classifiers wired as +// main() does, ServeNAT forwards a request whose dashboard-issued credentials +// — Authorization header, ?token= query parameter, nz-jwt cookie — are all +// stripped, while foreign values and untouched query/cookie data survive +// byte-for-byte. +func TestServeNATWithRealClassifierStripsDashboardCredentialsFromAllChannels(t *testing.T) { + handler, server, taskStream := setupNATFlowTest(t) + + dashboardJWT := mintDashboardCredentialJWT(t, time.Hour) + foreignJWT := mintForeignJWT(t) + // No api_tokens row hashes to this PAT, so the classifier proves it was + // not issued by this dashboard and it must survive the tunnel. + foreignPAT := model.APITokenPrefix + "foreign-issued-pat" + dashboardPAT := model.APITokenPrefix + "dashboard-issued-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-flow", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + connection := &natFlowConn{closed: make(chan struct{})} + agent := &natFlowAgent{writeDone: make(chan struct{})} + writer := &natFlowWriter{conn: connection} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + dashboardJWT + "&token=" + url.QueryEscape(foreignJWT) + + "&token=" + foreignPAT + "&token=" + dashboardPAT}, + Header: make(http.Header), + Body: http.NoBody, + } + request.Header.Set("Authorization", "Bearer "+dashboardJWT) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+dashboardJWT+"; nz-jwt="+foreignJWT+"; nz-jwt="+foreignPAT) + request.Header.Set("X-Ordinary-Nat", "ordinary") + taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return handler.AgentConnected(nat.StreamID, agent) + } + + done := make(chan struct{}) + go func() { + rpc.ServeNAT(writer, request, &model.NAT{ServerID: server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-time.After(time.Second): + t.Fatal("agent did not receive the transferred NAT request") + } + require.NoError(t, connection.Close()) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ServeNAT did not finish") + } + + forwarded := string(agent.writtenBytes()) + // Dashboard credentials are gone from every channel. + require.NotContains(t, forwarded, "Authorization:") + require.NotContains(t, forwarded, dashboardJWT) + require.NotContains(t, forwarded, dashboardPAT) + // Foreign credentials and untouched data survive byte-for-byte. + require.Contains(t, forwarded, "POST /nat?keep=1&token="+url.QueryEscape(foreignJWT)+"&token="+foreignPAT+" HTTP/1.1") + require.Contains(t, forwarded, "Cookie: sid=abc; nz-jwt="+foreignJWT+"; nz-jwt="+foreignPAT) + require.Contains(t, forwarded, "X-Ordinary-Nat: ordinary") +} + +// TestServeNATWithRealClassifierKeepsForeignCredentials pins that the real +// classifier deterministically proves foreign values were not issued by this +// dashboard and leaves every channel untouched. +func TestServeNATWithRealClassifierKeepsForeignCredentials(t *testing.T) { + handler, server, taskStream := setupNATFlowTest(t) + + foreignJWT := mintForeignJWT(t) + connection := &natFlowConn{closed: make(chan struct{})} + agent := &natFlowAgent{writeDone: make(chan struct{})} + writer := &natFlowWriter{conn: connection} + request := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Path: "/nat", RawQuery: "token=" + foreignJWT}, + Header: make(http.Header), + Body: http.NoBody, + } + request.Header.Set("Authorization", "Bearer "+foreignJWT) + request.Header.Set("Cookie", "nz-jwt="+foreignJWT) + taskStream.onSend = func(task *proto.Task) error { + var nat model.TaskNAT + if err := json.Unmarshal([]byte(task.Data), &nat); err != nil { + return err + } + return handler.AgentConnected(nat.StreamID, agent) + } + + done := make(chan struct{}) + go func() { + rpc.ServeNAT(writer, request, &model.NAT{ServerID: server.ID, Host: "target.example"}) + close(done) + }() + select { + case <-agent.writeDone: + case <-time.After(time.Second): + t.Fatal("agent did not receive the transferred NAT request") + } + require.NoError(t, connection.Close()) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("ServeNAT did not finish") + } + + forwarded := string(agent.writtenBytes()) + require.Contains(t, forwarded, "POST /nat?token="+foreignJWT+" HTTP/1.1") + require.Contains(t, forwarded, "Authorization: Bearer "+foreignJWT) + require.Contains(t, forwarded, "Cookie: nz-jwt="+foreignJWT) +} + +type natFlowTaskStream struct { + onSend func(*proto.Task) error +} + +func (stream *natFlowTaskStream) Send(task *proto.Task) error { + if stream.onSend != nil { + return stream.onSend(task) + } + return nil +} +func (*natFlowTaskStream) Recv() (*proto.TaskResult, error) { return nil, io.EOF } +func (*natFlowTaskStream) SetHeader(metadata.MD) error { return nil } +func (*natFlowTaskStream) SendHeader(metadata.MD) error { return nil } +func (*natFlowTaskStream) SetTrailer(metadata.MD) {} +func (*natFlowTaskStream) Context() context.Context { return context.Background() } +func (*natFlowTaskStream) SendMsg(any) error { return nil } +func (*natFlowTaskStream) RecvMsg(any) error { return io.EOF } + +type natFlowWriter struct{ conn *natFlowConn } + +func (writer *natFlowWriter) Header() http.Header { return http.Header{} } +func (writer *natFlowWriter) Write(data []byte) (int, error) { return len(data), nil } +func (writer *natFlowWriter) WriteHeader(int) {} +func (writer *natFlowWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + go func() { _, _ = writer.conn.Read(make([]byte, 1)) }() + return writer.conn, bufio.NewReadWriter(bufio.NewReader(writer.conn), bufio.NewWriter(writer.conn)), nil +} + +type natFlowConn struct { + closed chan struct{} + closeOnce sync.Once +} + +func (conn *natFlowConn) Read([]byte) (int, error) { + <-conn.closed + return 0, io.EOF +} +func (conn *natFlowConn) Write(data []byte) (int, error) { return len(data), nil } +func (conn *natFlowConn) Close() error { + conn.closeOnce.Do(func() { close(conn.closed) }) + return nil +} +func (conn *natFlowConn) LocalAddr() net.Addr { return natFlowAddr{} } +func (conn *natFlowConn) RemoteAddr() net.Addr { return natFlowAddr{} } +func (conn *natFlowConn) SetDeadline(time.Time) error { return nil } +func (conn *natFlowConn) SetReadDeadline(time.Time) error { return nil } +func (conn *natFlowConn) SetWriteDeadline(time.Time) error { return nil } + +type natFlowAddr struct{} + +func (natFlowAddr) Network() string { return "test" } +func (natFlowAddr) String() string { return "test" } + +type natFlowAgent struct { + mu sync.Mutex + written bytes.Buffer + writeDone chan struct{} + once sync.Once +} + +func (agent *natFlowAgent) Read([]byte) (int, error) { return 0, io.EOF } +func (agent *natFlowAgent) Write(data []byte) (int, error) { + agent.mu.Lock() + agent.written.Write(data) + agent.mu.Unlock() + agent.once.Do(func() { close(agent.writeDone) }) + return len(data), nil +} +func (*natFlowAgent) Close() error { return nil } +func (agent *natFlowAgent) writtenBytes() []byte { + agent.mu.Lock() + defer agent.mu.Unlock() + return append([]byte(nil), agent.written.Bytes()...) +} diff --git a/cmd/dashboard/controller/dashboard_credential_test.go b/cmd/dashboard/controller/dashboard_credential_test.go index 9ad1070e..fbfe47ec 100644 --- a/cmd/dashboard/controller/dashboard_credential_test.go +++ b/cmd/dashboard/controller/dashboard_credential_test.go @@ -117,6 +117,52 @@ func TestIsDashboardCredentialClassification(t *testing.T) { }) } +// TestIsDashboardCredentialValueClassification pins the scheme-less classifier +// that backs IsDashboardCredential and drives the ?token= query parameter and +// nz-jwt cookie channels at NAT ingress (see IsDashboardCredentialValue). +func TestIsDashboardCredentialValueClassification(t *testing.T) { + setupDashboardCredentialTest(t) + + dashboardJWT := mintDashboardCredentialJWT(t, time.Hour) + expiredDashboardJWT := mintDashboardCredentialJWT(t, -time.Hour) + foreignJWT := mintForeignJWT(t) + + dashboardPAT := model.APITokenPrefix + "dashboard-credential-value-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-value-classification", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + t.Run("dashboard signed jwt is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredentialValue(dashboardJWT)) + }) + t.Run("expired dashboard signed jwt is still a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredentialValue(expiredDashboardJWT)) + }) + t.Run("dashboard api token is a dashboard credential", func(t *testing.T) { + require.True(t, IsDashboardCredentialValue(dashboardPAT)) + }) + t.Run("foreign signed jwt is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue(foreignJWT)) + }) + t.Run("unknown api token is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue(model.APITokenPrefix+"unknown-to-the-dashboard")) + }) + t.Run("random string is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue("random-not-a-token")) + }) + t.Run("empty value is not a dashboard credential", func(t *testing.T) { + require.False(t, IsDashboardCredentialValue("")) + }) + t.Run("bearer scheme is not part of the value contract", func(t *testing.T) { + // The query and cookie channels carry the bare value; a value that + // itself starts with "Bearer " is not something the panel issues and + // fails signature verification. + require.False(t, IsDashboardCredentialValue("Bearer "+dashboardJWT)) + }) +} + func TestIsDashboardCredentialFailsClosedWhenParserUnavailable(t *testing.T) { setupDashboardCredentialTest(t) dashboardCredentialJWTParser = nil diff --git a/cmd/dashboard/dashboard_credential_wiring_test.go b/cmd/dashboard/dashboard_credential_wiring_test.go new file mode 100644 index 00000000..95be89ff --- /dev/null +++ b/cmd/dashboard/dashboard_credential_wiring_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" + + "github.com/nezhahq/nezha/cmd/dashboard/rpc" +) + +// TestWireNATDashboardCredentialGate pins the NAT ingress wiring contract: +// main() must connect the real dashboard-credential classifiers +// (controller.IsDashboardCredential / controller.IsDashboardCredentialValue) +// through wireNATDashboardCredentialGate before any listener serves traffic. +// That the wired functions really classify per channel is pinned end to end +// by the controller package's real-classifier NAT flow test. +func TestWireNATDashboardCredentialGate(t *testing.T) { + if rpc.NATDashboardCredentialGateWired() { + t.Fatal("NAT dashboard credential gate already wired before the test") + } + + wireNATDashboardCredentialGate() + + if !rpc.NATDashboardCredentialGateWired() { + t.Fatal("wireNATDashboardCredentialGate left the NAT dashboard credential gate unwired") + } + // Leave the process in the wired state main() establishes. + t.Cleanup(wireNATDashboardCredentialGate) +} diff --git a/cmd/dashboard/main.go b/cmd/dashboard/main.go index 50d02590..4b64c441 100644 --- a/cmd/dashboard/main.go +++ b/cmd/dashboard/main.go @@ -169,11 +169,7 @@ func main() { grpcHandler := rpc.ServeRPC() httpHandler := controller.ServeWeb(frontendDist) controller.InitUpgrader() - // The NAT ingress strips dashboard-issued Authorization values (panel JWT / - // panel PAT) before tunneling a request to the agent and forwards foreign - // credentials untouched. ServeWeb built the JWT parser the classifier - // reuses, so the gate is wired right after it. - rpc.SetNATDashboardCredentialGate(controller.IsDashboardCredential) + wireNATDashboardCredentialGate() muxHandler := newHTTPandGRPCMux(httpHandler, grpcHandler) muxServerHTTP := &http.Server{ @@ -233,6 +229,18 @@ func main() { close(errChan) } +// wireNATDashboardCredentialGate connects the NAT ingress to the real +// dashboard-credential classifiers. The NAT ingress strips dashboard-issued +// credentials (panel JWT / panel PAT) from every channel the panel's own +// TokenLookup accepts — the Authorization header, the ?token= query parameter +// and the nz-jwt cookie — before tunneling a request to the agent, and +// forwards foreign values untouched. ServeWeb built the JWT parser the +// classifiers reuse, so the gate is wired right after it and before any +// listener starts serving. +func wireNATDashboardCredentialGate() { + rpc.SetNATDashboardCredentialGate(controller.IsDashboardCredential, controller.IsDashboardCredentialValue) +} + func newHTTPandGRPCMux(httpHandler http.Handler, grpcHandler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { natConfig := singleton.NATShared.GetNATConfigByDomain(r.Host) diff --git a/cmd/dashboard/rpc/nat.go b/cmd/dashboard/rpc/nat.go index 5c519493..7b8be72b 100644 --- a/cmd/dashboard/rpc/nat.go +++ b/cmd/dashboard/rpc/nat.go @@ -84,10 +84,11 @@ func ServeNAT(w http.ResponseWriter, r *http.Request, natConfig *model.NAT) { // dashboard — the panel-signed JWT (verified by signature only, expiry // deliberately ignored) and the panel API token — must never become origin // credentials for the configured NAT backend. prepareNATCapability has - // already stripped every Authorization value IsDashboardCredential - // classified as dashboard-issued; whatever remains is either absent or a - // foreign credential, which is forwarded untouched because the NAT backend - // may authenticate with credentials of its own. + // already stripped them from every channel the panel's own TokenLookup + // accepts (Authorization header, ?token= query parameter, nz-jwt cookie); + // whatever remains on those channels is either absent or a foreign + // credential, which is forwarded untouched because the NAT backend may + // authenticate with credentials of its own. wWrapped, err := utils.NewRequestWrapper(r, w) if err != nil { return diff --git a/cmd/dashboard/rpc/nat_capability_agentcompat.go b/cmd/dashboard/rpc/nat_capability_agentcompat.go index e2c3bc67..16fdf503 100644 --- a/cmd/dashboard/rpc/nat_capability_agentcompat.go +++ b/cmd/dashboard/rpc/nat_capability_agentcompat.go @@ -23,23 +23,25 @@ func prepareNATCapability(request *http.Request, natConfig *model.NAT) (natCapab values := request.Header.Values(agentcompatcontract.IOStreamCapabilityHeader) if len(values) == 0 { // No capability header: legacy NAT path. Strip dashboard-issued - // Authorization values (panel JWT or panel PAT) so they never become - // origin credentials; foreign ones are ordinary request data. - stripDashboardCredential(request) + // credentials (panel JWT or panel PAT) from the Authorization header, + // the ?token= query parameter and the nz-jwt cookie so they never + // become origin credentials; foreign values are ordinary request data. + stripDashboardCredentials(request) return natCapabilityLease{}, nil } request.Header.Del(agentcompatcontract.IOStreamCapabilityHeader) if len(values) != 1 || values[0] == "" { // Invalid capability: nothing is forwarded, but dashboard-issued // credentials still must not survive into errors or task data. - stripDashboardCredential(request) + stripDashboardCredentials(request) return natCapabilityLease{}, errors.New("invalid NAT capability") } access, handle, err := serviceRPC.NezhaHandlerSingleton.ConsumeAgentCompatNATCapabilityForProfile(values[0], natConfig.ServerID, natConfig.ID) // From here the request bytes may reach the agent (active lease) or the - // request ends in a 503; either way dashboard-issued Authorization values - // are stripped, foreign ones are forwarded untouched. - stripDashboardCredential(request) + // request ends in a 503; either way dashboard-issued credentials are + // stripped from every TokenLookup channel, foreign ones are forwarded + // untouched. + stripDashboardCredentials(request) if err != nil { return natCapabilityLease{}, errors.New("invalid NAT capability") } diff --git a/cmd/dashboard/rpc/nat_capability_agentcompat_test.go b/cmd/dashboard/rpc/nat_capability_agentcompat_test.go index e1e369b4..2bfae4a8 100644 --- a/cmd/dashboard/rpc/nat_capability_agentcompat_test.go +++ b/cmd/dashboard/rpc/nat_capability_agentcompat_test.go @@ -7,6 +7,7 @@ import ( "context" "log" "net/http" + "net/url" "strings" "testing" @@ -62,16 +63,21 @@ func TestPrepareNATCapabilityRejectsDuplicateHeaderAfterRemovingAllValues(t *tes func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentials(t *testing.T) { // Given — legacy path without a capability header, then the invalid - // capability path: both must drop dashboard-issued Authorization values. + // capability path: both must drop dashboard-issued credentials from the + // Authorization header, the ?token= query parameter and the nz-jwt cookie. installNATCredentialTestGate(t) for name, credential := range map[string]string{ - "dashboard jwt": natTestDashboardJWT, - "expired dashboard jwt": natTestDashboardExpiredJWT, - "dashboard api token": natTestDashboardAPIToken, + "dashboard jwt": natTestDashboardJWTValue, + "expired dashboard jwt": natTestDashboardExpiredJWTValue, + "dashboard api token": natTestDashboardPATValue, } { t.Run(name+"/legacy path", func(t *testing.T) { - request := &http.Request{Header: make(http.Header)} - request.Header.Set("Authorization", credential) + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential}, + } + request.Header.Set("Authorization", "Bearer "+credential) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential) lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) @@ -82,13 +88,23 @@ func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentials(t *testing.T) t.Fatal("legacy NAT path unexpectedly activated") } if got := request.Header.Get("Authorization"); got != "" { - t.Fatalf("legacy NAT path retained dashboard credential %q", got) + t.Fatalf("legacy NAT path retained dashboard credential in Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("legacy NAT path retained dashboard credential in query %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("legacy NAT path retained dashboard credential in cookie %q", got) } }) t.Run(name+"/invalid capability path", func(t *testing.T) { - request := &http.Request{Header: make(http.Header)} + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential}, + } request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, "malformed") - request.Header.Set("Authorization", credential) + request.Header.Set("Authorization", "Bearer "+credential) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential) lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) @@ -99,7 +115,13 @@ func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentials(t *testing.T) t.Fatal("malformed capability unexpectedly activated") } if got := request.Header.Get("Authorization"); got != "" { - t.Fatalf("invalid capability path retained dashboard credential %q", got) + t.Fatalf("invalid capability path retained dashboard credential in Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("invalid capability path retained dashboard credential in query %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("invalid capability path retained dashboard credential in cookie %q", got) } }) } @@ -124,9 +146,13 @@ func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentialAfterConsumingC if err != nil { t.Fatalf("register capability: %v", err) } - request := &http.Request{Header: make(http.Header)} + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestDashboardJWTValue}, + } request.Header.Set(agentcompatcontract.IOStreamCapabilityHeader, capability.String()) request.Header.Set("Authorization", natTestDashboardJWT) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestDashboardJWTValue) // When lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) @@ -139,7 +165,13 @@ func TestPrepareNATCapabilityAgentCompatStripsDashboardCredentialAfterConsumingC t.Fatal("valid NAT capability did not activate") } if got := request.Header.Get("Authorization"); got != "" { - t.Fatalf("dashboard credential retained after capability consumption: %q", got) + t.Fatalf("dashboard credential retained in Authorization after capability consumption: %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("dashboard credential retained in query after capability consumption: %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("dashboard credential retained in cookie after capability consumption: %q", got) } } @@ -180,6 +212,40 @@ func TestPrepareNATCapabilityAgentCompatKeepsForeignAndMissingAuthorization(t *t } } +func TestPrepareNATCapabilityAgentCompatKeepsForeignAndMissingQueryAndCookie(t *testing.T) { + // Given — legacy path: foreign query tokens and cookies are ordinary + // request data, and absent channels must stay untouched. + installNATCredentialTestGate(t) + for name, tc := range map[string]struct{ rawQuery, cookie string }{ + "foreign values": {"keep=1&token=" + natTestForeignJWTValue + "&a=%2Fx", "sid=abc; nz-jwt=" + natTestForeignJWTValue}, + "empty values": {"keep=1&token=", "nz-jwt="}, + "missing channel": {"keep=1", "sid=abc"}, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: tc.rawQuery}, + } + request.Header.Set("Cookie", tc.cookie) + + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + if err != nil { + t.Fatalf("legacy NAT path returned error: %v", err) + } + if lease.active { + t.Fatal("legacy NAT path unexpectedly activated") + } + if got := request.URL.RawQuery; got != tc.rawQuery { + t.Fatalf("legacy NAT path changed query to %q, want %q", got, tc.rawQuery) + } + if got := request.Header.Get("Cookie"); got != tc.cookie { + t.Fatalf("legacy NAT path changed cookie to %q, want %q", got, tc.cookie) + } + }) + } +} + func TestServeNATAgentCompatSensitiveHeadersStayOutOfErrorsAndLogs(t *testing.T) { installNATCredentialTestGate(t) request := &http.Request{Header: make(http.Header)} diff --git a/cmd/dashboard/rpc/nat_capability_default.go b/cmd/dashboard/rpc/nat_capability_default.go index 865c8f08..b0480ce6 100644 --- a/cmd/dashboard/rpc/nat_capability_default.go +++ b/cmd/dashboard/rpc/nat_capability_default.go @@ -19,10 +19,11 @@ type natCapabilityLease struct { func prepareNATCapability(request *http.Request, _ *model.NAT) (natCapabilityLease, error) { // The default build has no capability protocol: every request takes the - // legacy NAT path, so any dashboard-issued Authorization value (panel JWT - // or panel PAT) is stripped before the request is tunneled to the agent. - // Foreign Authorization values are ordinary request data and stay put. - stripDashboardCredential(request) + // legacy NAT path, so any dashboard-issued credential (panel JWT or panel + // PAT) carried by the Authorization header, the ?token= query parameter or + // the nz-jwt cookie is stripped before the request is tunneled to the + // agent. Foreign values are ordinary request data and stay put. + stripDashboardCredentials(request) return natCapabilityLease{}, nil } diff --git a/cmd/dashboard/rpc/nat_capability_default_test.go b/cmd/dashboard/rpc/nat_capability_default_test.go index 71154084..40827939 100644 --- a/cmd/dashboard/rpc/nat_capability_default_test.go +++ b/cmd/dashboard/rpc/nat_capability_default_test.go @@ -4,6 +4,7 @@ package rpc import ( "net/http" + "net/url" "testing" "github.com/nezhahq/nezha/model" @@ -39,13 +40,17 @@ func TestPrepareNATCapabilityDefaultStripsDashboardCredentials(t *testing.T) { // Given installNATCredentialTestGate(t) for name, credential := range map[string]string{ - "dashboard jwt": natTestDashboardJWT, - "expired dashboard jwt": natTestDashboardExpiredJWT, - "dashboard api token": natTestDashboardAPIToken, + "dashboard jwt": natTestDashboardJWTValue, + "expired dashboard jwt": natTestDashboardExpiredJWTValue, + "dashboard api token": natTestDashboardPATValue, } { t.Run(name, func(t *testing.T) { - request := &http.Request{Header: make(http.Header)} - request.Header.Set("Authorization", credential) + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + credential}, + } + request.Header.Set("Authorization", "Bearer "+credential) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+credential) // When lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) @@ -58,7 +63,13 @@ func TestPrepareNATCapabilityDefaultStripsDashboardCredentials(t *testing.T) { t.Fatal("default NAT capability hook unexpectedly activated") } if got := request.Header.Get("Authorization"); got != "" { - t.Fatalf("default NAT capability hook retained dashboard credential %q", got) + t.Fatalf("default NAT capability hook retained dashboard credential in Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("default NAT capability hook retained dashboard credential in query %q", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("default NAT capability hook retained dashboard credential in cookie %q", got) } }) } @@ -101,3 +112,38 @@ func TestPrepareNATCapabilityDefaultKeepsForeignAndMissingAuthorization(t *testi }) } } + +func TestPrepareNATCapabilityDefaultKeepsForeignAndMissingQueryAndCookie(t *testing.T) { + // Given + installNATCredentialTestGate(t) + for name, tc := range map[string]struct{ rawQuery, cookie string }{ + "foreign values": {"keep=1&token=" + natTestForeignJWTValue + "&a=%2Fx", "sid=abc; nz-jwt=" + natTestForeignJWTValue}, + "empty values": {"keep=1&token=", "nz-jwt="}, + "missing channel": {"keep=1", "sid=abc"}, + } { + t.Run(name, func(t *testing.T) { + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: tc.rawQuery}, + } + request.Header.Set("Cookie", tc.cookie) + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("default NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("default NAT capability hook unexpectedly activated") + } + if got := request.URL.RawQuery; got != tc.rawQuery { + t.Fatalf("default NAT capability hook changed query to %q, want %q", got, tc.rawQuery) + } + if got := request.Header.Get("Cookie"); got != tc.cookie { + t.Fatalf("default NAT capability hook changed cookie to %q, want %q", got, tc.cookie) + } + }) + } +} diff --git a/cmd/dashboard/rpc/nat_credential.go b/cmd/dashboard/rpc/nat_credential.go index eb6470a8..a9e82f4a 100644 --- a/cmd/dashboard/rpc/nat_credential.go +++ b/cmd/dashboard/rpc/nat_credential.go @@ -1,29 +1,69 @@ package rpc -import "net/http" +import ( + "net/http" + "net/url" + "strings" +) -// natDashboardCredentialGate classifies an Authorization header value as a -// credential issued by this dashboard (panel-signed JWT or panel API token). -// main() injects controller.IsDashboardCredential; the setter keeps the rpc -// package free of a dependency on the controller package, the same pattern as -// SetReceiptGateListener and SetMCPKillSwitchObserver. -var natDashboardCredentialGate func(authz string) bool +// Channel names of the panel's own TokenLookup (controller/jwt.go +// "header: Authorization, query: token, cookie: nz-jwt"). A dashboard JWT +// can therefore reach the NAT ingress through the Authorization header, the +// ?token= query parameter or the nz-jwt cookie, and all three must be +// classified before the request is tunneled to the agent. +const ( + natDashboardJWTCookieName = "nz-jwt" + natDashboardJWTQueryParameter = "token" +) -// SetNATDashboardCredentialGate wires the dashboard-credential classifier used -// by the NAT ingress. Passing nil restores the fail-closed default in -// stripDashboardCredential. -func SetNATDashboardCredentialGate(fn func(authz string) bool) { - natDashboardCredentialGate = fn +var ( + // natDashboardAuthorizationGate classifies an Authorization header value + // (case-sensitive "Bearer " scheme included) as a credential issued by + // this dashboard. + natDashboardAuthorizationGate func(authz string) bool + // natDashboardCredentialValueGate classifies a raw credential value — the + // part after the "Bearer " scheme, exactly as the ?token= query parameter + // and the nz-jwt cookie carry it — as a credential issued by this + // dashboard. + natDashboardCredentialValueGate func(value string) bool +) + +// SetNATDashboardCredentialGate wires the dashboard-credential classifiers +// used by the NAT ingress. main() injects +// controller.IsDashboardCredential and controller.IsDashboardCredentialValue; +// the setter keeps the rpc package free of a dependency on the controller +// package, the same pattern as SetReceiptGateListener and +// SetMCPKillSwitchObserver. A nil function fails closed: every value on that +// channel is then treated as dashboard-issued and stripped. +func SetNATDashboardCredentialGate(authorizationClass func(authz string) bool, valueClass func(value string) bool) { + natDashboardAuthorizationGate = authorizationClass + natDashboardCredentialValueGate = valueClass } -// stripDashboardCredential removes the Authorization header when it carries a -// credential issued by this dashboard (panel-signed JWT or panel API token). -// Such credentials authenticate the dashboard itself and must never become -// origin credentials for the NAT backend. Foreign credentials are forwarded -// untouched: the NAT backend may authenticate with credentials of its own, and -// this dashboard must not make policy for them. -func stripDashboardCredential(request *http.Request) { - if natDashboardCredentialGate == nil { +// NATDashboardCredentialGateWired reports whether the dashboard-credential +// classifiers have been wired. It exists so the main() wiring contract can be +// pinned by a test. +func NATDashboardCredentialGateWired() bool { + return natDashboardAuthorizationGate != nil && natDashboardCredentialValueGate != nil +} + +// stripDashboardCredentials removes credentials issued by this dashboard from +// every channel the panel's own TokenLookup accepts: the Authorization +// header, the ?token= query parameter and the nz-jwt cookie. Such credentials +// authenticate the dashboard itself and must never become origin credentials +// for the NAT backend. Foreign values are forwarded untouched: the NAT +// backend may authenticate with credentials of its own, and this dashboard +// must not make policy for them. +func stripDashboardCredentials(request *http.Request) { + stripDashboardAuthorizationHeader(request) + stripDashboardCredentialQuery(request) + stripDashboardCredentialCookie(request) +} + +// stripDashboardAuthorizationHeader removes the Authorization header when it +// carries a dashboard-issued credential. +func stripDashboardAuthorizationHeader(request *http.Request) { + if natDashboardAuthorizationGate == nil { // Gate not wired (early startup or tests): fail closed and keep the // historical behavior of stripping every Authorization header. request.Header.Del("Authorization") @@ -33,9 +73,116 @@ func stripDashboardCredential(request *http.Request) { // dashboard-issued value in it must still not survive into the tunnel, so // the whole header is dropped when one is recognized. for _, value := range request.Header.Values("Authorization") { - if natDashboardCredentialGate(value) { + if natDashboardAuthorizationGate(value) { request.Header.Del("Authorization") return } } } + +// stripDashboardCredentialQuery removes ?token= query parameters whose value +// is a dashboard-issued credential. Every other element of the query string — +// including foreign token values — is preserved byte-for-byte, so the +// forwarded request keeps the exact query the client sent. +func stripDashboardCredentialQuery(request *http.Request) { + if request.URL == nil || request.URL.RawQuery == "" { + return + } + elements := strings.Split(request.URL.RawQuery, "&") + kept := elements[:0] + changed := false + for _, element := range elements { + if isDashboardCredentialQueryElement(element) { + changed = true + continue + } + kept = append(kept, element) + } + if changed { + request.URL.RawQuery = strings.Join(kept, "&") + } +} + +// isDashboardCredentialQueryElement reports whether a raw query element +// carries a dashboard-issued credential in the panel's token parameter. +func isDashboardCredentialQueryElement(element string) bool { + rawName, rawValue, _ := strings.Cut(element, "=") + // The panel reads the parameter by its decoded name (gin-jwt c.Query), + // so an escaped key must be decoded before comparing. + name, err := url.QueryUnescape(rawName) + if err != nil || name != natDashboardJWTQueryParameter { + return false + } + if natDashboardCredentialValueGate == nil { + return true // fail closed + } + // A value that fails to decode is one the panel's own query parsing drops + // as well, so it can never be a panel credential. + decoded, err := url.QueryUnescape(rawValue) + if err != nil { + return false + } + return natDashboardCredentialValueGate(decoded) +} + +// stripDashboardCredentialCookie removes nz-jwt cookies whose value is a +// dashboard-issued credential. All other cookie pairs — including foreign +// nz-jwt values — are preserved verbatim. +func stripDashboardCredentialCookie(request *http.Request) { + lines := request.Header.Values("Cookie") + if len(lines) == 0 { + return + } + changed := false + kept := make([]string, 0, len(lines)) + for _, line := range lines { + nextLine, lineChanged := stripDashboardCredentialCookieLine(line) + changed = changed || lineChanged + if nextLine != "" { + kept = append(kept, nextLine) + } + } + if !changed { + return + } + request.Header.Del("Cookie") + for _, line := range kept { + request.Header.Add("Cookie", line) + } +} + +// stripDashboardCredentialCookieLine rewrites a single Cookie header line, +// dropping the dashboard-issued nz-jwt pairs. Kept pairs keep their raw form, +// whitespace included — except that a pair following a dropped one no longer +// carries the removed pair's share of the "; " separator. +func stripDashboardCredentialCookieLine(line string) (string, bool) { + pairs := strings.Split(line, ";") + kept := pairs[:0] + droppedBeforeFirstKept := true + changed := false + for _, pair := range pairs { + name, value, _ := strings.Cut(pair, "=") + if strings.TrimSpace(name) == natDashboardJWTCookieName && + (natDashboardCredentialValueGate == nil || isDashboardCredentialCookieValue(value)) { + changed = true + continue + } + if droppedBeforeFirstKept { + pair = strings.TrimLeft(pair, " ") + } + droppedBeforeFirstKept = false + kept = append(kept, pair) + } + if !changed { + return line, false + } + return strings.Join(kept, ";"), true +} + +// isDashboardCredentialCookieValue classifies a raw cookie value the same way +// the panel's cookie parsing would hand it to the JWT verifier: surrounding +// whitespace and quoting are presentation, not part of the credential. +func isDashboardCredentialCookieValue(rawValue string) bool { + value := strings.Trim(strings.TrimSpace(rawValue), `"`) + return natDashboardCredentialValueGate(value) +} diff --git a/cmd/dashboard/rpc/nat_credential_test.go b/cmd/dashboard/rpc/nat_credential_test.go index 0d5690d1..2f7a2be9 100644 --- a/cmd/dashboard/rpc/nat_credential_test.go +++ b/cmd/dashboard/rpc/nat_credential_test.go @@ -1,31 +1,43 @@ package rpc import ( + "net/http" + "net/url" "testing" "github.com/nezhahq/nezha/model" ) -// Values standing in for the credentials this dashboard issues. The fake gate -// below mirrors the contract of controller.IsDashboardCredential; the real -// classification semantics — panel JWTs accepted regardless of expiry, -// foreign-signed JWTs and random strings rejected — are pinned by the -// controller package tests (dashboard_credential_test.go). +// Values standing in for the credentials this dashboard issues. The fake +// gates below mirror the contract of controller.IsDashboardCredential and +// controller.IsDashboardCredentialValue; the real classification semantics — +// panel JWTs accepted regardless of expiry, foreign-signed JWTs and random +// strings rejected — are pinned by the controller package tests +// (dashboard_credential_test.go) and by the flow test that wires the real +// classifiers into ServeNAT (dashboard_credential_nat_flow_test.go). const ( - natTestDashboardJWT = "Bearer dashboard-signed-jwt" - natTestDashboardExpiredJWT = "Bearer dashboard-signed-expired-jwt" - natTestDashboardAPIToken = "Bearer " + model.APITokenPrefix + "dashboard-issued-pat" - natTestForeignJWT = "Bearer foreign-signed-jwt" - natTestForeignRandom = "Bearer foreign-random-string" + natTestDashboardJWTValue = "dashboard-signed-jwt" + natTestDashboardExpiredJWTValue = "dashboard-signed-expired-jwt" + natTestDashboardPATValue = model.APITokenPrefix + "dashboard-issued-pat" + natTestForeignJWTValue = "foreign-signed-jwt" + natTestForeignRandomValue = "foreign-random-string" + + natTestDashboardJWT = "Bearer " + natTestDashboardJWTValue + natTestDashboardExpiredJWT = "Bearer " + natTestDashboardExpiredJWTValue + natTestDashboardAPIToken = "Bearer " + natTestDashboardPATValue + natTestForeignJWT = "Bearer " + natTestForeignJWTValue + natTestForeignRandom = "Bearer " + natTestForeignRandomValue ) -// installNATCredentialTestGate wires a fake dashboard-credential classifier +// installNATCredentialTestGate wires fake dashboard-credential classifiers // for the duration of the test: exactly the dashboard-issued test values above -// are classified as dashboard credentials, everything else is foreign. +// are classified as dashboard credentials on every channel, everything else +// is foreign. func installNATCredentialTestGate(t *testing.T) { t.Helper() - original := natDashboardCredentialGate - natDashboardCredentialGate = func(authz string) bool { + originalAuthorization := natDashboardAuthorizationGate + originalValue := natDashboardCredentialValueGate + natDashboardAuthorizationGate = func(authz string) bool { switch authz { case natTestDashboardJWT, natTestDashboardExpiredJWT, natTestDashboardAPIToken: return true @@ -33,5 +45,109 @@ func installNATCredentialTestGate(t *testing.T) { return false } } - t.Cleanup(func() { natDashboardCredentialGate = original }) + natDashboardCredentialValueGate = func(value string) bool { + switch value { + case natTestDashboardJWTValue, natTestDashboardExpiredJWTValue, natTestDashboardPATValue: + return true + default: + return false + } + } + t.Cleanup(func() { + natDashboardAuthorizationGate = originalAuthorization + natDashboardCredentialValueGate = originalValue + }) +} + +func TestStripDashboardCredentialsQueryChannel(t *testing.T) { + // Given + installNATCredentialTestGate(t) + request := &http.Request{URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestDashboardJWTValue + "&token=" + natTestForeignJWTValue + "&a=%2Fx"}} + + // When + stripDashboardCredentials(request) + + // Then + if got := request.URL.RawQuery; got != "keep=1&token="+natTestForeignJWTValue+"&a=%2Fx" { + t.Fatalf("query channel kept %q, want dashboard token dropped and foreign values byte-preserved", got) + } +} + +func TestStripDashboardCredentialsQueryChannelEscapedValue(t *testing.T) { + // Given — a dashboard token percent-encoded in the query must still be + // recognized, while the escaped foreign element stays verbatim. + installNATCredentialTestGate(t) + request := &http.Request{URL: &url.URL{Path: "/nat", RawQuery: "token=" + url.QueryEscape(natTestDashboardJWTValue) + "&token=" + url.QueryEscape(natTestForeignJWTValue)}} + + // When + stripDashboardCredentials(request) + + // Then + if got := request.URL.RawQuery; got != "token="+url.QueryEscape(natTestForeignJWTValue) { + t.Fatalf("query channel kept %q, want escaped dashboard token dropped and foreign element verbatim", got) + } +} + +func TestStripDashboardCredentialsCookieChannel(t *testing.T) { + // Given + installNATCredentialTestGate(t) + request := &http.Request{Header: make(http.Header)} + request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestDashboardJWTValue+"; nz-jwt="+natTestForeignJWTValue+"; theme=dark") + + // When + stripDashboardCredentials(request) + + // Then + if got := request.Header.Get("Cookie"); got != "sid=abc; nz-jwt="+natTestForeignJWTValue+"; theme=dark" { + t.Fatalf("cookie channel kept %q, want dashboard pair dropped and foreign pairs verbatim", got) + } +} + +func TestStripDashboardCredentialsCookieChannelQuotedValue(t *testing.T) { + // Given — quoting is presentation the panel's cookie parsing strips before + // verifying, so a quoted dashboard token must be recognized. + installNATCredentialTestGate(t) + request := &http.Request{Header: make(http.Header)} + request.Header.Set("Cookie", `nz-jwt="`+natTestDashboardJWTValue+`"; sid=abc`) + + // When + stripDashboardCredentials(request) + + // Then + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("cookie channel kept %q, want quoted dashboard token dropped", got) + } +} + +func TestPrepareNATCapabilityFailsClosedWithoutGate(t *testing.T) { + // Given — unwired gates: every value on every channel is treated as + // dashboard-issued and stripped, foreign ones included. + SetNATDashboardCredentialGate(nil, nil) + t.Cleanup(func() { SetNATDashboardCredentialGate(nil, nil) }) + request := &http.Request{ + Header: make(http.Header), + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestForeignJWTValue}, + } + request.Header.Set("Authorization", natTestForeignRandom) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestForeignJWTValue) + + // When + lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) + + // Then + if err != nil { + t.Fatalf("fail-closed NAT capability hook returned error: %v", err) + } + if lease.active { + t.Fatal("fail-closed NAT capability hook unexpectedly activated") + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("fail-closed hook retained Authorization %q", got) + } + if got := request.URL.RawQuery; got != "keep=1" { + t.Fatalf("fail-closed hook kept query %q, want token parameter stripped", got) + } + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("fail-closed hook kept cookie %q, want nz-jwt pair stripped", got) + } } From 34ac50394d2f0c8fe9d481d464af6174587bd7f5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 1 Oct 2026 19:26:01 +0800 Subject: [PATCH 37/48] update contributors[no ci] --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b9b4a3b2..cdff6f4b 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ Creling Core F NikoCat233 -adminsama +Coming Wel Acgpiano eya46 guoyongchang @@ -132,12 +132,12 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ lvyaoting quanljh unclezs +虚空 缘生 严浩 凌 榆木 93 -Coming Wel :D Esdese GreenTeodoro839 @@ -153,6 +153,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ TomyJan Tony ace-consultoria +adminsama Chra Mubelotix Ryrenz From 1420dd450ca65e7b993387f4b5e7181b1279f2de Mon Sep 17 00:00:00 2001 From: naiba Date: Thu, 1 Oct 2026 11:48:23 +0000 Subject: [PATCH 38/48] fix(nat): bound dashboard credential classification --- .../controller/dashboard_credential.go | 90 ++++++-- .../dashboard_credential_nat_flow_test.go | 4 +- .../controller/dashboard_credential_test.go | 30 +++ .../dashboard_credential_wiring_test.go | 4 +- cmd/dashboard/main.go | 4 +- cmd/dashboard/rpc/nat_credential.go | 206 ++++++++++++++---- cmd/dashboard/rpc/nat_credential_test.go | 95 ++++++-- 7 files changed, 338 insertions(+), 95 deletions(-) diff --git a/cmd/dashboard/controller/dashboard_credential.go b/cmd/dashboard/controller/dashboard_credential.go index 00af53ea..f16e3e34 100644 --- a/cmd/dashboard/controller/dashboard_credential.go +++ b/cmd/dashboard/controller/dashboard_credential.go @@ -1,12 +1,13 @@ package controller import ( - "errors" + "context" "log" "strings" jwt "github.com/golang-jwt/jwt/v4" "gorm.io/gorm" + "gorm.io/gorm/logger" ginjwt "github.com/appleboy/gin-jwt/v2" "github.com/nezhahq/nezha/model" @@ -15,8 +16,8 @@ import ( // dashboardCredentialJWTParser verifies that a JWT was signed by this // dashboard. It is derived from the auth middleware created in routers() and -// exists only for IsDashboardCredential; it must never authenticate dashboard -// requests on its own. +// exists only for dashboard-credential classification; it must never +// authenticate dashboard requests on its own. var dashboardCredentialJWTParser *ginjwt.GinJWTMiddleware // newDashboardCredentialJWTParser derives a signature-only parser from the @@ -69,32 +70,75 @@ func IsDashboardCredential(authz string) bool { // issued by this dashboard (unknown PAT hash, failed signature verification) // returns false. func IsDashboardCredentialValue(value string) bool { - plaintext := strings.TrimSpace(value) - if strings.HasPrefix(plaintext, model.APITokenPrefix) { - return isDashboardAPIToken(plaintext) + classified := ClassifyDashboardCredentialValues(context.Background(), []string{value}) + if len(classified) != 1 { + return true } - return isDashboardSignedJWT(plaintext) + return classified[0] } -// isDashboardAPIToken resolves a PAT-shaped value against the api_tokens -// table. The plaintext token is only ever hashed here; it is never logged. -func isDashboardAPIToken(plaintext string) bool { +// ClassifyDashboardCredentialValues classifies a bounded request batch. JWTs +// are checked independently, while all PAT-shaped values are hashed and +// resolved with one silent IN query. This prevents attacker-controlled NAT +// query parameters or cookies from amplifying one HTTP request into thousands +// of SQLite queries and record-not-found log entries. +func ClassifyDashboardCredentialValues(ctx context.Context, values []string) []bool { + if ctx == nil { + ctx = context.Background() + } + results := make([]bool, len(values)) + patIndexes := make(map[string][]int) + for i, value := range values { + plaintext := strings.TrimSpace(value) + if strings.HasPrefix(plaintext, model.APITokenPrefix) { + hash := model.HashAPIToken(plaintext) + patIndexes[hash] = append(patIndexes[hash], i) + continue + } + results[i] = isDashboardSignedJWT(plaintext) + } + if len(patIndexes) == 0 { + return results + } if singleton.DB == nil { logDashboardCredentialUnclassifiable("api token lookup unavailable") - return true + for _, indexes := range patIndexes { + for _, i := range indexes { + results[i] = true + } + } + return results } - var tok model.APIToken - err := singleton.DB.Where("token_hash = ?", model.HashAPIToken(plaintext)).First(&tok).Error - if err == nil { - return true + + hashes := make([]string, 0, len(patIndexes)) + for hash := range patIndexes { + hashes = append(hashes, hash) } - if errors.Is(err, gorm.ErrRecordNotFound) { - // Deterministic proof the token was not issued by this dashboard: - // forwarding it leaks nothing the dashboard ever validated. - return false + type tokenHashRow struct { + TokenHash string } - logDashboardCredentialUnclassifiable("api token lookup failed") - return true + var rows []tokenHashRow + err := singleton.DB.WithContext(ctx). + Session(&gorm.Session{Logger: logger.Discard}). + Model(&model.APIToken{}). + Select("token_hash"). + Where("token_hash IN ?", hashes). + Find(&rows).Error + if err != nil { + logDashboardCredentialUnclassifiable("api token lookup failed") + for _, indexes := range patIndexes { + for _, i := range indexes { + results[i] = true + } + } + return results + } + for _, row := range rows { + for _, i := range patIndexes[row.TokenHash] { + results[i] = true + } + } + return results } // isDashboardSignedJWT verifies the token signature with the dashboard's @@ -112,7 +156,7 @@ func isDashboardSignedJWT(token string) bool { } // logDashboardCredentialUnclassifiable records a classification failure. The -// Authorization value itself is never included in the message. +// credential value itself is never included in the message. func logDashboardCredentialUnclassifiable(reason string) { - log.Printf("NEZHA>> NAT ingress: Authorization value could not be classified (%s); stripping it as a precaution", reason) + log.Printf("NEZHA>> NAT ingress: credential value could not be classified (%s); stripping it as a precaution", reason) } diff --git a/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go b/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go index 26a8f6ff..770e6f5e 100644 --- a/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go +++ b/cmd/dashboard/controller/dashboard_credential_nat_flow_test.go @@ -49,8 +49,8 @@ func setupNATFlowTest(t *testing.T) (*rpcService.NezhaHandler, *model.Server, *n // Wire the gate exactly as cmd/dashboard/main.go does, with the real // classifiers built by setupDashboardCredentialTest. - rpc.SetNATDashboardCredentialGate(IsDashboardCredential, IsDashboardCredentialValue) - t.Cleanup(func() { rpc.SetNATDashboardCredentialGate(nil, nil) }) + rpc.SetNATDashboardCredentialGate(ClassifyDashboardCredentialValues) + t.Cleanup(func() { rpc.SetNATDashboardCredentialGate(nil) }) return handler, server, taskStream } diff --git a/cmd/dashboard/controller/dashboard_credential_test.go b/cmd/dashboard/controller/dashboard_credential_test.go index fbfe47ec..9c1cc81c 100644 --- a/cmd/dashboard/controller/dashboard_credential_test.go +++ b/cmd/dashboard/controller/dashboard_credential_test.go @@ -2,6 +2,7 @@ package controller import ( "bytes" + "context" "log" "testing" "time" @@ -163,6 +164,35 @@ func TestIsDashboardCredentialValueClassification(t *testing.T) { }) } +func TestClassifyDashboardCredentialValuesBatchesPATLookup(t *testing.T) { + setupDashboardCredentialTest(t) + + dashboardPAT := model.APITokenPrefix + "dashboard-batch-pat" + require.NoError(t, singleton.DB.Create(&model.APIToken{ + UserID: 1, + Name: "nat-batch-classification", + TokenHash: model.HashAPIToken(dashboardPAT), + }).Error) + + queryCount := 0 + const callbackName = "test:count-dashboard-credential-batch-query" + require.NoError(t, singleton.DB.Callback().Query().Before("gorm:query").Register(callbackName, func(*gorm.DB) { + queryCount++ + })) + t.Cleanup(func() { singleton.DB.Callback().Query().Remove(callbackName) }) + + values := []string{ + dashboardPAT, + model.APITokenPrefix + "unknown-one", + dashboardPAT, + model.APITokenPrefix + "unknown-two", + } + classified := ClassifyDashboardCredentialValues(context.Background(), values) + + require.Equal(t, []bool{true, false, true, false}, classified) + require.Equal(t, 1, queryCount, "all PAT hashes in one request must use one database query") +} + func TestIsDashboardCredentialFailsClosedWhenParserUnavailable(t *testing.T) { setupDashboardCredentialTest(t) dashboardCredentialJWTParser = nil diff --git a/cmd/dashboard/dashboard_credential_wiring_test.go b/cmd/dashboard/dashboard_credential_wiring_test.go index 95be89ff..d53e7c8f 100644 --- a/cmd/dashboard/dashboard_credential_wiring_test.go +++ b/cmd/dashboard/dashboard_credential_wiring_test.go @@ -7,8 +7,8 @@ import ( ) // TestWireNATDashboardCredentialGate pins the NAT ingress wiring contract: -// main() must connect the real dashboard-credential classifiers -// (controller.IsDashboardCredential / controller.IsDashboardCredentialValue) +// main() must connect the real batched dashboard-credential classifier +// (controller.ClassifyDashboardCredentialValues) // through wireNATDashboardCredentialGate before any listener serves traffic. // That the wired functions really classify per channel is pinned end to end // by the controller package's real-classifier NAT flow test. diff --git a/cmd/dashboard/main.go b/cmd/dashboard/main.go index 4b64c441..4c41ae0a 100644 --- a/cmd/dashboard/main.go +++ b/cmd/dashboard/main.go @@ -235,10 +235,10 @@ func main() { // TokenLookup accepts — the Authorization header, the ?token= query parameter // and the nz-jwt cookie — before tunneling a request to the agent, and // forwards foreign values untouched. ServeWeb built the JWT parser the -// classifiers reuse, so the gate is wired right after it and before any +// classifier reuses, so the gate is wired right after it and before any // listener starts serving. func wireNATDashboardCredentialGate() { - rpc.SetNATDashboardCredentialGate(controller.IsDashboardCredential, controller.IsDashboardCredentialValue) + rpc.SetNATDashboardCredentialGate(controller.ClassifyDashboardCredentialValues) } func newHTTPandGRPCMux(httpHandler http.Handler, grpcHandler http.Handler) http.Handler { diff --git a/cmd/dashboard/rpc/nat_credential.go b/cmd/dashboard/rpc/nat_credential.go index a9e82f4a..153f556e 100644 --- a/cmd/dashboard/rpc/nat_credential.go +++ b/cmd/dashboard/rpc/nat_credential.go @@ -1,9 +1,11 @@ package rpc import ( + "context" "net/http" "net/url" "strings" + "sync" ) // Channel names of the panel's own TokenLookup (controller/jwt.go @@ -14,37 +16,42 @@ import ( const ( natDashboardJWTCookieName = "nz-jwt" natDashboardJWTQueryParameter = "token" + // Keep credential classification work constant even when an unauthenticated + // NAT request contains thousands of token parameters or cookies. Values + // beyond this per-request budget fail closed and are stripped. + maxNATDashboardCredentialValues = 16 ) var ( - // natDashboardAuthorizationGate classifies an Authorization header value - // (case-sensitive "Bearer " scheme included) as a credential issued by - // this dashboard. - natDashboardAuthorizationGate func(authz string) bool - // natDashboardCredentialValueGate classifies a raw credential value — the - // part after the "Bearer " scheme, exactly as the ?token= query parameter - // and the nz-jwt cookie carry it — as a credential issued by this - // dashboard. - natDashboardCredentialValueGate func(value string) bool + natDashboardCredentialGateMu sync.RWMutex + // natDashboardCredentialValueGate classifies a bounded batch of raw + // credential values. Batching lets the controller resolve all PAT hashes in + // one query instead of turning attacker-controlled query/cookie fan-out into + // one database query per value. + natDashboardCredentialValueGate func(ctx context.Context, values []string) []bool ) -// SetNATDashboardCredentialGate wires the dashboard-credential classifiers +// SetNATDashboardCredentialGate wires the dashboard-credential classifier // used by the NAT ingress. main() injects -// controller.IsDashboardCredential and controller.IsDashboardCredentialValue; +// controller.ClassifyDashboardCredentialValues; // the setter keeps the rpc package free of a dependency on the controller // package, the same pattern as SetReceiptGateListener and // SetMCPKillSwitchObserver. A nil function fails closed: every value on that // channel is then treated as dashboard-issued and stripped. -func SetNATDashboardCredentialGate(authorizationClass func(authz string) bool, valueClass func(value string) bool) { - natDashboardAuthorizationGate = authorizationClass +func SetNATDashboardCredentialGate(valueClass func(ctx context.Context, values []string) []bool) { + natDashboardCredentialGateMu.Lock() natDashboardCredentialValueGate = valueClass + natDashboardCredentialGateMu.Unlock() } // NATDashboardCredentialGateWired reports whether the dashboard-credential -// classifiers have been wired. It exists so the main() wiring contract can be +// classifier has been wired. It exists so the main() wiring contract can be // pinned by a test. func NATDashboardCredentialGateWired() bool { - return natDashboardAuthorizationGate != nil && natDashboardCredentialValueGate != nil + natDashboardCredentialGateMu.RLock() + wired := natDashboardCredentialValueGate != nil + natDashboardCredentialGateMu.RUnlock() + return wired } // stripDashboardCredentials removes credentials issued by this dashboard from @@ -55,15 +62,103 @@ func NATDashboardCredentialGateWired() bool { // backend may authenticate with credentials of its own, and this dashboard // must not make policy for them. func stripDashboardCredentials(request *http.Request) { - stripDashboardAuthorizationHeader(request) - stripDashboardCredentialQuery(request) - stripDashboardCredentialCookie(request) + classification := classifyNATDashboardCredentialValues(request) + stripDashboardAuthorizationHeader(request, classification) + stripDashboardCredentialQuery(request, classification) + stripDashboardCredentialCookie(request, classification) +} + +type natDashboardCredentialClassification struct { + wired bool + limited bool + issued map[string]bool +} + +func (c natDashboardCredentialClassification) isDashboardCredential(value string) bool { + if !c.wired { + return true + } + if issued, ok := c.issued[value]; ok { + return issued + } + // Only candidate values collected from this request reach this method. A + // missing result therefore means the distinct-value budget was exceeded; + // fail closed so a dashboard credential cannot be hidden after junk values. + return c.limited +} + +func classifyNATDashboardCredentialValues(request *http.Request) natDashboardCredentialClassification { + natDashboardCredentialGateMu.RLock() + gate := natDashboardCredentialValueGate + natDashboardCredentialGateMu.RUnlock() + classification := natDashboardCredentialClassification{ + wired: gate != nil, + issued: make(map[string]bool), + } + if gate == nil { + return classification + } + + values, limited := collectNATDashboardCredentialValues(request) + classification.limited = limited + results := gate(request.Context(), values) + if len(results) != len(values) { + // A broken classifier must not turn into credential forwarding. + for _, value := range values { + classification.issued[value] = true + } + classification.limited = true + return classification + } + for i, value := range values { + classification.issued[value] = results[i] + } + return classification +} + +func collectNATDashboardCredentialValues(request *http.Request) ([]string, bool) { + values := make([]string, 0, maxNATDashboardCredentialValues) + seen := make(map[string]struct{}, maxNATDashboardCredentialValues) + limited := false + add := func(value string) { + if _, ok := seen[value]; ok { + return + } + if len(values) == maxNATDashboardCredentialValues { + limited = true + return + } + seen[value] = struct{}{} + values = append(values, value) + } + + for _, authorization := range request.Header.Values("Authorization") { + raw := strings.TrimSpace(authorization) + if strings.HasPrefix(raw, "Bearer ") { + add(strings.TrimPrefix(raw, "Bearer ")) + } + } + if request.URL != nil { + for _, element := range strings.Split(request.URL.RawQuery, "&") { + if value, ok := dashboardCredentialQueryElementValue(element); ok { + add(value) + } + } + } + for _, line := range request.Header.Values("Cookie") { + for _, pair := range strings.Split(line, ";") { + if value, ok := dashboardCredentialCookiePairValue(pair); ok { + add(value) + } + } + } + return values, limited } // stripDashboardAuthorizationHeader removes the Authorization header when it // carries a dashboard-issued credential. -func stripDashboardAuthorizationHeader(request *http.Request) { - if natDashboardAuthorizationGate == nil { +func stripDashboardAuthorizationHeader(request *http.Request, classification natDashboardCredentialClassification) { + if !classification.wired { // Gate not wired (early startup or tests): fail closed and keep the // historical behavior of stripping every Authorization header. request.Header.Del("Authorization") @@ -73,7 +168,8 @@ func stripDashboardAuthorizationHeader(request *http.Request) { // dashboard-issued value in it must still not survive into the tunnel, so // the whole header is dropped when one is recognized. for _, value := range request.Header.Values("Authorization") { - if natDashboardAuthorizationGate(value) { + raw := strings.TrimSpace(value) + if strings.HasPrefix(raw, "Bearer ") && classification.isDashboardCredential(strings.TrimPrefix(raw, "Bearer ")) { request.Header.Del("Authorization") return } @@ -84,7 +180,7 @@ func stripDashboardAuthorizationHeader(request *http.Request) { // is a dashboard-issued credential. Every other element of the query string — // including foreign token values — is preserved byte-for-byte, so the // forwarded request keeps the exact query the client sent. -func stripDashboardCredentialQuery(request *http.Request) { +func stripDashboardCredentialQuery(request *http.Request, classification natDashboardCredentialClassification) { if request.URL == nil || request.URL.RawQuery == "" { return } @@ -92,7 +188,12 @@ func stripDashboardCredentialQuery(request *http.Request) { kept := elements[:0] changed := false for _, element := range elements { - if isDashboardCredentialQueryElement(element) { + if !classification.wired && dashboardCredentialQueryElementName(element) { + changed = true + continue + } + value, candidate := dashboardCredentialQueryElementValue(element) + if candidate && classification.isDashboardCredential(value) { changed = true continue } @@ -103,32 +204,35 @@ func stripDashboardCredentialQuery(request *http.Request) { } } -// isDashboardCredentialQueryElement reports whether a raw query element -// carries a dashboard-issued credential in the panel's token parameter. -func isDashboardCredentialQueryElement(element string) bool { +// dashboardCredentialQueryElementValue extracts the decoded value from a raw +// query element carrying the panel's token parameter. +func dashboardCredentialQueryElementValue(element string) (string, bool) { rawName, rawValue, _ := strings.Cut(element, "=") // The panel reads the parameter by its decoded name (gin-jwt c.Query), // so an escaped key must be decoded before comparing. name, err := url.QueryUnescape(rawName) if err != nil || name != natDashboardJWTQueryParameter { - return false - } - if natDashboardCredentialValueGate == nil { - return true // fail closed + return "", false } // A value that fails to decode is one the panel's own query parsing drops // as well, so it can never be a panel credential. decoded, err := url.QueryUnescape(rawValue) if err != nil { - return false + return "", false } - return natDashboardCredentialValueGate(decoded) + return decoded, true +} + +func dashboardCredentialQueryElementName(element string) bool { + rawName, _, _ := strings.Cut(element, "=") + name, err := url.QueryUnescape(rawName) + return err == nil && name == natDashboardJWTQueryParameter } // stripDashboardCredentialCookie removes nz-jwt cookies whose value is a // dashboard-issued credential. All other cookie pairs — including foreign // nz-jwt values — are preserved verbatim. -func stripDashboardCredentialCookie(request *http.Request) { +func stripDashboardCredentialCookie(request *http.Request, classification natDashboardCredentialClassification) { lines := request.Header.Values("Cookie") if len(lines) == 0 { return @@ -136,7 +240,7 @@ func stripDashboardCredentialCookie(request *http.Request) { changed := false kept := make([]string, 0, len(lines)) for _, line := range lines { - nextLine, lineChanged := stripDashboardCredentialCookieLine(line) + nextLine, lineChanged := stripDashboardCredentialCookieLine(line, classification) changed = changed || lineChanged if nextLine != "" { kept = append(kept, nextLine) @@ -155,15 +259,18 @@ func stripDashboardCredentialCookie(request *http.Request) { // dropping the dashboard-issued nz-jwt pairs. Kept pairs keep their raw form, // whitespace included — except that a pair following a dropped one no longer // carries the removed pair's share of the "; " separator. -func stripDashboardCredentialCookieLine(line string) (string, bool) { +func stripDashboardCredentialCookieLine(line string, classification natDashboardCredentialClassification) (string, bool) { pairs := strings.Split(line, ";") kept := pairs[:0] droppedBeforeFirstKept := true changed := false for _, pair := range pairs { - name, value, _ := strings.Cut(pair, "=") - if strings.TrimSpace(name) == natDashboardJWTCookieName && - (natDashboardCredentialValueGate == nil || isDashboardCredentialCookieValue(value)) { + if !classification.wired && dashboardCredentialCookiePairName(pair) { + changed = true + continue + } + value, candidate := dashboardCredentialCookiePairValue(pair) + if candidate && classification.isDashboardCredential(value) { changed = true continue } @@ -179,10 +286,23 @@ func stripDashboardCredentialCookieLine(line string) (string, bool) { return strings.Join(kept, ";"), true } -// isDashboardCredentialCookieValue classifies a raw cookie value the same way -// the panel's cookie parsing would hand it to the JWT verifier: surrounding -// whitespace and quoting are presentation, not part of the credential. -func isDashboardCredentialCookieValue(rawValue string) bool { +// dashboardCredentialCookiePairValue extracts a cookie value the same way the +// panel hands it to the JWT verifier: surrounding whitespace/quotes are +// presentation, then Gin applies QueryUnescape. +func dashboardCredentialCookiePairValue(pair string) (string, bool) { + name, rawValue, found := strings.Cut(pair, "=") + if !found || strings.TrimSpace(name) != natDashboardJWTCookieName { + return "", false + } value := strings.Trim(strings.TrimSpace(rawValue), `"`) - return natDashboardCredentialValueGate(value) + decoded, err := url.QueryUnescape(value) + if err != nil { + return "", false + } + return decoded, true +} + +func dashboardCredentialCookiePairName(pair string) bool { + name, _, found := strings.Cut(pair, "=") + return found && strings.TrimSpace(name) == natDashboardJWTCookieName } diff --git a/cmd/dashboard/rpc/nat_credential_test.go b/cmd/dashboard/rpc/nat_credential_test.go index 2f7a2be9..234e78f2 100644 --- a/cmd/dashboard/rpc/nat_credential_test.go +++ b/cmd/dashboard/rpc/nat_credential_test.go @@ -1,8 +1,11 @@ package rpc import ( + "context" + "fmt" "net/http" "net/url" + "strings" "testing" "github.com/nezhahq/nezha/model" @@ -35,27 +38,21 @@ const ( // is foreign. func installNATCredentialTestGate(t *testing.T) { t.Helper() - originalAuthorization := natDashboardAuthorizationGate - originalValue := natDashboardCredentialValueGate - natDashboardAuthorizationGate = func(authz string) bool { - switch authz { - case natTestDashboardJWT, natTestDashboardExpiredJWT, natTestDashboardAPIToken: - return true - default: - return false + natDashboardCredentialGateMu.RLock() + original := natDashboardCredentialValueGate + natDashboardCredentialGateMu.RUnlock() + SetNATDashboardCredentialGate(func(_ context.Context, values []string) []bool { + results := make([]bool, len(values)) + for i, value := range values { + switch value { + case natTestDashboardJWTValue, natTestDashboardExpiredJWTValue, natTestDashboardPATValue: + results[i] = true + } } - } - natDashboardCredentialValueGate = func(value string) bool { - switch value { - case natTestDashboardJWTValue, natTestDashboardExpiredJWTValue, natTestDashboardPATValue: - return true - default: - return false - } - } + return results + }) t.Cleanup(func() { - natDashboardAuthorizationGate = originalAuthorization - natDashboardCredentialValueGate = originalValue + SetNATDashboardCredentialGate(original) }) } @@ -119,17 +116,69 @@ func TestStripDashboardCredentialsCookieChannelQuotedValue(t *testing.T) { } } +func TestStripDashboardCredentialsCookieChannelEscapedValue(t *testing.T) { + // Gin's cookie TokenLookup URL-decodes the value before JWT validation. + // NAT must classify the same decoded value or an encoded dashboard JWT can + // be accepted by the panel and then leak unchanged to the backend. + installNATCredentialTestGate(t) + request := &http.Request{Header: make(http.Header)} + escaped := strings.ReplaceAll(natTestDashboardJWTValue, "-", "%2D") + request.Header.Set("Cookie", "nz-jwt="+escaped+"; sid=abc") + + stripDashboardCredentials(request) + + if got := request.Header.Get("Cookie"); got != "sid=abc" { + t.Fatalf("cookie channel kept %q, want escaped dashboard token dropped", got) + } +} + +func TestStripDashboardCredentialsDeduplicatesAndBoundsClassification(t *testing.T) { + natDashboardCredentialGateMu.RLock() + original := natDashboardCredentialValueGate + natDashboardCredentialGateMu.RUnlock() + t.Cleanup(func() { SetNATDashboardCredentialGate(original) }) + + classifiedValues := 0 + gateCalls := 0 + SetNATDashboardCredentialGate(func(_ context.Context, values []string) []bool { + gateCalls++ + classifiedValues += len(values) + return make([]bool, len(values)) + }) + + query := make(url.Values) + for range 100 { + query.Add("token", model.APITokenPrefix+"same-value") + } + for i := range 100 { + query.Add("token", fmt.Sprintf("%sunique-%d", model.APITokenPrefix, i)) + } + request := &http.Request{Header: make(http.Header), URL: &url.URL{RawQuery: query.Encode()}} + + stripDashboardCredentials(request) + + if gateCalls != 1 { + t.Fatalf("classifier called %d times, want one batched call", gateCalls) + } + if classifiedValues > maxNATDashboardCredentialValues { + t.Fatalf("classified %d credential values, want at most %d per request", classifiedValues, maxNATDashboardCredentialValues) + } + if strings.Contains(request.URL.RawQuery, model.APITokenPrefix+"unique-99") { + t.Fatal("credential beyond the classification budget survived; overflow must fail closed") + } +} + func TestPrepareNATCapabilityFailsClosedWithoutGate(t *testing.T) { // Given — unwired gates: every value on every channel is treated as // dashboard-issued and stripped, foreign ones included. - SetNATDashboardCredentialGate(nil, nil) - t.Cleanup(func() { SetNATDashboardCredentialGate(nil, nil) }) + SetNATDashboardCredentialGate(nil) + t.Cleanup(func() { SetNATDashboardCredentialGate(nil) }) request := &http.Request{ Header: make(http.Header), - URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestForeignJWTValue}, + URL: &url.URL{Path: "/nat", RawQuery: "keep=1&token=" + natTestForeignJWTValue + "&token=%ZZ"}, } request.Header.Set("Authorization", natTestForeignRandom) - request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestForeignJWTValue) + request.Header.Set("Cookie", "sid=abc; nz-jwt="+natTestForeignJWTValue+"; nz-jwt=%ZZ") // When lease, err := prepareNATCapability(request, &model.NAT{Common: model.Common{ID: 91}, ServerID: 81}) From 8328047aa8aeca0a2ec19c00c930e968eeddee58 Mon Sep 17 00:00:00 2001 From: Kris <116849421+cantoblanco@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:31:32 +0200 Subject: [PATCH 39/48] fix(alerts): evaluate fresh reports and order notification delivery (#1242) * fix(alerts): evaluate fresh reports and make notification transitions reliable * fix(alerts): identify equal-timestamp reports by sequence * fix(alerts): preserve notification transition semantics --------- Co-authored-by: naiba --- cmd/dashboard/controller/alertrule.go | 6 + model/alert_evaluation_test.go | 205 +++++++++ model/alert_timed_test.go | 104 +++++ model/alertrule.go | 149 ++++++- model/notification.go | 102 ++--- model/rule.go | 262 +++++++----- model/server.go | 24 +- service/singleton/alertsentinel.go | 211 ++++++++-- .../singleton/alertsentinel_freshness_test.go | 281 +++++++++++++ service/singleton/alertsentinel_test.go | 2 +- service/singleton/notification.go | 359 +++++++++++++--- service/singleton/notification_test.go | 393 ++++++++++++++++++ service/singleton/server.go | 1 + service/singleton/servicesentinel.go | 330 +++++++++------ .../singleton/servicesentinel_window_test.go | 63 +++ service/singleton/singleton.go | 2 +- 16 files changed, 2122 insertions(+), 372 deletions(-) create mode 100644 model/alert_evaluation_test.go create mode 100644 model/alert_timed_test.go create mode 100644 service/singleton/alertsentinel_freshness_test.go create mode 100644 service/singleton/servicesentinel_window_test.go diff --git a/cmd/dashboard/controller/alertrule.go b/cmd/dashboard/controller/alertrule.go index 85021a47..5e5ce2d0 100644 --- a/cmd/dashboard/controller/alertrule.go +++ b/cmd/dashboard/controller/alertrule.go @@ -178,6 +178,9 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { if !rule.IsSupportedType() { return singleton.Localizer.ErrorT("unsupported rule type") } + if !rule.HasSafeThresholds() { + return singleton.Localizer.ErrorT("invalid rule thresholds") + } switch rule.Cover { case model.RuleCoverAll, model.RuleCoverIgnoreAll: default: @@ -192,6 +195,9 @@ func validateRule(c *gin.Context, r *model.AlertRule) error { return singleton.Localizer.ErrorT("duration is too large") } } else { + if !rule.HasSafeCycleUnit() { + return singleton.Localizer.ErrorT("invalid cycle unit") + } if rule.CycleInterval < 1 { return singleton.Localizer.ErrorT("cycle_interval need to be at least 1") } diff --git a/model/alert_evaluation_test.go b/model/alert_evaluation_test.go new file mode 100644 index 00000000..3267685c --- /dev/null +++ b/model/alert_evaluation_test.go @@ -0,0 +1,205 @@ +package model + +import ( + "math" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestAlertEvaluationDistinguishesUnknownFromBreach(t *testing.T) { + server := &Server{Common: Common{ID: 1}} + rule := &Rule{Type: "cpu", Max: 80, Cover: RuleCoverAll} + if passed, known := rule.Evaluate(nil, server, nil); !passed || known { + t.Fatalf("missing state = (%v, %v), want unknown", passed, known) + } + lease := server.AttachStateStream(runtimeOwnershipStream{}) + if !lease.UpdateState(&HostState{CPU: 99}, time.Now()) { + t.Fatal("state update rejected") + } + if passed, known := rule.Evaluate(nil, server, nil); passed || !known { + t.Fatalf("fresh high CPU = (%v, %v), want known breach", passed, known) + } + if !lease.Clear() { + t.Fatal("stream clear rejected") + } + // The raw metric remains available for historical display. The sentinel + // enforces freshness and must not count this value after disconnect. + if !server.RuntimeSnapshot().LastActive.IsZero() { + t.Fatal("disconnect did not clear online visibility") + } +} + +func newCycleTestDB(t *testing.T, migrate bool) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if migrate { + if err := db.AutoMigrate(&Transfer{}); err != nil { + t.Fatal(err) + } + } + return db +} + +func newCycleTestStats() *CycleTransferStats { + return &CycleTransferStats{ + ServerName: make(map[uint64]string), + Transfer: make(map[uint64]uint64), + NextUpdate: make(map[uint64]time.Time), + } +} + +func TestCycleRuleRejectsInvalidConfiguration(t *testing.T) { + start := time.Now().Add(-time.Hour) + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100} + if !rule.HasSafeCycleConfiguration() { + t.Fatal("normal hourly cycle rejected") + } + for _, tc := range []struct { + name string + mutate func(*Rule) + }{ + {"unknown unit", func(r *Rule) { r.CycleUnit = "fortnight" }}, + {"negative limit", func(r *Rule) { r.Max = -1 }}, + {"NaN limit", func(r *Rule) { r.Max = math.NaN() }}, + {"reversed limits", func(r *Rule) { r.Min, r.Max = 200, 100 }}, + {"future start", func(r *Rule) { future := time.Now().Add(time.Hour); r.CycleStart = &future }}, + } { + t.Run(tc.name, func(t *testing.T) { + copy := *rule + tc.mutate(©) + if copy.HasSafeCycleConfiguration() { + t.Fatal("unsafe cycle accepted") + } + }) + } +} + +func TestCycleBoundsUseOneClockAtBoundary(t *testing.T) { + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 100} + before := start.Add(59*time.Minute + 59*time.Second) + from, to := rule.transferDurationBounds(before) + if !from.Equal(start) || !to.Equal(start.Add(time.Hour)) { + t.Fatalf("before boundary = %v..%v", from, to) + } + from, to = rule.transferDurationBounds(start.Add(time.Hour)) + if !from.Equal(start.Add(time.Hour)) || !to.Equal(start.Add(2*time.Hour)) { + t.Fatalf("at boundary = %v..%v", from, to) + } +} + +func TestCycleQueryFailureIsUnknownAndDoesNotCache(t *testing.T) { + start := time.Now().Add(-time.Hour) + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50} + server := &Server{Common: Common{ID: 1}, State: &HostState{}} + passed, known := rule.Evaluate(newCycleTestStats(), server, newCycleTestDB(t, false)) + if !passed || known || len(rule.NextTransferAt) != 0 { + t.Fatalf("SQL error yielded passed=%v known=%v cache=%v", passed, known, rule.NextTransferAt) + } +} + +func TestCycleQueryExcludesFutureRowsAndHandlesMinOnly(t *testing.T) { + start := time.Now().Add(-90 * time.Minute) + db := newCycleTestDB(t, true) + server := &Server{Common: Common{ID: 1}, State: &HostState{}} + for _, row := range []Transfer{ + {Common: Common{CreatedAt: time.Now().Add(-time.Minute)}, ServerID: 1, Out: 10}, + {Common: Common{CreatedAt: time.Now().Add(10 * time.Minute)}, ServerID: 1, Out: 100}, + } { + if err := db.Create(&row).Error; err != nil { + t.Fatal(err) + } + } + rule := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Max: 50} + stats := newCycleTestStats() + passed, known := rule.Evaluate(stats, server, db) + if !passed || !known || stats.Transfer[1] != 10 { + t.Fatalf("future row included: passed=%v known=%v amount=%d", passed, known, stats.Transfer[1]) + } + minOnly := &Rule{Type: "transfer_out_cycle", CycleStart: &start, CycleInterval: 1, CycleUnit: "hour", Min: 20} + passed, known = minOnly.Evaluate(newCycleTestStats(), server, db) + if passed || !known { + t.Fatalf("min-only cycle = (%v,%v), want known breach", passed, known) + } +} + +func TestAlertEvaluationAllNetworkSpeedUsesBothDirections(t *testing.T) { + for _, tc := range []struct { + name string + in, out uint64 + breach bool + }{ + {"inbound only", 100, 0, true}, + {"outbound below combined limit", 0, 30, false}, + {"both directions", 25, 30, true}, + } { + t.Run(tc.name, func(t *testing.T) { + server := &Server{Common: Common{ID: 1}, State: &HostState{NetInSpeed: tc.in, NetOutSpeed: tc.out}} + rule := &Rule{Type: "net_all_speed", Max: 50, Cover: RuleCoverAll} + passed, known := rule.Evaluate(nil, server, nil) + if !known || passed == tc.breach { + t.Fatalf("got passed=%v known=%v, want breach=%v", passed, known, tc.breach) + } + }) + } +} + +func TestAlertEvaluationUnavailableSensorsAreUnknown(t *testing.T) { + tests := []struct { + name string + rule string + state *HostState + host *Host + }{ + {"missing memory host", "memory", &HostState{MemUsed: 1}, nil}, + {"zero memory total", "memory", &HostState{MemUsed: 1}, &Host{}}, + {"missing GPU", "gpu_max", &HostState{}, &Host{}}, + {"missing temperature", "temperature_max", &HostState{}, &Host{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := &Server{Common: Common{ID: 1}, State: tc.state, Host: tc.host} + passed, known := (&Rule{Type: tc.rule, Max: 50, Cover: RuleCoverAll}).Evaluate(nil, server, nil) + if !passed || known { + t.Fatalf("got passed=%v known=%v, want unknown", passed, known) + } + }) + } +} + +func TestOfflineRuleDoesNotRequireMetricState(t *testing.T) { + server := &Server{Common: Common{ID: 1}} + InitServer(server) + passed, known := (&Rule{Type: "offline", Cover: RuleCoverAll}).Evaluate(nil, server, nil) + if passed || !known { + t.Fatalf("offline without state = (%v, %v), want known breach", passed, known) + } +} + +func TestCompoundAlertUsesOneImmutableAgentReport(t *testing.T) { + server := &Server{Common: Common{ID: 1}} + alert := &AlertRule{Rules: []*Rule{ + {Type: "cpu", Max: 80, Cover: RuleCoverAll}, + {Type: "memory", Max: 80, Cover: RuleCoverAll}, + }} + oldReport := RuntimeSnapshot{ + State: &HostState{CPU: 99, MemUsed: 0}, + Host: &Host{MemTotal: 100}, + } + newReport := RuntimeSnapshot{ + State: &HostState{CPU: 0, MemUsed: 99}, + Host: &Host{MemTotal: 100}, + } + for _, report := range []RuntimeSnapshot{oldReport, newReport} { + point, known := alert.SnapshotStatusWithRuntime(nil, server, report, nil) + if !known || len(point) != 2 || point[0] == point[1] { + t.Fatalf("compound point combined incompatible reports: known=%v point=%v", known, point) + } + } +} diff --git a/model/alert_timed_test.go b/model/alert_timed_test.go new file mode 100644 index 00000000..9a3eb16e --- /dev/null +++ b/model/alert_timed_test.go @@ -0,0 +1,104 @@ +package model + +import ( + "testing" + "time" +) + +func timedRule(kind string, duration uint64) *AlertRule { + return &AlertRule{Rules: []*Rule{{Type: kind, Duration: duration, Max: 80, Cover: RuleCoverAll}}} +} + +func timedPoints(start time.Time, values []bool, spacing time.Duration) []TimedAlertPoint { + points := make([]TimedAlertPoint, len(values)) + for i, value := range values { + points[i] = TimedAlertPoint{At: start.Add(time.Duration(i) * spacing), Values: []bool{value}} + } + return points +} + +func TestTimedAlertNeedsElapsedWindowAndDistinctReports(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 5) + points := timedPoints(start, []bool{false, false, false, false}, AlertSampleInterval) + if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known { + t.Fatal("four reports decided a five-tick window") + } + points = append(points, TimedAlertPoint{At: start.Add(12 * time.Second), Values: []bool{false}}) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed { + t.Fatalf("complete high-CPU window: known=%v passed=%v", known, passed) + } + // Repeating the same observation does not advance wall-clock coverage. + if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known { + t.Fatal("stale sample created a new complete window") + } +} + +func TestTimedAlertGapIsUnknownAndCannotResolve(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 5) + points := []TimedAlertPoint{ + {At: start, Values: []bool{false}}, + {At: start.Add(30 * time.Second), Values: []bool{true}}, + } + if known, _ := rule.CheckTimed(points, start.Add(30*time.Second)); known { + t.Fatal("telemetry gap must not be interpreted as healthy") + } +} + +func TestTimedAlertIrregularReportsUseElapsedTime(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 6) + points := timedPoints(start, []bool{false, false, false, false}, 5*time.Second) + if known, passed := rule.CheckTimed(points, start.Add(15*time.Second)); !known || passed { + t.Fatalf("irregular reports cover the wall-clock window: known=%v passed=%v", known, passed) + } +} + +func TestTimedAlertExactSeventyPercentIsNotIncident(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 10) + values := []bool{false, false, false, false, false, false, false, true, true, true} + points := timedPoints(start, values, AlertSampleInterval) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed { + t.Fatalf("70%% failed is below the >70%% trigger: known=%v passed=%v", known, passed) + } +} + +func TestTimedAlertTenTickWindowDoesNotDecideAtNine(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("cpu", 10) + points := timedPoints(start, []bool{false, false, false, false, false, false, false, false, false}, AlertSampleInterval) + if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known { + t.Fatal("nine reports decided a ten-tick window one tick early") + } + points = append(points, TimedAlertPoint{At: start.Add(27 * time.Second), Values: []bool{false}}) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed { + t.Fatalf("full ten-tick breach: known=%v passed=%v", known, passed) + } +} + +func TestTimedOfflineRequiresSustainedAbsence(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := timedRule("offline", 5) + points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || passed { + t.Fatalf("continuous offline interval: known=%v passed=%v", known, passed) + } + points = append(points, TimedAlertPoint{At: start.Add(15 * time.Second), Values: []bool{true}}) + if known, passed := rule.CheckTimed(points, points[len(points)-1].At); !known || !passed { + t.Fatalf("online observation should resolve offline: known=%v passed=%v", known, passed) + } +} + +func TestTimedCompoundRuleWaitsForAllKnown(t *testing.T) { + start := time.Unix(1_700_000_000, 0) + rule := &AlertRule{Rules: []*Rule{ + {Type: "cpu", Duration: 5, Max: 80, Cover: RuleCoverAll}, + {Type: "memory", Duration: 5, Max: 80, Cover: RuleCoverAll}, + }} + points := timedPoints(start, []bool{false, false, false, false, false}, AlertSampleInterval) + if known, _ := rule.CheckTimed(points, points[len(points)-1].At); known { + t.Fatal("missing second condition treated as decisive") + } +} diff --git a/model/alertrule.go b/model/alertrule.go index 886ea0ce..c4f65dae 100644 --- a/model/alertrule.go +++ b/model/alertrule.go @@ -2,12 +2,134 @@ package model import ( "slices" + "time" "github.com/gin-gonic/gin" "github.com/goccy/go-json" "gorm.io/gorm" ) +const ( + AlertSampleInterval = 3 * time.Second + AlertSampleMaxAge = 6 * time.Second +) + +// TimedAlertPoint records one distinct observation. Its timestamp is the +// dashboard receipt time of the Agent report, or the sentinel tick for an +// offline-only rule. +type TimedAlertPoint struct { + At time.Time + Values []bool +} + +// CheckTimed evaluates the legacy Duration as Duration * 3 seconds, avoiding +// both repeated stale samples and cadence-dependent early incidents. Unknown +// coverage cannot create an incident or a recovery. +func (r *AlertRule) CheckTimed(points []TimedAlertPoint, now time.Time) (known, passed bool) { + if r == nil || len(r.Rules) == 0 { + return false, false + } + allFailed := true + for index, rule := range r.Rules { + if rule == nil || !rule.IsSupportedType() { + return false, false + } + ruleKnown, rulePassed := rule.checkTimed(points, index, now) + if !ruleKnown { + return false, false + } + if rulePassed { + allFailed = false + } + } + return true, !allFailed +} + +func (rule *Rule) checkTimed(points []TimedAlertPoint, index int, now time.Time) (known, passed bool) { + if len(points) == 0 { + return false, false + } + if rule.IsTransferDurationRule() { + last := points[len(points)-1] + if index >= len(last.Values) || now.Sub(last.At) > AlertSampleMaxAge { + return false, false + } + return true, last.Values[index] + } + window := time.Duration(rule.Duration) * AlertSampleInterval + if window <= 0 { + return false, false + } + end := now.Add(AlertSampleInterval) // the current observation represents one tick + start := end.Add(-window) + if points[0].At.After(start) { + return false, false // the configured wall-clock window has not elapsed + } + var covered, failed time.Duration + for i, point := range points { + if index >= len(point.Values) { + return false, false + } + segmentStart := point.At + if start.After(segmentStart) { + segmentStart = start + } + segmentEnd := point.At.Add(AlertSampleMaxAge) + if end.Before(segmentEnd) { + segmentEnd = end + } + if i+1 < len(points) { + if points[i+1].At.Before(segmentEnd) { + segmentEnd = points[i+1].At + } + } else { + if tickEnd := point.At.Add(AlertSampleInterval); tickEnd.Before(segmentEnd) { + segmentEnd = tickEnd + } + } + if !segmentEnd.After(segmentStart) { + continue + } + span := segmentEnd.Sub(segmentStart) + covered += span + if !point.Values[index] { + failed += span + } + } + if covered < window-window/10 { + return false, false + } + if rule.IsOfflineRule() { + if covered-failed >= AlertSampleInterval { + return true, true + } + if failed >= window-AlertSampleInterval && covered >= window-AlertSampleInterval { + return true, false + } + return false, false + } + if float64(failed) > float64(window)*0.70 { + return true, false + } + if float64(failed+window-covered) <= float64(window)*0.70 { + return true, true + } + return false, false +} + +func (r *AlertRule) TimedRetention() time.Duration { + longest := AlertSampleMaxAge + for _, rule := range r.Rules { + if rule == nil || rule.IsTransferDurationRule() { + continue + } + if duration, ok := rule.DurationInt(); ok { + longest = max(longest, time.Duration(duration)*AlertSampleInterval+AlertSampleMaxAge) + } + } + return longest +} + const ( ModeAlwaysTrigger = 0 ModeOnetimeTrigger = 1 @@ -75,6 +197,9 @@ func (r *AlertRule) IsSafeToEvaluate() bool { if rule == nil || !rule.IsSupportedType() { return false } + if !rule.HasSafeThresholds() { + return false + } switch rule.Cover { case RuleCoverAll, RuleCoverIgnoreAll: default: @@ -152,18 +277,38 @@ func (r *AlertRule) HasPermission(ctx *gin.Context) bool { // Snapshot 对传入的Server进行该报警规则下所有type的检查 返回每项检查结果 func (r *AlertRule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) []bool { + point, _ := r.SnapshotStatus(cycleTransferStats, server, db) + return point +} + +// SnapshotStatus preserves the distinction between a healthy measurement and +// telemetry that is unavailable. A compound alert cannot be decided while any +// of its applicable conditions is unknown. +func (r *AlertRule) SnapshotStatus(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) ([]bool, bool) { + if server == nil { + return nil, false + } + return r.SnapshotStatusWithRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db) +} + +// SnapshotStatusWithRuntime uses one immutable report for every condition. +func (r *AlertRule) SnapshotStatusWithRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) ([]bool, bool) { point := make([]bool, len(r.Rules)) + known := true for i, rule := range r.Rules { if rule == nil || !rule.IsSupportedType() { // Invalid persisted rules are ignored instead of being interpreted as // a failed condition or allowed to panic the sentinel. point[i] = true + known = false continue } - point[i] = rule.Snapshot(cycleTransferStats, server, db) + var valid bool + point[i], valid = rule.EvaluateRuntime(cycleTransferStats, server, runtime, db) + known = known && valid } - return point + return point, known } // Check 传入包含当前报警规则下所有type检查结果 返回报警持续时间与是否通过报警检查(通过则返回true) diff --git a/model/notification.go b/model/notification.go index 635eae93..db151433 100644 --- a/model/notification.go +++ b/model/notification.go @@ -180,63 +180,65 @@ func (ns *NotificationServerBundle) replaceParamsInString(str string, message st if ns.Server != nil { runtime := ns.Server.RuntimeSnapshot() - if runtime.State == nil || runtime.Host == nil { - return str - } - state := runtime.State - host := runtime.Host replacements = append(replacements, "#SERVER.NAME#", mod(ns.Server.Name), "#SERVER.ID#", mod(fmt.Sprintf("%d", ns.Server.ID)), - - // Converted metrics - "#SERVER.CPU#", mod(ns.formatUsage(false, state.CPU)), - "#SERVER.MEM#", mod(ns.formatUsage(true, float64(state.MemUsed)/float64(host.MemTotal))), - "#SERVER.SWAP#", mod(ns.formatUsage(true, float64(state.SwapUsed)/float64(host.SwapTotal))), - "#SERVER.DISK#", mod(ns.formatUsage(true, float64(state.DiskUsed)/float64(host.DiskTotal))), - "#SERVER.SPEEDIN#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetInSpeed))), - "#SERVER.SPEEDOUT#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetOutSpeed))), - "#SERVER.TRANSFERIN#", mod(ns.formatSize(state.NetInTransfer)), - "#SERVER.TRANSFEROUT#", mod(ns.formatSize(state.NetOutTransfer)), - - // Raw metrics - "#SERVER.CPUUSED#", mod(fmt.Sprintf("%f", state.CPU)), - "#SERVER.MEMUSED#", mod(fmt.Sprintf("%d", state.MemUsed)), - "#SERVER.SWAPUSED#", mod(fmt.Sprintf("%d", state.SwapUsed)), - "#SERVER.DISKUSED#", mod(fmt.Sprintf("%d", state.DiskUsed)), - "#SERVER.MEMTOTAL#", mod(fmt.Sprintf("%d", host.MemTotal)), - "#SERVER.SWAPTOTAL#", mod(fmt.Sprintf("%d", host.SwapTotal)), - "#SERVER.DISKTOTAL#", mod(fmt.Sprintf("%d", host.DiskTotal)), - "#SERVER.NETINSPEED#", mod(fmt.Sprintf("%d", state.NetInSpeed)), - "#SERVER.NETOUTSPEED#", mod(fmt.Sprintf("%d", state.NetOutSpeed)), - "#SERVER.NETINTRANSFER#", mod(fmt.Sprintf("%d", state.NetInTransfer)), - "#SERVER.NETOUTTRANSFER#", mod(fmt.Sprintf("%d", state.NetOutTransfer)), - "#SERVER.LOAD1#", mod(fmt.Sprintf("%f", state.Load1)), - "#SERVER.LOAD5#", mod(fmt.Sprintf("%f", state.Load5)), - "#SERVER.LOAD15#", mod(fmt.Sprintf("%f", state.Load15)), - "#SERVER.TCPCONNCOUNT#", mod(fmt.Sprintf("%d", state.TcpConnCount)), - "#SERVER.UDPCONNCOUNT#", mod(fmt.Sprintf("%d", state.UdpConnCount)), ) + if runtime.State != nil && runtime.Host != nil { + state := runtime.State + host := runtime.Host + replacements = append(replacements, + // Converted metrics + "#SERVER.CPU#", mod(ns.formatUsage(false, state.CPU)), + "#SERVER.MEM#", mod(ns.formatUsage(true, float64(state.MemUsed)/float64(host.MemTotal))), + "#SERVER.SWAP#", mod(ns.formatUsage(true, float64(state.SwapUsed)/float64(host.SwapTotal))), + "#SERVER.DISK#", mod(ns.formatUsage(true, float64(state.DiskUsed)/float64(host.DiskTotal))), + "#SERVER.SPEEDIN#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetInSpeed))), + "#SERVER.SPEEDOUT#", mod(fmt.Sprintf("%s/s", ns.formatSize(state.NetOutSpeed))), + "#SERVER.TRANSFERIN#", mod(ns.formatSize(state.NetInTransfer)), + "#SERVER.TRANSFEROUT#", mod(ns.formatSize(state.NetOutTransfer)), - var ipv4, ipv6, validIP string - ip := ns.Server.GeoIP.IP - if ip.IPv4Addr != "" && ip.IPv6Addr != "" { - ipv4 = ip.IPv4Addr - ipv6 = ip.IPv6Addr - validIP = ipv4 - } else if ip.IPv4Addr != "" { - ipv4 = ip.IPv4Addr - validIP = ipv4 - } else { - ipv6 = ip.IPv6Addr - validIP = ipv6 + // Raw metrics + "#SERVER.CPUUSED#", mod(fmt.Sprintf("%f", state.CPU)), + "#SERVER.MEMUSED#", mod(fmt.Sprintf("%d", state.MemUsed)), + "#SERVER.SWAPUSED#", mod(fmt.Sprintf("%d", state.SwapUsed)), + "#SERVER.DISKUSED#", mod(fmt.Sprintf("%d", state.DiskUsed)), + "#SERVER.MEMTOTAL#", mod(fmt.Sprintf("%d", host.MemTotal)), + "#SERVER.SWAPTOTAL#", mod(fmt.Sprintf("%d", host.SwapTotal)), + "#SERVER.DISKTOTAL#", mod(fmt.Sprintf("%d", host.DiskTotal)), + "#SERVER.NETINSPEED#", mod(fmt.Sprintf("%d", state.NetInSpeed)), + "#SERVER.NETOUTSPEED#", mod(fmt.Sprintf("%d", state.NetOutSpeed)), + "#SERVER.NETINTRANSFER#", mod(fmt.Sprintf("%d", state.NetInTransfer)), + "#SERVER.NETOUTTRANSFER#", mod(fmt.Sprintf("%d", state.NetOutTransfer)), + "#SERVER.LOAD1#", mod(fmt.Sprintf("%f", state.Load1)), + "#SERVER.LOAD5#", mod(fmt.Sprintf("%f", state.Load5)), + "#SERVER.LOAD15#", mod(fmt.Sprintf("%f", state.Load15)), + "#SERVER.TCPCONNCOUNT#", mod(fmt.Sprintf("%d", state.TcpConnCount)), + "#SERVER.UDPCONNCOUNT#", mod(fmt.Sprintf("%d", state.UdpConnCount)), + ) } - replacements = append(replacements, - "#SERVER.IP#", mod(validIP), - "#SERVER.IPV4#", mod(ipv4), - "#SERVER.IPV6#", mod(ipv6), - ) + var ipv4, ipv6, validIP string + if ns.Server.GeoIP != nil { + ip := ns.Server.GeoIP.IP + if ip.IPv4Addr != "" && ip.IPv6Addr != "" { + ipv4 = ip.IPv4Addr + ipv6 = ip.IPv6Addr + validIP = ipv4 + } else if ip.IPv4Addr != "" { + ipv4 = ip.IPv4Addr + validIP = ipv4 + } else { + ipv6 = ip.IPv6Addr + validIP = ipv6 + } + + replacements = append(replacements, + "#SERVER.IP#", mod(validIP), + "#SERVER.IPV4#", mod(ipv4), + "#SERVER.IPV6#", mod(ipv6), + ) + } } replacer := strings.NewReplacer(replacements...) diff --git a/model/rule.go b/model/rule.go index 2d492b10..1155ce71 100644 --- a/model/rule.go +++ b/model/rule.go @@ -1,6 +1,8 @@ package model import ( + "log" + "math" "slices" "strings" "time" @@ -90,36 +92,80 @@ func (u *Rule) DurationInt() (duration int, ok bool) { // which is the documented legacy spelling for hours. func (u *Rule) HasSafeCycleConfiguration() bool { return u != nil && u.IsTransferDurationRule() && u.CycleStart != nil && - u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval + !u.CycleStart.After(time.Now()) && u.CycleInterval > 0 && u.CycleInterval <= MaxAlertRuleCycleInterval && + u.HasSafeThresholds() && u.HasSafeCycleUnit() } -// Snapshot 未通过规则返回 false, 通过返回 true -func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool { - if u == nil || server == nil || !u.IsSupportedType() { +func (u *Rule) HasSafeCycleUnit() bool { + if u == nil { + return false + } + switch strings.ToLower(u.CycleUnit) { + case "", "hour", "day", "week", "month", "year": return true + default: + return false + } +} + +func (u *Rule) HasSafeThresholds() bool { + if u == nil || math.IsNaN(u.Min) || math.IsNaN(u.Max) || math.IsInf(u.Min, 0) || math.IsInf(u.Max, 0) || u.Min < 0 || u.Max < 0 { + return false + } + return u.Min == 0 || u.Max == 0 || u.Min <= u.Max +} + +// Snapshot is retained for callers which only need a boolean result. The alert +// sentinel uses Evaluate so missing telemetry is not mistaken for a breach. +func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) bool { + passed, _ := u.Evaluate(cycleTransferStats, server, db) + return passed +} + +// Evaluate returns known=false when a metric cannot be evaluated from the +// current report. Unknown must neither trigger nor resolve an incident. +func (u *Rule) Evaluate(cycleTransferStats *CycleTransferStats, server *Server, db *gorm.DB) (passed, known bool) { + if server == nil { + return true, false + } + return u.EvaluateRuntime(cycleTransferStats, server, server.RuntimeSnapshot(), db) +} + +// EvaluateRuntime evaluates all conditions against the same immutable Agent +// report. Callers evaluating a compound rule must share this snapshot. +func (u *Rule) EvaluateRuntime(cycleTransferStats *CycleTransferStats, server *Server, runtime RuntimeSnapshot, db *gorm.DB) (passed, known bool) { + if u == nil || server == nil || !u.IsSupportedType() { + return true, false } if u.IsTransferDurationRule() && (!u.HasSafeCycleConfiguration() || cycleTransferStats == nil) { - return true + return true, false } // 监控全部但是排除了此服务器 if u.Cover == RuleCoverAll && u.Ignore[server.ID] { - return true + return true, true } // 忽略全部但是指定监控了此服务器 if u.Cover == RuleCoverIgnoreAll && !u.Ignore[server.ID] { - return true + return true, true } + evalNow := time.Now() + var cycleStart, cycleEnd time.Time + if u.IsTransferDurationRule() { + cycleStart, cycleEnd = u.transferDurationBounds(evalNow) + } // 循环区间流量检测 · 短期无需重复检测 - if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(time.Now()) { - return u.LastCycleStatus[server.ID] + if u.IsTransferDurationRule() && u.NextTransferAt[server.ID].After(evalNow) { + return u.LastCycleStatus[server.ID], true } var src float64 - runtime := server.RuntimeSnapshot() + if u.IsOfflineRule() { + return !runtime.LastActive.IsZero() && time.Since(runtime.LastActive) <= 6*time.Second, true + } if runtime.State == nil { - return false + return true, false } state := runtime.State @@ -128,22 +174,31 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, src = float64(state.CPU) case "gpu", "gpu_max": if len(state.GPU) == 0 { - return true + return true, false } src = slices.Max(state.GPU) case "memory": if runtime.Host == nil { - return false + return true, false + } + if runtime.Host.MemTotal == 0 { + return true, false } src = percentage(state.MemUsed, runtime.Host.MemTotal) case "swap": if runtime.Host == nil { - return false + return true, false + } + if runtime.Host.SwapTotal == 0 { + return true, false } src = percentage(state.SwapUsed, runtime.Host.SwapTotal) case "disk": if runtime.Host == nil { - return false + return true, false + } + if runtime.Host.DiskTotal == 0 { + return true, false } src = percentage(state.DiskUsed, runtime.Host.DiskTotal) case "net_in_speed": @@ -151,38 +206,50 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, case "net_out_speed": src = float64(state.NetOutSpeed) case "net_all_speed": - src = float64(state.NetOutSpeed + state.NetOutSpeed) + src = float64(state.NetInSpeed) + float64(state.NetOutSpeed) case "transfer_in": src = float64(state.NetInTransfer) case "transfer_out": src = float64(state.NetOutTransfer) case "transfer_all": src = float64(state.NetOutTransfer + state.NetInTransfer) - case "offline": - if runtime.LastActive.IsZero() { - src = 0 - } else { - src = float64(runtime.LastActive.Unix()) - } case "transfer_in_cycle": + if db == nil { + return true, false + } src = float64(utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot)) if u.CycleInterval != 0 { var res NResult - db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res) + if err := db.Model(&Transfer{}).Select("SUM(`in`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil { + log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err) + return true, false + } src += float64(res.N) } case "transfer_out_cycle": + if db == nil { + return true, false + } src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot)) if u.CycleInterval != 0 { var res NResult - db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res) + if err := db.Model(&Transfer{}).Select("SUM(`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil { + log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err) + return true, false + } src += float64(res.N) } case "transfer_all_cycle": + if db == nil { + return true, false + } src = float64(utils.SubUintChecked(state.NetOutTransfer, runtime.PrevTransferOutSnapshot) + utils.SubUintChecked(state.NetInTransfer, runtime.PrevTransferInSnapshot)) if u.CycleInterval != 0 { var res NResult - db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND server_id = ?", u.GetTransferDurationStart().UTC(), server.ID).Scan(&res) + if err := db.Model(&Transfer{}).Select("SUM(`in`+`out`) AS n").Where("datetime(`created_at`) >= datetime(?) AND datetime(`created_at`) < datetime(?) AND datetime(`created_at`) <= datetime(?) AND server_id = ?", cycleStart.UTC(), cycleEnd.UTC(), evalNow.UTC(), server.ID).Scan(&res).Error; err != nil { + log.Printf("NEZHA>> Alert cycle transfer query failed for rule %s server %d: %v", u.Type, server.ID, err) + return true, false + } src += float64(res.N) } case "load1": @@ -205,23 +272,32 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, } } if len(temp) == 0 { - return true + return true, false } src = slices.Max(temp) default: - return true + return true, false } // 循环区间流量检测 · 更新下次需要检测时间 if u.IsTransferDurationRule() { - seconds := max(1800*((u.Max-src)/u.Max), 180) + seconds := float64(180) + if u.Max > 0 { + seconds = max(1800*((u.Max-src)/u.Max), 180) + } else if u.Min > 0 { + seconds = 1800 + } if u.NextTransferAt == nil { u.NextTransferAt = make(map[uint64]time.Time) } if u.LastCycleStatus == nil { u.LastCycleStatus = make(map[uint64]bool) } - u.NextTransferAt[server.ID] = time.Now().Add(time.Second * time.Duration(seconds)) + nextAt := evalNow.Add(time.Second * time.Duration(seconds)) + if nextAt.After(cycleEnd) { + nextAt = cycleEnd + } + u.NextTransferAt[server.ID] = nextAt if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { u.LastCycleStatus[server.ID] = false } else { @@ -233,17 +309,15 @@ func (u *Rule) Snapshot(cycleTransferStats *CycleTransferStats, server *Server, cycleTransferStats.Transfer[server.ID] = uint64(src) cycleTransferStats.NextUpdate[server.ID] = u.NextTransferAt[server.ID] // 自动更新周期流量展示起止时间 - cycleTransferStats.From = u.GetTransferDurationStart() - cycleTransferStats.To = u.GetTransferDurationEnd() + cycleTransferStats.From = cycleStart + cycleTransferStats.To = cycleEnd } - if u.Type == "offline" && float64(time.Now().Unix())-src > 6 { - return false - } else if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { - return false + if (u.Max > 0 && src > u.Max) || (u.Min > 0 && src < u.Min) { + return false, true } - return true + return true, true } // IsTransferDurationRule 判断该规则是否属于周期流量规则 属于则返回true @@ -263,83 +337,49 @@ func (u *Rule) IsOfflineRule() bool { return u != nil && u.Type == "offline" } -// GetTransferDurationStart 获取周期流量的起始时间 +// transferDurationBounds computes both edges from the same clock reading. +// Separate time.Now calls can otherwise straddle a cycle boundary. +func (u *Rule) transferDurationBounds(now time.Time) (time.Time, time.Time) { + if !u.HasSafeCycleConfiguration() { + return time.Time{}, time.Time{} + } + start := *u.CycleStart + var end time.Time + switch strings.ToLower(u.CycleUnit) { + case "year": + end = start.AddDate(int(u.CycleInterval), 0, 0) + for !now.Before(end) { + start, end = end, end.AddDate(int(u.CycleInterval), 0, 0) + } + case "month": + end = start.AddDate(0, int(u.CycleInterval), 0) + for !now.Before(end) { + start, end = end, end.AddDate(0, int(u.CycleInterval), 0) + } + case "week": + end = start.AddDate(0, 0, 7*int(u.CycleInterval)) + for !now.Before(end) { + start, end = end, end.AddDate(0, 0, 7*int(u.CycleInterval)) + } + case "day": + end = start.AddDate(0, 0, int(u.CycleInterval)) + for !now.Before(end) { + start, end = end, end.AddDate(0, 0, int(u.CycleInterval)) + } + default: // empty is the legacy spelling for hour + interval := 3600 * int64(u.CycleInterval) + start = time.Unix(u.CycleStart.Unix()+(now.Unix()-u.CycleStart.Unix())/interval*interval, 0) + end = time.Unix(start.Unix()+interval, 0) + } + return start, end +} + func (u *Rule) GetTransferDurationStart() time.Time { - // Accept uppercase and lowercase - unit := strings.ToLower(u.CycleUnit) - startTime := *u.CycleStart - var nextTime time.Time - switch unit { - case "year": - nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0) - } - case "month": - nextTime = startTime.AddDate(0, int(u.CycleInterval), 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0) - } - case "week": - nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval)) - } - case "day": - nextTime = startTime.AddDate(0, 0, int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval)) - } - default: - // For hour unit or not set. - interval := 3600 * int64(u.CycleInterval) - startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0) - } - - return startTime + start, _ := u.transferDurationBounds(time.Now()) + return start } -// GetTransferDurationEnd 获取周期流量结束时间 func (u *Rule) GetTransferDurationEnd() time.Time { - // Accept uppercase and lowercase - unit := strings.ToLower(u.CycleUnit) - startTime := *u.CycleStart - var nextTime time.Time - switch unit { - case "year": - nextTime = startTime.AddDate(int(u.CycleInterval), 0, 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(int(u.CycleInterval), 0, 0) - } - case "month": - nextTime = startTime.AddDate(0, int(u.CycleInterval), 0) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, int(u.CycleInterval), 0) - } - case "week": - nextTime = startTime.AddDate(0, 0, 7*int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, 7*int(u.CycleInterval)) - } - case "day": - nextTime = startTime.AddDate(0, 0, int(u.CycleInterval)) - for time.Now().After(nextTime) { - startTime = nextTime - nextTime = nextTime.AddDate(0, 0, int(u.CycleInterval)) - } - default: - // For hour unit or not set. - interval := 3600 * int64(u.CycleInterval) - startTime = time.Unix(u.CycleStart.Unix()+(time.Now().Unix()-u.CycleStart.Unix())/interval*interval, 0) - nextTime = time.Unix(startTime.Unix()+interval, 0) - } - - return nextTime + _, end := u.transferDurationBounds(time.Now()) + return end } diff --git a/model/server.go b/model/server.go index fd566199..ada61db9 100644 --- a/model/server.go +++ b/model/server.go @@ -16,6 +16,7 @@ import ( ) var runtimeHolderInitMu sync.Mutex +var nextServerReportSequence atomic.Uint64 type Server struct { Common @@ -73,15 +74,16 @@ type taskStreamHolder struct { } type serverRuntimeHolder struct { - mu sync.Mutex - canonical *Server - stream pb.NezhaService_ReportSystemStateServer - generation uint64 - state *HostState - host *Host - lastActive time.Time - prevIn uint64 - prevOut uint64 + mu sync.Mutex + canonical *Server + stream pb.NezhaService_ReportSystemStateServer + generation uint64 + state *HostState + host *Host + lastActive time.Time + reportSequence uint64 + prevIn uint64 + prevOut uint64 } type StateStreamLease struct { @@ -204,6 +206,7 @@ func (lease StateStreamLease) updateState(receiver *Server, state *HostState, la return false } } + lease.holder.reportSequence = nextServerReportSequence.Add(1) return true } @@ -334,6 +337,7 @@ type RuntimeSnapshot struct { State *HostState Host *Host LastActive time.Time + ReportSequence uint64 PrevTransferInSnapshot uint64 PrevTransferOutSnapshot uint64 } @@ -360,7 +364,7 @@ func (s *Server) RuntimeSnapshot() RuntimeSnapshot { holder.host = cloneHost(s.Host) } } - return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut} + return RuntimeSnapshot{State: cloneHostState(holder.state), Host: cloneHost(holder.host), LastActive: holder.lastActive, ReportSequence: holder.reportSequence, PrevTransferInSnapshot: holder.prevIn, PrevTransferOutSnapshot: holder.prevOut} } func (s *Server) SetTransferSnapshots(inbound, outbound uint64) bool { diff --git a/service/singleton/alertsentinel.go b/service/singleton/alertsentinel.go index 82f56846..838f936c 100644 --- a/service/singleton/alertsentinel.go +++ b/service/singleton/alertsentinel.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "sync" + "sync/atomic" "time" "github.com/jinzhu/copier" @@ -26,11 +27,122 @@ type NotificationHistory struct { var ( AlertsLock sync.RWMutex Alerts []*model.AlertRule - alertsStore map[uint64]map[uint64][][]bool // [alert_id][server_id] -> [timeTick][ruleId] 时间点对应的rule的检查结果 - alertsPrevState map[uint64]map[uint64]uint8 // [alert_id][server_id] -> 对应报警规则的上一次报警状态 - AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats // [alert_id] -> 对应报警规则的周期流量统计 + alertsStore map[uint64]map[uint64][]model.TimedAlertPoint // distinct reports per alert/server + alertsPrevState map[uint64]map[uint64]uint8 // [alert_id][server_id] -> 对应报警规则的上一次报警状态 + alertsLastMetricSeq map[uint64]map[uint64]uint64 // last distinct Agent metric report per alert/server + AlertsCycleTransferStatsStore map[uint64]*model.CycleTransferStats // [alert_id] -> 对应报警规则的周期流量统计 + alertDeliveryMu sync.Mutex + alertDeliveries map[uint64]map[uint64]*alertDeliveryEntry + alertEventSequence atomic.Uint64 ) +type alertDeliveryEntry struct { + cancel chan struct{} + done chan struct{} + sendMu *sync.Mutex +} + +type alertDeliveryPhase uint8 + +const ( + alertDeliveryIncident alertDeliveryPhase = iota + alertDeliveryRecovery +) + +func cancelAlertDeliveries(alertID uint64) { + alertDeliveryMu.Lock() + defer alertDeliveryMu.Unlock() + for _, entry := range alertDeliveries[alertID] { + close(entry.cancel) + } + delete(alertDeliveries, alertID) +} + +func cancelAlertDeliveriesForServers(serverIDs []uint64) { + alertDeliveryMu.Lock() + defer alertDeliveryMu.Unlock() + for alertID, byServer := range alertDeliveries { + for _, serverID := range serverIDs { + if entry := byServer[serverID]; entry != nil { + close(entry.cancel) + delete(byServer, serverID) + } + } + if len(byServer) == 0 { + delete(alertDeliveries, alertID) + } + } +} + +func runAlertDelivery(cancel <-chan struct{}, send func() bool, always bool, retryDelay time.Duration) { + for { + select { + case <-cancel: + return + default: + } + delivered := send() + if delivered && !always { + return + } + delay := retryDelay + if delivered { + delay = firstNotificationDelay + } + timer := time.NewTimer(delay) + select { + case <-cancel: + timer.Stop() + return + case <-timer.C: + } + } +} + +func startAlertDelivery(alert *model.AlertRule, server *model.Server, message, muteLabel string, phase alertDeliveryPhase) { + // A mute entry belongs to one transition, not all later incidents of the + // same phase. Concurrent in-flight sends may finish after a recovery. + muteLabel = fmt.Sprintf("%s:event-%d", muteLabel, alertEventSequence.Add(1)) + alertDeliveryMu.Lock() + if alertDeliveries == nil { + alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry) + } + if alertDeliveries[alert.ID] == nil { + alertDeliveries[alert.ID] = make(map[uint64]*alertDeliveryEntry) + } + sendMu := new(sync.Mutex) + if old := alertDeliveries[alert.ID][server.ID]; old != nil { + close(old.cancel) + sendMu = old.sendMu + } + entry := &alertDeliveryEntry{cancel: make(chan struct{}), done: make(chan struct{}), sendMu: sendMu} + alertDeliveries[alert.ID][server.ID] = entry + alertDeliveryMu.Unlock() + + groupID := alert.NotificationGroupID + always := phase == alertDeliveryIncident && alert.TriggerMode == model.ModeAlwaysTrigger + go func() { + defer close(entry.done) + runAlertDelivery(entry.cancel, func() bool { + // Serialize incident/recovery delivery for this alert/server pair. + // A recovery can cancel a pending incident, but never overtake an + // already-running network request. + entry.sendMu.Lock() + defer entry.sendMu.Unlock() + select { + case <-entry.cancel: + return true + default: + } + return NotificationShared.SendNotification(groupID, message, muteLabel, server) + }, always, 30*time.Second) + }() +} + +func shouldRunAlertFailTasks(triggerMode uint8, newIncident bool) bool { + return newIncident || triggerMode == model.ModeAlwaysTrigger +} + // addCycleTransferStatsInfo 向AlertsCycleTransferStatsStore中添加周期流量报警统计信息 func addCycleTransferStatsInfo(alert *model.AlertRule) { if alert == nil || !alert.Enabled() || !alert.IsSafeToEvaluate() { @@ -59,8 +171,17 @@ func addCycleTransferStatsInfo(alert *model.AlertRule) { // AlertSentinelStart 报警器启动 func AlertSentinelStart() { - alertsStore = make(map[uint64]map[uint64][][]bool) + alertDeliveryMu.Lock() + for _, byServer := range alertDeliveries { + for _, entry := range byServer { + close(entry.cancel) + } + } + alertDeliveries = make(map[uint64]map[uint64]*alertDeliveryEntry) + alertDeliveryMu.Unlock() + alertsStore = make(map[uint64]map[uint64][]model.TimedAlertPoint) alertsPrevState = make(map[uint64]map[uint64]uint8) + alertsLastMetricSeq = make(map[uint64]map[uint64]uint64) AlertsCycleTransferStatsStore = make(map[uint64]*model.CycleTransferStats) AlertsLock.Lock() if err := DB.Find(&Alerts).Error; err != nil { @@ -71,8 +192,9 @@ func AlertSentinelStart() { log.Printf("NEZHA>> Skipping invalid nil alert rule loaded from database") continue } - alertsStore[alert.ID] = make(map[uint64][][]bool) + alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint) alertsPrevState[alert.ID] = make(map[uint64]uint8) + alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64) if !alert.IsSafeToEvaluate() { log.Printf("NEZHA>> Skipping invalid alert rule %d loaded from database", alert.ID) continue @@ -101,8 +223,10 @@ func AlertSentinelStart() { func OnRefreshOrAddAlert(alert *model.AlertRule) { AlertsLock.Lock() defer AlertsLock.Unlock() + cancelAlertDeliveries(alert.ID) delete(alertsStore, alert.ID) delete(alertsPrevState, alert.ID) + delete(alertsLastMetricSeq, alert.ID) var isEdit bool for i := range Alerts { if Alerts[i].ID == alert.ID { @@ -113,8 +237,9 @@ func OnRefreshOrAddAlert(alert *model.AlertRule) { if !isEdit { Alerts = append(Alerts, alert) } - alertsStore[alert.ID] = make(map[uint64][][]bool) + alertsStore[alert.ID] = make(map[uint64][]model.TimedAlertPoint) alertsPrevState[alert.ID] = make(map[uint64]uint8) + alertsLastMetricSeq[alert.ID] = make(map[uint64]uint64) delete(AlertsCycleTransferStatsStore, alert.ID) addCycleTransferStatsInfo(alert) } @@ -123,8 +248,10 @@ func OnDeleteAlert(id []uint64) { AlertsLock.Lock() defer AlertsLock.Unlock() for _, i := range id { + cancelAlertDeliveries(i) delete(alertsStore, i) delete(alertsPrevState, i) + delete(alertsLastMetricSeq, i) currentAlerts := Alerts[:0] for _, alert := range Alerts { if alert.ID != i { @@ -176,25 +303,67 @@ func checkStatusForServer(alert *model.AlertRule, server *model.Server) { } }() - alertsStore[alert.ID][server.ID] = append(alertsStore[alert. - ID][server.ID], alert.Snapshot(AlertsCycleTransferStatsStore[alert.ID], server, DB)) + // Offline-only rules deliberately observe elapsed time on every tick. Metric + // rules, including compound rules, must never count the same Agent report + // repeatedly or treat a disconnected Agent's retained State as fresh data. + metricRule := false + runtime := server.RuntimeSnapshot() + for _, rule := range alert.Rules { + if !rule.IsOfflineRule() { + metricRule = true + break + } + } + if metricRule { + if runtime.LastActive.IsZero() || runtime.State == nil || time.Since(runtime.LastActive) > model.AlertSampleMaxAge || time.Since(runtime.LastActive) < 0 { + alertsStore[alert.ID][server.ID] = nil + return + } + if runtime.ReportSequence == 0 || runtime.ReportSequence == alertsLastMetricSeq[alert.ID][server.ID] { + return + } + alertsLastMetricSeq[alert.ID][server.ID] = runtime.ReportSequence + } + point, known := alert.SnapshotStatusWithRuntime(AlertsCycleTransferStatsStore[alert.ID], server, runtime, DB) + if !known { + alertsStore[alert.ID][server.ID] = nil + return + } + sampledAt := time.Now() + if metricRule { + sampledAt = runtime.LastActive + } + alertsStore[alert.ID][server.ID] = append(alertsStore[alert.ID][server.ID], model.TimedAlertPoint{At: sampledAt, Values: point}) + // Bound memory even while coverage is insufficient to decide a state. + cutoff := time.Now().Add(-alert.TimedRetention()) + samples := alertsStore[alert.ID][server.ID] + first := 0 + for first < len(samples) && samples[first].At.Before(cutoff) { + first++ + } + if first > 0 { + alertsStore[alert.ID][server.ID] = append([]model.TimedAlertPoint(nil), samples[first:]...) + } // 发送通知，分为触发报警和恢复通知 - _, passed := alert.Check(alertsStore[alert.ID][server.ID]) + known, passed := alert.CheckTimed(alertsStore[alert.ID][server.ID], time.Now()) + if !known { + return + } // 保存当前服务器状态信息 curServer := model.Server{} copier.Copy(&curServer, server) // 本次未通过检查 if !passed { - // 始终触发模式或上次检查不为失败时触发报警（跳过单次触发+上次失败的情况） - if alert.TriggerMode == model.ModeAlwaysTrigger || alertsPrevState[alert.ID][server.ID] != _RuleCheckFail { + newIncident := alertsPrevState[alert.ID][server.ID] != _RuleCheckFail + if newIncident { alertsPrevState[alert.ID][server.ID] = _RuleCheckFail message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Incident"), server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) + startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), alertDeliveryIncident) + } + if shouldRunAlertFailTasks(alert.TriggerMode, newIncident) { go CronShared.SendTriggerTasks(alert.FailTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncident(server.ID, alert.ID), &curServer) - // 清除恢复通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID)) } } else { // 本次通过检查但上一次的状态为失败，则发送恢复通知 @@ -202,20 +371,8 @@ func checkStatusForServer(alert *model.AlertRule, server *model.Server) { message := fmt.Sprintf("[%s] %s(%s) %s", Localizer.T("Resolved"), server.Name, IPDesensitize(server.GeoIP.IP.Join()), alert.Name) go CronShared.SendTriggerTasks(alert.RecoverTriggerTasks, curServer.ID, alert.UserID) - go NotificationShared.SendNotification(alert.NotificationGroupID, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), &curServer) - // 清除失败通知的静音缓存 - NotificationShared.UnMuteNotification(alert.NotificationGroupID, NotificationMuteLabel.ServerIncident(server.ID, alert.ID)) + startAlertDelivery(alert, &curServer, message, NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery) } alertsPrevState[alert.ID][server.ID] = _RuleCheckPass } - // 清理旧数据：保留窗口由规则定义决定（各规则 Duration 的最大值）， - // 而非 Check 的判定结果。window==0 表示没有任何有效规则需要回看历史 - // （例如全部 Duration<=0），此时清空采样避免切片无限增长。 - window := alert.RetentionWindow() - samples := alertsStore[alert.ID][server.ID] - if window <= 0 { - alertsStore[alert.ID][server.ID] = samples[:0] - } else if window < len(samples) { - alertsStore[alert.ID][server.ID] = samples[len(samples)-window:] - } } diff --git a/service/singleton/alertsentinel_freshness_test.go b/service/singleton/alertsentinel_freshness_test.go new file mode 100644 index 00000000..b68aeae5 --- /dev/null +++ b/service/singleton/alertsentinel_freshness_test.go @@ -0,0 +1,281 @@ +package singleton + +import ( + "context" + "testing" + "time" + + "github.com/patrickmn/go-cache" + "google.golang.org/grpc/metadata" + + "github.com/nezhahq/nezha/model" + pb "github.com/nezhahq/nezha/proto" +) + +type alertTestStateStream struct{} + +func (alertTestStateStream) Send(*pb.Receipt) error { return nil } +func (alertTestStateStream) Recv() (*pb.State, error) { return nil, context.Canceled } +func (alertTestStateStream) SetHeader(metadata.MD) error { return nil } +func (alertTestStateStream) SendHeader(metadata.MD) error { return nil } +func (alertTestStateStream) SetTrailer(metadata.MD) {} +func (alertTestStateStream) Context() context.Context { return context.Background() } +func (alertTestStateStream) SendMsg(any) error { return nil } +func (alertTestStateStream) RecvMsg(any) error { return nil } + +func TestAlertSentinelCountsOnlyFreshDistinctMetricReports(t *testing.T) { + oldStore, oldPrev, oldLast, oldCycle := alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore + alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{7: {1: nil}} + alertsPrevState = map[uint64]map[uint64]uint8{7: {}} + alertsLastMetricSeq = map[uint64]map[uint64]uint64{7: {}} + AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} + t.Cleanup(func() { + alertsStore, alertsPrevState, alertsLastMetricSeq, AlertsCycleTransferStatsStore = oldStore, oldPrev, oldLast, oldCycle + }) + + alert := &model.AlertRule{ + Common: model.Common{ID: 7}, + Rules: []*model.Rule{{Type: "cpu", Max: 80, Duration: 100, Cover: model.RuleCoverAll}}, + } + server := &model.Server{Common: model.Common{ID: 1}} + model.InitServer(server) + lease := server.AttachStateStream(alertTestStateStream{}) + fixedReportTime := time.Now() + if !lease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) { + t.Fatal("state update rejected") + } + checkStatusForServer(alert, server) + if got := len(alertsStore[7][1]); got != 1 { + t.Fatalf("first report produced %d samples, want 1", got) + } + for i := 0; i < 100; i++ { + checkStatusForServer(alert, server) + } + if got := len(alertsStore[7][1]); got != 1 { + t.Fatalf("same report was counted %d times, want 1", got) + } + if !lease.Clear() { + t.Fatal("stream clear rejected") + } + checkStatusForServer(alert, server) + if got := len(alertsStore[7][1]); got != 0 { + t.Fatalf("disconnect retained %d high samples, want 0", got) + } + alertsPrevState[7][1] = _RuleCheckFail // incident was active before disconnect + newLease := server.AttachStateStream(alertTestStateStream{}) + // Distinct reports may share a clock timestamp on coarse-resolution hosts. + if !newLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) { + t.Fatal("reconnect state update rejected") + } + checkStatusForServer(alert, server) + if got := len(alertsStore[7][1]); got != 1 || alertsStore[7][1][0].Values[0] { + t.Fatalf("reconnect window = %v, want one high sample", alertsStore[7][1]) + } + if alertsPrevState[7][1] != _RuleCheckFail { + t.Fatal("incomplete warm-up falsely resolved active incident") + } + replacement := &model.Server{Common: model.Common{ID: 1}} + model.InitServer(replacement) + replacementLease := replacement.AttachStateStream(alertTestStateStream{}) + if !replacementLease.UpdateState(&model.HostState{CPU: 99}, fixedReportTime) { + t.Fatal("replacement server state update rejected") + } + checkStatusForServer(alert, replacement) + if got := len(alertsStore[7][1]); got != 2 { + t.Fatalf("replacement holder reused a report identity; samples=%d", got) + } +} + +func TestAlertNewIncidentNotMutedByEarlierInflightIncident(t *testing.T) { + oldCache, oldNotifications := Cache, NotificationShared + Cache = cache.New(time.Minute, time.Minute) + n := &model.Notification{Common: model.Common{ID: 601}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 603}, Name: "alert-test"}, []uint64{n.ID}) + NotificationShared = nc + t.Cleanup(func() { + cancelAlertDeliveries(602) + NotificationShared, Cache = oldNotifications, oldCache + }) + started := make(chan struct{}) + release := make(chan struct{}) + delivered := make(chan string, 3) + nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error { + if desc == "first incident" { + close(started) + <-release + } + delivered <- desc + return nil + } + alert := &model.AlertRule{ + Common: model.Common{ID: 602}, + NotificationGroupID: 603, + TriggerMode: model.ModeOnetimeTrigger, + } + server := &model.Server{Common: model.Common{ID: 604}} + label := NotificationMuteLabel.ServerIncident(server.ID, alert.ID) + startAlertDelivery(alert, server, "first incident", label, alertDeliveryIncident) + <-started + startAlertDelivery(alert, server, "recovered", NotificationMuteLabel.ServerIncidentResolved(server.ID, alert.ID), alertDeliveryRecovery) + startAlertDelivery(alert, server, "second incident", label, alertDeliveryIncident) + close(release) + timer := time.NewTimer(time.Second) + defer timer.Stop() + seen := map[string]bool{} + for len(seen) < 2 { + select { + case message := <-delivered: + seen[message] = true + case <-timer.C: + t.Fatalf("new incident was suppressed after old send completed: %v", seen) + } + } + if !seen["first incident"] || !seen["second incident"] || seen["recovered"] { + t.Fatalf("unexpected transition deliveries: %v", seen) + } + alertDeliveryMu.Lock() + done := alertDeliveries[alert.ID][server.ID].done + alertDeliveryMu.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("new incident delivery did not finish") + } +} + +func TestAlertRecoveryDeliveryStopsAfterSuccessInAlwaysMode(t *testing.T) { + oldCache, oldNotifications := Cache, NotificationShared + Cache = cache.New(time.Minute, time.Minute) + n := &model.Notification{Common: model.Common{ID: 611}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 613}, Name: "recovery-test"}, []uint64{n.ID}) + NotificationShared = nc + t.Cleanup(func() { + cancelAlertDeliveries(612) + NotificationShared, Cache = oldNotifications, oldCache + }) + + delivered := make(chan struct{}, 2) + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + delivered <- struct{}{} + return nil + } + alert := &model.AlertRule{ + Common: model.Common{ID: 612}, + NotificationGroupID: 613, + TriggerMode: model.ModeAlwaysTrigger, + } + server := &model.Server{Common: model.Common{ID: 614}} + startAlertDelivery(alert, server, "recovered", "recovery-event", alertDeliveryRecovery) + + select { + case <-delivered: + case <-time.After(time.Second): + t.Fatal("recovery notification was not delivered") + } + alertDeliveryMu.Lock() + done := alertDeliveries[alert.ID][server.ID].done + alertDeliveryMu.Unlock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("successful recovery delivery kept repeating in always mode") + } + select { + case <-delivered: + t.Fatal("recovery notification was delivered more than once") + default: + } +} + +func TestServerDeleteCancelsAlertDelivery(t *testing.T) { + alertDeliveryMu.Lock() + oldDeliveries := alertDeliveries + cancel := make(chan struct{}) + alertDeliveries = map[uint64]map[uint64]*alertDeliveryEntry{ + 621: {622: {cancel: cancel, done: make(chan struct{})}}, + } + alertDeliveryMu.Unlock() + t.Cleanup(func() { + alertDeliveryMu.Lock() + alertDeliveries = oldDeliveries + alertDeliveryMu.Unlock() + }) + + servers := &ServerClass{ + class: class[uint64, *model.Server]{ + list: map[uint64]*model.Server{ + 622: {Common: model.Common{ID: 622}, UUID: "delete-alert-worker"}, + }, + }, + uuidToID: map[string]uint64{"delete-alert-worker": 622}, + } + servers.Delete([]uint64{622}) + + select { + case <-cancel: + default: + t.Fatal("server deletion did not cancel its alert delivery") + } + alertDeliveryMu.Lock() + _, retained := alertDeliveries[621] + alertDeliveryMu.Unlock() + if retained { + t.Fatal("server deletion retained an empty alert delivery map") + } +} + +func TestAlwaysAlertContinuesFailureTriggerTasks(t *testing.T) { + if !shouldRunAlertFailTasks(model.ModeAlwaysTrigger, false) { + t.Fatal("always mode stopped failure trigger tasks after the initial incident") + } + if shouldRunAlertFailTasks(model.ModeOnetimeTrigger, false) { + t.Fatal("one-time mode repeated failure trigger tasks") + } + if !shouldRunAlertFailTasks(model.ModeOnetimeTrigger, true) { + t.Fatal("one-time mode skipped failure trigger tasks for a new incident") + } +} + +func TestAlertDeliveryRetriesFailureAndStopsAfterSuccess(t *testing.T) { + cancel := make(chan struct{}) + done := make(chan struct{}) + attempts := 0 + go func() { + runAlertDelivery(cancel, func() bool { + attempts++ + return attempts == 2 + }, false, time.Millisecond) + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + close(cancel) + t.Fatal("delivery did not retry and finish") + } + if attempts != 2 { + t.Fatalf("got %d attempts, want 2", attempts) + } +} + +func TestAlertDeliveryCancelsAfterRecovery(t *testing.T) { + cancel := make(chan struct{}) + done := make(chan struct{}) + started := make(chan struct{}) + go func() { + runAlertDelivery(cancel, func() bool { + close(started) + return true + }, true, time.Millisecond) + close(done) + }() + <-started + close(cancel) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("recovered incident delivery was not cancelled") + } +} diff --git a/service/singleton/alertsentinel_test.go b/service/singleton/alertsentinel_test.go index e227e57b..a78bf334 100644 --- a/service/singleton/alertsentinel_test.go +++ b/service/singleton/alertsentinel_test.go @@ -122,7 +122,7 @@ func TestCheckStatusForServerRecoversUnexpectedEvaluatorPanic(t *testing.T) { originalStore := alertsStore originalPrevState := alertsPrevState originalCycleStore := AlertsCycleTransferStatsStore - alertsStore = map[uint64]map[uint64][][]bool{} + alertsStore = map[uint64]map[uint64][]model.TimedAlertPoint{} alertsPrevState = map[uint64]map[uint64]uint8{} AlertsCycleTransferStatsStore = map[uint64]*model.CycleTransferStats{} t.Cleanup(func() { diff --git a/service/singleton/notification.go b/service/singleton/notification.go index 31bf3fe2..a5a53bfa 100644 --- a/service/singleton/notification.go +++ b/service/singleton/notification.go @@ -22,8 +22,228 @@ type NotificationClass struct { groupToIDList map[uint64]map[uint64]*model.Notification idToGroupList map[uint64]map[uint64]struct{} - groupList map[uint64]string - groupMu sync.RWMutex + groupList map[uint64]string + groupMu sync.RWMutex + deliveryMu sync.Mutex + deliveryKeys map[string]*notificationKeyLock + asyncInFlight map[string]struct{} + asyncRetryAfter map[string]time.Time + asyncRetryDelay map[string]time.Duration + muteGeneration map[string]uint64 + serviceDelivery map[uint64]*orderedServiceDelivery + sendForTest func(*model.Notification, string, *model.Server) error +} + +type serviceDeliveryRequest struct { + groupID uint64 + desc string + label string +} + +type orderedServiceDelivery struct { + current serviceDeliveryRequest + pending *serviceDeliveryRequest + update chan struct{} + cancel chan struct{} + done chan struct{} +} + +// SendServiceState preserves incident→recovery order without allowing a +// slow webhook to accumulate one goroutine per probe result. Only the latest +// pending phase is retained; a failed current phase retries with backoff. +func (c *NotificationClass) SendServiceState(serviceID, groupID uint64, desc, label string) { + req := serviceDeliveryRequest{groupID: groupID, desc: desc, label: label} + c.deliveryMu.Lock() + if c.serviceDelivery == nil { + c.serviceDelivery = make(map[uint64]*orderedServiceDelivery) + } + state := c.serviceDelivery[serviceID] + if state == nil { + state = &orderedServiceDelivery{current: req, update: make(chan struct{}, 1), cancel: make(chan struct{}), done: make(chan struct{})} + c.serviceDelivery[serviceID] = state + c.deliveryMu.Unlock() + go c.runServiceDelivery(serviceID, state) + return + } + if state.pending == nil && state.current.label == label { + c.deliveryMu.Unlock() + return + } + if state.pending == nil || state.pending.label != label { + state.pending = &req + select { + case state.update <- struct{}{}: + default: + } + } + c.deliveryMu.Unlock() +} + +func (c *NotificationClass) runServiceDelivery(serviceID uint64, state *orderedServiceDelivery) { + defer close(state.done) + delay := 30 * time.Second + for { + select { + case <-state.cancel: + return + default: + } + delivered := c.SendNotification(state.current.groupID, state.current.desc, state.current.label) + c.deliveryMu.Lock() + // Do not advance to a later phase until every recipient has received the + // current one. Otherwise a failed incident can be discarded as soon as a + // recovery is queued, leaving that recipient with a recovery but no incident. + if delivered && state.pending != nil { + next := state.pending + state.current = *next + state.pending = nil + delay = 30 * time.Second + select { + case <-state.update: + default: + } + c.deliveryMu.Unlock() + continue + } + if delivered { + if c.serviceDelivery[serviceID] == state { + delete(c.serviceDelivery, serviceID) + } + c.deliveryMu.Unlock() + return + } + c.deliveryMu.Unlock() + timer := time.NewTimer(delay) + select { + case <-state.cancel: + timer.Stop() + return + case <-state.update: + timer.Stop() + case <-timer.C: + delay = min(delay*2, 15*time.Minute) + } + } +} + +func (c *NotificationClass) CancelServiceState(serviceID uint64) { + if c == nil { + return + } + c.deliveryMu.Lock() + if state := c.serviceDelivery[serviceID]; state != nil { + close(state.cancel) + delete(c.serviceDelivery, serviceID) + } + c.deliveryMu.Unlock() +} + +// SendNotificationAsync coalesces concurrent service-probe reports for the +// same event before allocating a webhook goroutine. Alert transitions use the +// synchronous SendNotification path so they retain their retry semantics. +func (c *NotificationClass) SendNotificationAsync(groupID uint64, desc, muteLabel string, ext ...*model.Server) { + c.sendNotificationAsync(groupID, desc, muteLabel, nil, ext...) +} + +func (c *NotificationClass) sendNotificationAsync(groupID uint64, desc, muteLabel string, onSuccess func(), ext ...*model.Server) { + key := fmt.Sprintf("%d:%s", groupID, muteLabel) + c.deliveryMu.Lock() + if c.asyncInFlight == nil { + c.asyncInFlight = make(map[string]struct{}) + } + if _, busy := c.asyncInFlight[key]; busy { + c.deliveryMu.Unlock() + return + } + if time.Now().Before(c.asyncRetryAfter[key]) { + c.deliveryMu.Unlock() + return + } + c.asyncInFlight[key] = struct{}{} + c.deliveryMu.Unlock() + go func() { + delivered := c.SendNotification(groupID, desc, muteLabel, ext...) + if delivered && onSuccess != nil { + onSuccess() + } + c.deliveryMu.Lock() + if delivered { + delete(c.asyncRetryAfter, key) + delete(c.asyncRetryDelay, key) + } else { + if c.asyncRetryAfter == nil { + c.asyncRetryAfter = make(map[string]time.Time) + c.asyncRetryDelay = make(map[string]time.Duration) + } + delay := c.asyncRetryDelay[key] + if delay == 0 { + delay = 30 * time.Second + } else { + delay = min(delay*2, 15*time.Minute) + } + c.asyncRetryDelay[key] = delay + c.asyncRetryAfter[key] = time.Now().Add(delay) + c.pruneAsyncBackoffLocked() + } + delete(c.asyncInFlight, key) + c.deliveryMu.Unlock() + }() +} + +const maxAsyncBackoffKeys = 4096 + +// Bound memory when a failing webhook sees many short-lived event labels. +// Evicted old event keys are no longer active after their phase changes. +func (c *NotificationClass) pruneAsyncBackoffLocked() { + if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys { + return + } + now := time.Now() + for key, until := range c.asyncRetryAfter { + if !now.Before(until) { + delete(c.asyncRetryAfter, key) + delete(c.asyncRetryDelay, key) + } + } + for key := range c.asyncRetryAfter { + if len(c.asyncRetryAfter) <= maxAsyncBackoffKeys { + break + } + delete(c.asyncRetryAfter, key) + delete(c.asyncRetryDelay, key) + } +} + +type notificationKeyLock struct { + mu sync.Mutex + refs int +} + +func (c *NotificationClass) lockDeliveryKey(key string) func() { + if key == "" { + return func() {} + } + c.deliveryMu.Lock() + if c.deliveryKeys == nil { + c.deliveryKeys = make(map[string]*notificationKeyLock) + } + entry := c.deliveryKeys[key] + if entry == nil { + entry = new(notificationKeyLock) + c.deliveryKeys[key] = entry + } + entry.refs++ + c.deliveryMu.Unlock() + entry.mu.Lock() + return func() { + entry.mu.Unlock() + c.deliveryMu.Lock() + entry.refs-- + if entry.refs == 0 { + delete(c.deliveryKeys, key) + } + c.deliveryMu.Unlock() + } } func NewNotificationClass() *NotificationClass { @@ -202,44 +422,29 @@ func (c *NotificationClass) sortList() { func (c *NotificationClass) UnMuteNotification(notificationGroupID uint64, muteLabel string) { fullMuteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID)) - Cache.Delete(fullMuteLabel) + c.listMu.RLock() + for id := range c.groupToIDList[notificationGroupID] { + key := notificationMethodMuteLabel(fullMuteLabel, id) + c.deliveryMu.Lock() + if c.muteGeneration == nil { + c.muteGeneration = make(map[string]uint64) + } + c.muteGeneration[key]++ + Cache.Delete(key) + c.deliveryMu.Unlock() + } + c.listMu.RUnlock() +} + +func notificationMethodMuteLabel(label string, id uint64) string { + return fmt.Sprintf("%s:method-%d", label, id) } // SendNotification 向指定的通知方式组的所有通知方式发送通知 -func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) { +func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc string, muteLabel string, ext ...*model.Server) bool { + fullMuteLabel := "" if muteLabel != "" { - // 将通知方式组名称加入静音标志 - muteLabel := NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID)) - // 通知防骚扰策略 - var flag bool - if cacheN, has := Cache.Get(muteLabel); has { - nHistory := cacheN.(NotificationHistory) - // 每次提醒都增加一倍等待时间，最后每天最多提醒一次 - if time.Now().After(nHistory.Until) { - flag = true - nHistory.Duration *= 2 - if nHistory.Duration > time.Hour*24 { - nHistory.Duration = time.Hour * 24 - } - nHistory.Until = time.Now().Add(nHistory.Duration) - // 缓存有效期加 10 分钟 - Cache.Set(muteLabel, nHistory, nHistory.Duration+time.Minute*10) - } - } else { - // 新提醒直接通知 - flag = true - Cache.Set(muteLabel, NotificationHistory{ - Duration: firstNotificationDelay, - Until: time.Now().Add(firstNotificationDelay), - }, firstNotificationDelay+time.Minute*10) - } - - if !flag { - if Conf.Debug { - log.Println("NEZHA>> Muted repeated notification", desc, muteLabel) - } - return - } + fullMuteLabel = NotificationMuteLabel.AppendNotificationGroupName(muteLabel, c.GetGroupName(notificationGroupID)) } // Copy the group under the lock. Webhook delivery can take minutes and must // not block notification or notification-group updates while it is in flight. @@ -250,24 +455,82 @@ func (c *NotificationClass) SendNotification(notificationGroupID uint64, desc st } c.listMu.RUnlock() - for _, n := range notifications { - log.Printf("NEZHA>> Try to notify %s", n.Name) + if len(notifications) == 0 { + log.Printf("NEZHA>> Notification group %d has no recipients", notificationGroupID) + return false } + allDelivered := true for _, n := range notifications { - ns := model.NotificationServerBundle{ - Notification: n, - Server: nil, - Loc: Loc, - } + var server *model.Server if len(ext) > 0 { - ns.Server = ext[0] + server = ext[0] } - if err := ns.Send(desc); err != nil { - log.Printf("NEZHA>> Sending notification to %s failed: %v", n.Name, err) - } else { - log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name) + if !c.sendRecipient(n, desc, fullMuteLabel, server) { + allDelivered = false } } + return allDelivered +} + +func (c *NotificationClass) sendRecipient(n *model.Notification, desc, fullMuteLabel string, server *model.Server) bool { + if n == nil { + return false + } + cacheKey := "" + if fullMuteLabel != "" { + cacheKey = notificationMethodMuteLabel(fullMuteLabel, n.ID) + } + unlock := c.lockDeliveryKey(cacheKey) + defer unlock() + var generation uint64 + if cacheKey != "" { + c.deliveryMu.Lock() + generation = c.muteGeneration[cacheKey] + c.deliveryMu.Unlock() + } + var history NotificationHistory + if cacheKey != "" { + if value, exists := Cache.Get(cacheKey); exists { + history = value.(NotificationHistory) + if time.Now().Before(history.Until) { + return true // this recipient already received this phase + } + } + } + log.Printf("NEZHA>> Try to notify %s", n.Name) + var err error + if c.sendForTest != nil { + err = c.sendForTest(n, desc, server) + } else { + err = (&model.NotificationServerBundle{Notification: n, Server: server, Loc: Loc}).Send(desc) + } + if err != nil { + // Webhook errors can contain a credential-bearing request URL. Keep + // failure type without ever writing the URL/token to dashboard logs. + log.Printf("NEZHA>> Sending notification to %s failed (%s)", n.Name, notificationFailureSummary(err)) + return false + } + log.Printf("NEZHA>> Sending notification to %s succeeded", n.Name) + if cacheKey != "" { + if history.Duration == 0 { + history.Duration = firstNotificationDelay + } else { + history.Duration = min(history.Duration*2, 24*time.Hour) + } + history.Until = time.Now().Add(history.Duration) + c.deliveryMu.Lock() + if c.muteGeneration[cacheKey] == generation { + Cache.Set(cacheKey, history, history.Duration+10*time.Minute) + } + c.deliveryMu.Unlock() + } + return true +} + +func notificationFailureSummary(err error) string { + // Even wrapped URL errors can include credentials in path/query. Log only + // the error type; the HTTP status path already returns a dedicated type. + return fmt.Sprintf("%T", err) } type _NotificationMuteLabel struct{} diff --git a/service/singleton/notification_test.go b/service/singleton/notification_test.go index 5e08de31..39ea4102 100644 --- a/service/singleton/notification_test.go +++ b/service/singleton/notification_test.go @@ -1,12 +1,405 @@ package singleton import ( + "errors" + "fmt" + "net/url" + "strings" + "sync/atomic" "testing" "time" + "github.com/patrickmn/go-cache" + "github.com/nezhahq/nezha/model" ) +func TestNotificationFailureDoesNotMuteRetryOrResendSuccessfulRecipient(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + first := &model.Notification{Common: model.Common{ID: 301}, Name: "first"} + second := &model.Notification{Common: model.Common{ID: 302}, Name: "second"} + nc := newNotificationClassWithItems(first, second) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 303}, Name: "retry-test"}, []uint64{first.ID, second.ID}) + counts := map[uint64]int{} + nc.sendForTest = func(n *model.Notification, _ string, _ *model.Server) error { + counts[n.ID]++ + if n.ID == second.ID && counts[n.ID] == 1 { + return errors.New("temporary failure") + } + return nil + } + label := "alert-retry-test" + nc.UnMuteNotification(303, label) + if nc.SendNotification(303, "incident", label) { + t.Fatal("partial delivery must report failure") + } + if !nc.SendNotification(303, "incident", label) { + t.Fatal("failed recipient should be retried successfully") + } + if counts[first.ID] != 1 || counts[second.ID] != 2 { + t.Fatalf("delivery attempts = %v, want first=1 second=2", counts) + } + nc.UnMuteNotification(303, label) +} + +func TestNotificationConcurrentSameRecipientSendsOnce(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 401}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 402}, Name: "concurrent-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return nil + } + done := make(chan bool, 2) + go func() { done <- nc.SendNotification(402, "incident", "same-event") }() + <-entered + go func() { done <- nc.SendNotification(402, "incident", "same-event") }() + close(release) + if !<-done || !<-done { + t.Fatal("concurrent delivery returned failure") + } + if calls.Load() != 1 { + t.Fatalf("same event delivered %d times, want 1", calls.Load()) + } +} + +func TestNotificationAsyncCoalescesBlockedServiceReports(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 501}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 502}, Name: "service-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return nil + } + nc.SendNotificationAsync(502, "down", "service-down") + <-entered + for i := 0; i < 1000; i++ { + nc.SendNotificationAsync(502, "down", "service-down") + } + nc.deliveryMu.Lock() + inFlight := len(nc.asyncInFlight) + nc.deliveryMu.Unlock() + if inFlight != 1 || calls.Load() != 1 { + t.Fatalf("in-flight sends=%d attempts=%d, want one each", inFlight, calls.Load()) + } + close(release) + deadline := time.After(time.Second) + for { + nc.deliveryMu.Lock() + remaining := len(nc.asyncInFlight) + nc.deliveryMu.Unlock() + if remaining == 0 { + break + } + select { + case <-deadline: + t.Fatal("async notification did not finish") + case <-time.After(time.Millisecond): + } + } +} + +func TestNotificationFailureSummaryOmitsCredentialURL(t *testing.T) { + err := &url.Error{Op: "Post", URL: "https://example.test/bot-secret-token/send?key=secret-query", Err: errors.New("timeout")} + if summary := notificationFailureSummary(err); strings.Contains(summary, "secret") || strings.Contains(summary, "example.test") { + t.Fatalf("credential-bearing URL leaked: %s", summary) + } +} + +func TestNotificationAsyncBacksOffFastFailures(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 511}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 512}, Name: "backoff-test"}, []uint64{n.ID}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + calls.Add(1) + return errors.New("fast failure") + } + key := "512:service-down" + nc.SendNotificationAsync(512, "down", "service-down") + deadline := time.After(time.Second) + for { + nc.deliveryMu.Lock() + busy := len(nc.asyncInFlight) != 0 + after := nc.asyncRetryAfter[key] + nc.deliveryMu.Unlock() + if !busy && !after.IsZero() { + break + } + select { + case <-deadline: + t.Fatal("first failed send did not record backoff") + case <-time.After(time.Millisecond): + } + } + for i := 0; i < 1000; i++ { + nc.SendNotificationAsync(512, "down", "service-down") + } + if calls.Load() != 1 { + t.Fatalf("fast failures caused %d attempts before retry-after", calls.Load()) + } +} + +func TestUnMuteDuringInflightSendCannotRemuteRecovery(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 521}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 522}, Name: "generation-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return nil + } + done := make(chan bool, 1) + go func() { done <- nc.SendNotification(522, "first failure", "static-label") }() + <-entered + nc.UnMuteNotification(522, "static-label") + close(release) + if !<-done { + t.Fatal("first send failed unexpectedly") + } + if !nc.SendNotification(522, "next failure", "static-label") || calls.Load() != 2 { + t.Fatalf("old in-flight send re-muted next failure: calls=%d", calls.Load()) + } +} + +func TestAsyncBackoffMemoryIsBounded(t *testing.T) { + nc := &NotificationClass{ + asyncRetryAfter: make(map[string]time.Time), + asyncRetryDelay: make(map[string]time.Duration), + } + for i := 0; i < maxAsyncBackoffKeys+100; i++ { + key := fmt.Sprintf("old-event-%d", i) + nc.asyncRetryAfter[key] = time.Now().Add(time.Minute) + nc.asyncRetryDelay[key] = time.Minute + } + nc.pruneAsyncBackoffLocked() + if len(nc.asyncRetryAfter) > maxAsyncBackoffKeys || len(nc.asyncRetryDelay) != len(nc.asyncRetryAfter) { + t.Fatalf("unbounded backoff maps: after=%d delay=%d", len(nc.asyncRetryAfter), len(nc.asyncRetryDelay)) + } +} + +func TestServiceStateNotificationsKeepIncidentRecoveryOrder(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 531}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 532}, Name: "ordered-test"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + delivered := make(chan string, 3) + nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error { + if desc == "down" { + close(entered) + <-release + } + delivered <- desc + return nil + } + nc.SendServiceState(533, 532, "down", "state-down-1") + <-entered + nc.SendServiceState(533, 532, "good", "state-good-2") + select { + case message := <-delivered: + t.Fatalf("service transition overtook blocked incident: %s", message) + default: + } + close(release) + for _, want := range []string{"down", "good"} { + select { + case got := <-delivered: + if got != want { + t.Fatalf("delivery order got %s, want %s", got, want) + } + case <-time.After(time.Second): + t.Fatalf("missing %s delivery", want) + } + } + deadline := time.After(time.Second) + for { + nc.deliveryMu.Lock() + remaining := len(nc.serviceDelivery) + nc.deliveryMu.Unlock() + if remaining == 0 { + break + } + select { + case <-deadline: + t.Fatal("ordered delivery worker did not exit") + case <-time.After(time.Millisecond): + } + } +} + +func TestServiceStateRetriesFailedIncidentBeforePendingRecovery(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 535}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 536}, Name: "retry-order-test"}, []uint64{n.ID}) + t.Cleanup(func() { nc.CancelServiceState(537) }) + + attempts := make(chan string, 3) + releaseFirst := make(chan struct{}) + var downAttempts atomic.Int32 + nc.sendForTest = func(_ *model.Notification, desc string, _ *model.Server) error { + attempts <- desc + if desc == "down" && downAttempts.Add(1) == 1 { + <-releaseFirst + return errors.New("temporary incident failure") + } + return nil + } + + nc.SendServiceState(537, 536, "down", "state-down-1") + nc.deliveryMu.Lock() + state := nc.serviceDelivery[537] + nc.deliveryMu.Unlock() + select { + case got := <-attempts: + if got != "down" { + t.Fatalf("first delivery = %q, want down", got) + } + case <-time.After(time.Second): + t.Fatal("incident delivery did not start") + } + nc.SendServiceState(537, 536, "good", "state-good-2") + close(releaseFirst) + + for _, want := range []string{"down", "good"} { + select { + case got := <-attempts: + if got != want { + t.Fatalf("delivery after failed incident = %q, want %q", got, want) + } + case <-time.After(time.Second): + t.Fatalf("missing %s delivery", want) + } + } + select { + case <-state.done: + case <-time.After(time.Second): + t.Fatal("ordered delivery worker did not finish") + } +} + +func TestServiceStateCancelStopsOldConfiguration(t *testing.T) { + oldCache := Cache + Cache = cache.New(time.Minute, time.Minute) + t.Cleanup(func() { Cache = oldCache }) + n := &model.Notification{Common: model.Common{ID: 541}, Name: "hook"} + nc := newNotificationClassWithItems(n) + nc.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 542}, Name: "old-group"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + nc.sendForTest = func(*model.Notification, string, *model.Server) error { + if calls.Add(1) == 1 { + close(entered) + <-release + } + return errors.New("old webhook failed") + } + nc.SendServiceState(543, 542, "down", "old-event") + <-entered + nc.deliveryMu.Lock() + state := nc.serviceDelivery[543] + nc.deliveryMu.Unlock() + nc.CancelServiceState(543) + nc.deliveryMu.Lock() + remaining := len(nc.serviceDelivery) + nc.deliveryMu.Unlock() + if remaining != 0 { + t.Fatal("old service delivery worker remained registered after cancellation") + } + close(release) + select { + case <-state.done: + case <-time.After(time.Second): + t.Fatal("canceled old delivery worker did not exit") + } +} + +func TestServiceUpdateCancelsOldRecipientRetry(t *testing.T) { + ss := newServiceMonitorSecurityHarness(t) + service := &model.Service{ + Common: model.Common{ID: 551, UserID: 1}, + Name: "generic-probe", + Type: model.TaskTypeTCPPing, + Target: "example.invalid:443", + Duration: 3600, + Notify: true, + NotificationGroupID: 552, + Cover: model.ServiceCoverIgnoreAll, + } + addServiceMonitorSecurityService(t, ss, service) + n := &model.Notification{Common: model.Common{ID: 553}, Name: "old-hook"} + NotificationShared.InsertForTest(n) + NotificationShared.UpdateGroup(&model.NotificationGroup{Common: model.Common{ID: 552}, Name: "old-group"}, []uint64{n.ID}) + entered := make(chan struct{}) + release := make(chan struct{}) + NotificationShared.sendForTest = func(*model.Notification, string, *model.Server) error { + close(entered) + <-release + return errors.New("failed old recipient") + } + NotificationShared.SendServiceState(service.ID, 552, "old down", "old-state") + <-entered + NotificationShared.deliveryMu.Lock() + state := NotificationShared.serviceDelivery[service.ID] + NotificationShared.deliveryMu.Unlock() + updated := *service + updated.Notify = false + updated.NotificationGroupID = 554 + if err := ss.Update(&updated); err != nil { + t.Fatal(err) + } + close(release) + select { + case <-state.done: + case <-time.After(time.Second): + t.Fatal("old recipient retry did not stop after service update") + } + NotificationShared.deliveryMu.Lock() + remaining := len(NotificationShared.serviceDelivery) + NotificationShared.deliveryMu.Unlock() + if remaining != 0 { + t.Fatal("old recipient retained after service update") + } +} + func newNotificationClassWithItems(items ...*model.Notification) *NotificationClass { nc := NewEmptyNotificationClassForTest() for _, item := range items { diff --git a/service/singleton/server.go b/service/singleton/server.go index a74a7f62..64ffd645 100644 --- a/service/singleton/server.go +++ b/service/singleton/server.go @@ -117,6 +117,7 @@ func (c *ServerClass) Update(s *model.Server, uuid string) { func (c *ServerClass) Delete(idList []uint64) { c.lockLifecycleWrite() defer c.unlockLifecycleWrite() + cancelAlertDeliveriesForServers(idList) c.listMu.Lock() diff --git a/service/singleton/servicesentinel.go b/service/singleton/servicesentinel.go index a2c6ef8d..3745c915 100644 --- a/service/singleton/servicesentinel.go +++ b/service/singleton/servicesentinel.go @@ -2,6 +2,7 @@ package singleton import ( "cmp" + "crypto/sha256" "fmt" "iter" "log" @@ -9,6 +10,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/gin-gonic/gin" @@ -22,9 +24,11 @@ import ( ) const ( - _CurrentStatusSize = 30 // 统计 15 分钟内的数据为当前状态 + _CurrentStatusSize = 30 // initial capacity for the rolling service window ) +var serviceEventSequence atomic.Uint64 + type serviceResponseItem struct { model.ServiceResponseItem @@ -46,9 +50,101 @@ type _TodayStatsOfService struct { type serviceResponseData = _TodayStatsOfService type serviceTaskStatus struct { - lastStatus uint8 - t time.Time - result []*pb.TaskResult + lastStatus uint8 + stateEpoch uint64 + lastHistoryAt time.Time + history serviceResponseData + lastData string + result []serviceWindowBucket +} + +type serviceProbeSample struct { + successful bool + failed bool + delay float64 +} + +type serviceWindowBucket struct { + at time.Time + reporters map[uint64]serviceProbeSample +} + +func (status *serviceTaskStatus) appendResult(now time.Time, reporter uint64, result *pb.TaskResult) { + bucketAt := now.Truncate(30 * time.Second) + if len(status.result) > 0 && bucketAt.Before(status.result[len(status.result)-1].at) { + status.result = nil // wall-clock moved backwards; old buckets are not comparable + } + if len(status.result) == 0 || !status.result[len(status.result)-1].at.Equal(bucketAt) { + status.result = append(status.result, serviceWindowBucket{at: bucketAt, reporters: make(map[uint64]serviceProbeSample)}) + } + bucket := &status.result[len(status.result)-1] + sample := bucket.reporters[reporter] + if result.Successful { + sample.successful = true + sample.delay = float64(result.Delay) + } else { + sample.failed = true + } + bucket.reporters[reporter] = sample + cutoff := now.Add(-15 * time.Minute) + first := 0 + for first < len(status.result) && status.result[first].at.Before(cutoff) { + first++ + } + if first > 0 { + status.result = append([]serviceWindowBucket(nil), status.result[first:]...) + } + if result.Successful { + status.history.Up++ + status.history.Delay = (status.history.Delay*float64(status.history.Up-1) + float64(result.Delay)) / float64(status.history.Up) + } else { + status.history.Down++ + } + status.lastData = result.Data +} + +// flushServiceHistory persists the previous non-overlapping reporting period +// before the next observation is added, so long probe gaps cannot lose or +// double-count samples in the 15-minute availability history. +func (ss *ServiceSentinel) flushServiceHistory(serviceID uint64, status *serviceTaskStatus, now time.Time) { + if status.lastHistoryAt.IsZero() || now.Before(status.lastHistoryAt.Add(15*time.Minute)) { + return + } + if status.history.Up+status.history.Down == 0 { + status.lastHistoryAt = now + return + } + if !TSDBEnabled() { + if err := DB.Create(&model.ServiceHistory{ + CreatedAt: status.lastHistoryAt, + ServiceID: serviceID, + AvgDelay: status.history.Delay, + Data: status.lastData, + Up: status.history.Up, + Down: status.history.Down, + }).Error; err != nil { + log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err) + return + } + } + status.history = serviceResponseData{} + status.lastHistoryAt = now +} + +func (status *serviceTaskStatus) responseData() serviceResponseData { + var data serviceResponseData + for _, bucket := range status.result { + for _, sample := range bucket.reporters { + if sample.successful { + data.Up++ + data.Delay = (data.Delay*float64(data.Up-1) + sample.delay) / float64(data.Up) + } + if sample.failed { + data.Down++ + } + } + } + return data } type pingStore struct { @@ -228,7 +324,7 @@ func (ss *ServiceSentinel) loadServiceHistory() error { } ss.services[service.ID] = service ss.serviceCurrentStatusData[service.ID] = new(serviceTaskStatus) - ss.serviceCurrentStatusData[service.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize) + ss.serviceCurrentStatusData[service.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize) ss.serviceStatusToday[service.ID] = &_TodayStatsOfService{} validServices = append(validServices, service) } @@ -371,6 +467,9 @@ func (ss *ServiceSentinel) Update(m *model.Service) error { if err != nil { return err } + // A service edit may disable notifications or change their recipients. + // Cancel any retry worker holding the previous service configuration. + NotificationShared.CancelServiceState(m.ID) if ss.services[m.ID] != nil { // 停掉旧任务 CronShared.Remove(ss.services[m.ID].CronJobID) @@ -387,7 +486,7 @@ func (ss *ServiceSentinel) Update(m *model.Service) error { if ss.serviceCurrentStatusData[m.ID] == nil { ss.serviceCurrentStatusData[m.ID] = new(serviceTaskStatus) } - ss.serviceCurrentStatusData[m.ID].result = make([]*pb.TaskResult, 0, _CurrentStatusSize) + ss.serviceCurrentStatusData[m.ID].result = make([]serviceWindowBucket, 0, _CurrentStatusSize) ss.serviceStatusToday[m.ID] = &_TodayStatsOfService{} } // 更新这个任务 @@ -404,6 +503,7 @@ func (ss *ServiceSentinel) Delete(ids []uint64) { defer ss.servicesLock.Unlock() for _, id := range ids { + NotificationShared.CancelServiceState(id) delete(ss.serviceCurrentStatusData, id) delete(ss.serviceResponseDataStore, id) delete(ss.serviceResponsePing, id) @@ -669,32 +769,16 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass } currentTime := time.Now() - if serviceCurrentStatusData.t.IsZero() { - serviceCurrentStatusData.t = currentTime - } - - // 写入当前数据 - if serviceCurrentStatusData.t.Before(currentTime) { - serviceCurrentStatusData.t = currentTime.Add(30 * time.Second) - serviceCurrentStatusData.result = append(serviceCurrentStatusData.result, mh) + ss.flushServiceHistory(mh.GetId(), serviceCurrentStatusData, currentTime) + // Include every accepted reporter's result. A fixed count of samples is + // not a 15-minute window when probes have different intervals or reporters. + serviceCurrentStatusData.appendResult(currentTime, r.Reporter, mh) + if serviceCurrentStatusData.lastHistoryAt.IsZero() { + serviceCurrentStatusData.lastHistoryAt = currentTime } // 更新当前状态 - ss.serviceResponseDataStore[mh.GetId()] = serviceResponseData{} - - // 永远是最新的 30 个数据的状态 [01:00, 02:00, 03:00] -> [04:00, 02:00, 03: 00] - for _, cs := range serviceCurrentStatusData.result { - if cs.GetId() > 0 { - rd := ss.serviceResponseDataStore[mh.GetId()] - if cs.Successful { - rd.Up++ - rd.Delay = (rd.Delay*float64(rd.Up-1) + float64(cs.Delay)) / float64(rd.Up) - } else { - rd.Down++ - } - ss.serviceResponseDataStore[mh.GetId()] = rd - } - } + ss.serviceResponseDataStore[mh.GetId()] = serviceCurrentStatusData.responseData() // 计算在线率， var stateCode uint8 @@ -704,24 +788,11 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass if rd.Down+rd.Up > 0 { upPercent = rd.Up * 100 / (rd.Down + rd.Up) } - stateCode = GetStatusCode(upPercent) - } - - if len(serviceCurrentStatusData.result) == _CurrentStatusSize { - serviceCurrentStatusData.t = currentTime - if !TSDBEnabled() { - rd := ss.serviceResponseDataStore[mh.GetId()] - if err := DB.Create(&model.ServiceHistory{ - ServiceID: mh.GetId(), - AvgDelay: rd.Delay, - Data: mh.Data, - Up: rd.Up, - Down: rd.Down, - }).Error; err != nil { - log.Printf("NEZHA>> Failed to save service monitor metrics: %v", err) - } + if rd.Down+rd.Up == 0 { + stateCode = StatusNoData + } else { + stateCode = GetStatusCode(upPercent) } - serviceCurrentStatusData.result = serviceCurrentStatusData.result[:0] } // 延迟报警 @@ -734,77 +805,100 @@ func (ss *ServiceSentinel) processReport(r ReportData, serverShared *ServerClass lastStatus := serviceCurrentStatusData.lastStatus // 存储新的状态值 serviceCurrentStatusData.lastStatus = stateCode - - notifyCheck(&r, m, cs, mh, lastStatus, stateCode) - } - - // TLS 证书报警 - if ss.serviceReportBeforeTLSSideEffectsHook != nil { - ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId()) - } - var errMsg string - if strings.HasPrefix(mh.Data, "SSL证书错误：") { - // i/o timeout、connection timeout、EOF 错误 - if !strings.HasSuffix(mh.Data, "timeout") && - !strings.HasSuffix(mh.Data, "EOF") && - !strings.HasSuffix(mh.Data, "timed out") { - errMsg = mh.Data - if cs.Notify { - muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network") - go NotificationShared.SendNotification(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel) - } + if stateCode != lastStatus { + serviceCurrentStatusData.stateEpoch = serviceEventSequence.Add(1) } - } else { - // 清除网络错误静音缓存 - NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")) - var newCert = strings.Split(mh.Data, "|") - if len(newCert) > 1 { - enableNotify := cs.Notify + notifyCheck(&r, m, cs, mh, lastStatus, stateCode, serviceCurrentStatusData.stateEpoch) + } - // 首次获取证书信息时，缓存证书信息 - if ss.tlsCertCache[mh.GetId()] == "" { - ss.tlsCertCache[mh.GetId()] = mh.Data + // TLS 证书报警仅适用于 HTTPS 探针。其它探针的任意响应文本 + // 不得被当作证书资料解析。 + if mh.Type == model.TaskTypeHTTPGet && strings.HasPrefix(strings.ToLower(cs.Target), "https://") { + if ss.serviceReportBeforeTLSSideEffectsHook != nil { + ss.serviceReportBeforeTLSSideEffectsHook(mh.GetId()) + } + var errMsg string + if strings.HasPrefix(mh.Data, "SSL证书错误：") { + // i/o timeout、connection timeout、EOF 错误 + if !strings.HasSuffix(mh.Data, "timeout") && + !strings.HasSuffix(mh.Data, "EOF") && + !strings.HasSuffix(mh.Data, "timed out") { + errMsg = mh.Data + if cs.Notify { + muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), "network") + NotificationShared.SendNotificationAsync(cs.NotificationGroupID, Localizer.Tf("[TLS] Fetch cert info failed, Reporter: %s, Error: %s", cs.Name, errMsg), muteLabel) + } } + } else { + // 清除网络错误静音缓存 + NotificationShared.UnMuteNotification(cs.NotificationGroupID, NotificationMuteLabel.ServiceTLS(mh.GetId(), "network")) - oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|") - isCertChanged := false - expiresOld, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1]) - expiresNew, _ := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1]) - - // 证书变更时，更新缓存 - if oldCert[0] != newCert[0] && !expiresNew.Equal(expiresOld) { - isCertChanged = true - ss.tlsCertCache[mh.GetId()] = mh.Data - } - - notificationGroupID := cs.NotificationGroupID - serviceName := cs.Name - - // 需要发送提醒 - if enableNotify { - // 证书过期提醒 - if expiresNew.Before(time.Now().AddDate(0, 0, 7)) { - expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05") - errMsg = Localizer.Tf( - "The TLS certificate will expire within seven days. Expiration time: %s", - expiresTimeStr, - ) - - // 静音规则： 服务id+证书过期时间 - // 用于避免多个监测点对相同证书同时报警 - muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr)) - go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel) + var newCert = strings.Split(mh.Data, "|") + if len(newCert) > 1 { + enableNotify := cs.Notify + expiresNew, newErr := time.Parse("2006-01-02 15:04:05 -0700 MST", newCert[1]) + if newErr != nil { + log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId()) + return } - // 证书变更提醒 - if isCertChanged { - errMsg = Localizer.Tf( - "TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s", - oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05")) + // 首次获取证书信息时，缓存证书信息 + if ss.tlsCertCache[mh.GetId()] == "" { + ss.tlsCertCache[mh.GetId()] = mh.Data + } - // 证书变更后会自动更新缓存，所以不需要静音 - go NotificationShared.SendNotification(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), "") + oldCert := strings.Split(ss.tlsCertCache[mh.GetId()], "|") + isCertChanged := false + expiresOld, oldErr := time.Parse("2006-01-02 15:04:05 -0700 MST", oldCert[1]) + if oldErr != nil { + log.Printf("NEZHA>> Ignoring malformed TLS certificate expiry for service %d", mh.GetId()) + return + } + + // 只有通知成功后才推进基线；失败时下一次探针结果会重试。 + if oldCert[0] != newCert[0] || !expiresNew.Equal(expiresOld) { + isCertChanged = true + if !enableNotify { + ss.tlsCertCache[mh.GetId()] = mh.Data + } + } + + notificationGroupID := cs.NotificationGroupID + serviceName := cs.Name + + // 需要发送提醒 + if enableNotify { + // 证书过期提醒 + if expiresNew.Before(time.Now().AddDate(0, 0, 7)) { + expiresTimeStr := expiresNew.Format("2006-01-02 15:04:05") + errMsg = Localizer.Tf( + "The TLS certificate will expire within seven days. Expiration time: %s", + expiresTimeStr, + ) + + // 静音规则： 服务id+证书过期时间 + // 用于避免多个监测点对相同证书同时报警 + muteLabel := NotificationMuteLabel.ServiceTLS(mh.GetId(), fmt.Sprintf("expire_%s", expiresTimeStr)) + NotificationShared.SendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel) + } + + // 证书变更提醒 + if isCertChanged { + errMsg = Localizer.Tf( + "TLS certificate changed, old: issuer %s, expires at %s; new: issuer %s, expires at %s", + oldCert[0], expiresOld.Format("2006-01-02 15:04:05"), newCert[0], expiresNew.Format("2006-01-02 15:04:05")) + + oldRaw, newRaw, serviceID := ss.tlsCertCache[mh.GetId()], mh.Data, mh.GetId() + muteLabel := NotificationMuteLabel.ServiceTLS(serviceID, fmt.Sprintf("change_%x", sha256.Sum256([]byte(newRaw)))) + NotificationShared.sendNotificationAsync(notificationGroupID, fmt.Sprintf("[TLS] %s %s", serviceName, errMsg), muteLabel, func() { + ss.serviceResponseDataStoreLock.Lock() + if ss.serviceCurrentStatusData[serviceID] != nil && ss.tlsCertCache[serviceID] == oldRaw { + ss.tlsCertCache[serviceID] = newRaw + } + ss.serviceResponseDataStoreLock.Unlock() + }) + } } } } @@ -832,11 +926,11 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh if mh.Delay > ss.MaxLatency { // 延迟超过最大值 msg := Localizer.Tf("[Latency] %s %2f > %2f, Reporter: %s", ss.Name, mh.Delay, ss.MaxLatency, reporterServer.Name) - go NotificationShared.SendNotification(notificationGroupID, msg, minMuteLabel) + NotificationShared.SendNotificationAsync(notificationGroupID, msg, minMuteLabel) } else if mh.Delay < ss.MinLatency { // 延迟低于最小值 msg := Localizer.Tf("[Latency] %s %2f < %2f, Reporter: %s", ss.Name, mh.Delay, ss.MinLatency, reporterServer.Name) - go NotificationShared.SendNotification(notificationGroupID, msg, maxMuteLabel) + NotificationShared.SendNotificationAsync(notificationGroupID, msg, maxMuteLabel) } else { // 正常延迟， 清除静音缓存 NotificationShared.UnMuteNotification(notificationGroupID, minMuteLabel) @@ -845,7 +939,7 @@ func delayCheck(r *ReportData, m map[uint64]*model.Server, ss *model.Service, mh } func notifyCheck(r *ReportData, m map[uint64]*model.Server, - ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8) { + ss *model.Service, mh *pb.TaskResult, lastStatus, stateCode uint8, stateEpoch uint64) { // GHSA-jx78-55p5-rwv5: guard against concurrent server deletion (same TOCTOU // class as the 2026-07-21 fix, a few dozen lines lower in the same worker). // ServerShared has its own lock; m is a snapshot taken outside @@ -858,14 +952,9 @@ func notifyCheck(r *ReportData, m map[uint64]*model.Server, if isNeedSendNotification && reporterServer != nil { notificationGroupID := ss.NotificationGroupID notificationMsg := Localizer.Tf("[%s] %s Reporter: %s, Error: %s", StatusCodeToString(stateCode), ss.Name, reporterServer.Name, mh.Data) - muteLabel := NotificationMuteLabel.ServiceStateChanged(mh.GetId()) + muteLabel := fmt.Sprintf("%s:%d:%d", NotificationMuteLabel.ServiceStateChanged(mh.GetId()), stateCode, stateEpoch) - // 状态变更时，清除静音缓存 - if stateCode != lastStatus { - NotificationShared.UnMuteNotification(notificationGroupID, muteLabel) - } - - go NotificationShared.SendNotification(notificationGroupID, notificationMsg, muteLabel) + NotificationShared.SendServiceState(mh.GetId(), notificationGroupID, notificationMsg, muteLabel) } // 判断是否需要触发任务 @@ -890,9 +979,6 @@ const ( ) func GetStatusCode[T constraints.Float | constraints.Integer](percent T) uint8 { - if percent == 0 { - return StatusNoData - } if percent > 95 { return StatusGood } diff --git a/service/singleton/servicesentinel_window_test.go b/service/singleton/servicesentinel_window_test.go new file mode 100644 index 00000000..e5c85147 --- /dev/null +++ b/service/singleton/servicesentinel_window_test.go @@ -0,0 +1,63 @@ +package singleton + +import ( + "testing" + "time" + + pb "github.com/nezhahq/nezha/proto" +) + +func TestServiceWindowCountsFirstFailureAsDown(t *testing.T) { + status := new(serviceTaskStatus) + status.appendResult(time.Unix(1000, 0), 1, &pb.TaskResult{Successful: false}) + data := status.responseData() + if data.Down != 1 || data.Up != 0 || GetStatusCode(uint64(0)) != StatusDown { + t.Fatalf("first failed report = %+v, status %d; want Down", data, GetStatusCode(uint64(0))) + } +} + +func TestServiceWindowExpiresOldReportsAndDoesNotResetAtThirty(t *testing.T) { + status := new(serviceTaskStatus) + start := time.Unix(1000, 0) + for i := 0; i < 30; i++ { + status.appendResult(start.Add(time.Duration(i)*30*time.Second), 1, &pb.TaskResult{Successful: true}) + } + if data := status.responseData(); data.Up != 30 { + t.Fatalf("30th report reset window: %+v", data) + } + status.appendResult(start.Add(29*30*time.Second+time.Second), 2, &pb.TaskResult{Successful: false}) + if data := status.responseData(); data.Up != 30 || data.Down != 1 { + t.Fatalf("31st report corrupted window: %+v", data) + } + status.appendResult(start.Add(31*time.Minute), 1, &pb.TaskResult{Successful: false}) + if data := status.responseData(); data.Up != 0 || data.Down != 1 { + t.Fatalf("expired reports still influence status: %+v", data) + } +} + +func TestServiceWindowDeduplicatesFloodWithinBucket(t *testing.T) { + status := new(serviceTaskStatus) + now := time.Unix(1000, 0) + for i := 0; i < 10000; i++ { + status.appendResult(now, 1, &pb.TaskResult{Successful: false, Data: "ignored payload"}) + } + if len(status.result) != 1 || len(status.result[0].reporters) != 1 { + t.Fatalf("flood grew live window: buckets=%d", len(status.result)) + } + if data := status.responseData(); data.Down != 1 { + t.Fatalf("duplicate reporter skewed status: %+v", data) + } + if status.history.Down != 10000 { + t.Fatalf("history lost raw reports: %+v", status.history) + } +} + +func TestServiceWindowDoesNotEraseFailureWithLaterSuccess(t *testing.T) { + status := new(serviceTaskStatus) + now := time.Unix(1000, 0) + status.appendResult(now, 1, &pb.TaskResult{Successful: false}) + status.appendResult(now.Add(time.Second), 1, &pb.TaskResult{Successful: true, Delay: 20}) + if data := status.responseData(); data.Down != 1 || data.Up != 1 { + t.Fatalf("later success erased failure in same bucket: %+v", data) + } +} diff --git a/service/singleton/singleton.go b/service/singleton/singleton.go index bc906afd..8d41b70b 100644 --- a/service/singleton/singleton.go +++ b/service/singleton/singleton.go @@ -167,7 +167,7 @@ func CleanMonitorHistory() { for _, alert := range alerts { for _, rule := range alert.Rules { // 是不是流量记录规则 - if !rule.IsTransferDurationRule() { + if !rule.HasSafeCycleConfiguration() { continue } dataCouldRemoveBefore := rule.GetTransferDurationStart().UTC() From ec637a688c3433fe0737e3b19ef02353bfff05fb Mon Sep 17 00:00:00 2001 From: naiba Date: Thu, 1 Oct 2026 16:37:10 +0000 Subject: [PATCH 40/48] docs: replace sponsors with placeholder --- .github/sponsor/99cdn.jpeg | Bin 10865 -> 0 bytes .github/sponsor/axisnow.jpeg | Bin 10272 -> 0 bytes .github/sponsor/vmrack.jpeg | Bin 18994 -> 0 bytes README.md | 40 +---------------------------------- 4 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 .github/sponsor/99cdn.jpeg delete mode 100644 .github/sponsor/axisnow.jpeg delete mode 100644 .github/sponsor/vmrack.jpeg diff --git a/.github/sponsor/99cdn.jpeg b/.github/sponsor/99cdn.jpeg deleted file mode 100644 index d3e098652992b7adfd4c552dcc0f1f465f3827f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10865 zcmd6MWmsEHw{~zZ#T{B)3KVxJr9}%Aid$NwKylZS0xj~h05Z|vJ|Zg65`g-5ydI*iMifNm zKj;5dV?+S}7>E|s=e)c;(MW&!NXRIFtbeuhJb#{r^xqk?kpGntNiYlLUp^o=5B={L zM6y4Ij)d}e3p(P0|KLVK{%_n!*#84J(!a7H5hDL{6+)uF^T{LX-@O0S@LBi*;Opz`(x;T8^#~Q{B2nO*q^hJkx>x`7CIXGKkfgq17AX5 z#)h{72rvN}fGiXwdH^y35()tlyb~!0ffwx$z~8X{7y>5(7zQR5HqK)JG7<_3GAarh z0xu%+^Y2baB|sy5$|H?Vr1k-W-hr6cH#!@W;blb|iTW6XkXfa3G@R z^~EH98J%6xhQ+|A4k0mi9K$AMZg{-*SY(SJtB@BfR@KZO3_1D^-rq97p#421w7 z1%Sa=b3y?BNr8|4jx9OCnNE4>Ny=K@fj@r&!?s&j#^|~NWE5(^$eaURB zQ%-A;qyxsdexT#MRhyoRh+(Rq3df0MqX&npRLMNY&FpnB9Dw%ex^0~fb9NUFz&|em z;_nwoogYlVqJ$!(#&SXN#1SBmrU?H6t8EZIs7(fBwS=QZnFS3?oV9Kui_-7I@ggB8 zQiJUQKc%Ee2ZmP8Y&w1fDRNO}Yre}X6ey?&sl1!_^z@{gdLV&B9s=(^?t*HB0)ITS zTQ%hFyjABgnk!-!3Xeb?B^qvbXZMi#;)p45v_c2Ahzx)OsuKNG!zAeqlRCfVmz4qq zx(2_Pn<}i(%&uv;2pq@|sAxPCd(u^I-CM=4J9ee1mo|SZAF5Qa^BJ`Y zh6DWS?wzMkmVBu9L|wm`HC}-%WE<-_7Mw`sdq!C{3|9`D?|fk6VB!`Ikcs;`tXrA1 z0rnt#&H)*`w`jy59R9tn;w7L0Lg^YO%Dm6ov0|KAe`P zvk#X!;Pmo2H~@xCa;NL9bv?7D8aQ(s9ysSxrnrpNuG3BaLS@h1GfWm=q9g)3b5DGC znfVhsTw}6kD5I*(g>#p5*rA)7`IN~l^qGpnJi0RX(L8n9?L!a8t)l(OlooeFAxCUv zd!2&NWrHEzcn!(@g*PXHUW2##xC`r13%5mrXLm*?Myl#N8%O`UHr*7Ko+}^K@8t_8 zUxDOJNYJ`AL74cpi%;Dp867v>vuJs+rwCE8=+(OmYU9S|9^csEfDRDDi7-Q#L2M8= zZcBMf%Yatm(#)1~q;?LN4Gvh%1=g%|0q4MakRRV5Is4PgHtIM<$FMQvL08$aJbgFk zP4Jcx${jVRO-_>S{^j+Y8RTFid@3)2a-PacwO5R!eyh^0A9+D`0?{((tr&Kfbc>bp%s8F3rx5INj1!6>c`IGR0c+eS*D%8dm3;3ar^JPAK>k;+MvpLL z^Kx;g5ZD)xHE6Ez(hCj-vhJ_4x>4fS;w4czqDOW5nUQhPXu58Dj50*9eon3a3A58Jquxzt2F;YJp;d=Q$zPGBTiqC;&N_M+aNtL+n>SF6O9o9vxcT#ve1 z-`YF6)2=}~F4eBd?ilU?(5udofYIgG#fG~vd)QY(oV;JbF~Sj}FFI(SgRmc%uhlln zalPfPopRhK+6hY=TyrgnqYk@?Z2;LlFTdI_RFIdJhpnJDQ9u`3&+f&cx5KxFgV74? z`u!h1KL6=_lbB`;*qCo=5HTbldzjQb%9W~u_~38OT1R)VWn*dCd&kor)z2xV3@@B7 zWZ!4OhJhsKnSONS&Y2;U-RaClq;XYowIOOaWC^6fIwD%#OLKd`EKsFeD|pWQvJxs- zQm|))F?}A&$wiW|eQRhs*kWCX^;q+p<4}NY6Vfq9G@@GqU;`jDsgDn0?X`5+)+c0Y z(>wKNsCD=bN!c1mn)Z3!UD6z!NT}8}xoMud1R!VcC4UrTh z$Qll~X2FYW<3?A7>aQMc4IM|mM*+?)lXzr>MTzfjY8v+bqH3c+iL&Q0dJPAl&Ry@7 zct3l{ekM-BziH9J?eeoe=wZxJ`i?)#)n=t}#D2LQ>qjtAig98pHHfWSOtwR!1 z47BgbbBI7Mc=d}q%?|alDb70aHS3;j%9Ug_qW?NHvsTn11;KE8tQq1uK>A zY$)%v@7(OXT!=_TvqYnat_lvo_0jH1HsTN;t}^JrDX(-)tvf9`-P;9YW|n3E9===q zu);c1%Da;pH72W%RC$zSyxUtipy_Bj#s4RVwdy5U{@k_)b-`vIa3c zt_som=R+>wM?S*Wp=&!Gh=6C+=*nWpc02N;Q;cPLk4dSiByn)DSw6O8b)5 z2;!ZGsqzE54ZAiv)O#<8+x-*J<8Om0z>GvZlIkP7=G?pN{)@LB3lmRh+nfAXGY_Tj zr0&t7QP7i)Bf)hEwfA1mG3#49m@y`8@Qo!*8 zV)*2+aVcuj4#m0%31C-B<^VM-DXaJP2zEWPaBQsal@_>EH5`Q6L;^i&D{Ret44Yt1 zm|#5lgFENWH^fnc)&YTgY6Gz#1$}@6=B`#%!xPsr{vcO=>Q1PXnwUo47v4ouul2-) zA&pQ}W81?_@zgK;7eQo52JyDIba>kut78X-;>^k1oZkFvJ<-gU-kjG~Yv6BEkNE8Z zVB|qe36VB5P2P@OK4+Pi>6JsrR*)SSw)P#hyMG6azvUrYl3J&EIkE_Lr4>N3I9&(8 zn>eCQ%G>Y6kFQk4h}>mECq{Y`gK4H-`Tf27lL($Y!=y!24)se_m1Vg-1GlwW6p4N< zO*Kc>h3N91(;>@R<4Y}a&=ZLMWx@3x1YLx)h0Lnl(oD?GOF@mL)_m7M?#WZtA_Mal zE5z%_lkUiej759mr_??jiO<~qtA{UefEjnNUS~sh)tW`=@s?PIvZw;55Yc4it5>FX zN9DXy5GCLoL86IeL!>ytId6@8DD3hDHqjA*Oh^3T05Wkl^VLjpeFHe)<7^*pFiA~EM|0UKDI^)&5oRER zjjsp+Uc2-h2H%Ui=E4EKXWB*_NFk07gJln(_{g$#5DsG7;c;rMUXWj!DIS;4!Fo_o z%oC;#PTdDd(NTabfT3^zor|yle#rb{O6FYfojAEpR;Zg{R5yvV1fSnh6|f8hQC^v& zGPw3Md&qC$)U62{o9}ywHGjUq=s+@<$jjE<-gnwvn4T9SLUM}(DY>M7NP<27q;}v# z0BIrwb~@2a9Y`#GmviPL#r!;p0#vvTl&*l1270Xxy%3r!UyWHd479S2R}{qDwRi7P zbjOWRD>^DsJQeenfZ&jP9h9m)o8PZ-Db-xi-rZs;se5gMzuwbH8NU;Dulcz&t){!e z<5`D<(xSZnO%2GH=bj4!D};FB``?};8}ghdo&U;}IvadJFMQ5Gfl)!N-KQsYPsDMl zHYQ*tq1U4X=A9j)hMs`q6P@2sI5uPUZfftokEreEEgIc%u_-!mf4asjMt{3_W~(b& zt7ll^HPbx4yaNq~rmlSiLsZv&svsSmFCKCg>MasfGOSsW@@YpXFwT#FIdl;DOdJ~R zjm#i#jdu)+V_}NRkuJPQH>JwJaylxoF&yv(Tum491e(~{Ea@SD$=UTYX5P);`I8c> zeHxEJ8l4Kq_iL29XWq0Aso)y?_63Eb;j*pIc_U&fs49oP-XEpMQ%JS!T5&*-P01ic`=vW&m@Gz;*;lJkj4{Q(lciZ;S+!DpKZJL zWV>k4L_+|!Q|EcJxmkr{FLYk;wB{rB*0y75V-<3rT!;c~dL_kDNOh8CR_z0hbnuLskf{KdjF)A#d z7Xt&Ak^rB8@_%iA2|~F@g8wcE0jLrB)n5wA-*ONDfY5@_P?1p(auGoBF#rh_p=JF> zi}73JLqz|PiwH0fnvmKDUm{{&X@_V`5;FRinvQ-k*%f2sqzvlsjC0!BJNRTYoZ>1c z7+-0{t|4+!Ap{yk)_-&-L^2dqWHfXn48)JTzoi-!WE3C`__HI#6hu*A6SFiDLhW{kl+)Iy(>gyKY;w#|kj>j(9JGGEYte$e zICAaJ7=)HI_!JfC<*&?d8l(=1MY3~TV>Nn0=aLrG zt)Ef9ZRr#Zb0_O1^oN$y%jnad-Zb$y{Ua|pGG%ue5!MLS8`dxhZc+@D)wU6LH@!(-Xe9Soz>!j3Y4w=407uot*D*E42S1yOJH9a$ zi^(n0wnMdYk@!UFcv$poq$dXs$X|%Rly2`ySbH9$vcjKGNB}#K30bEzMfY0Wa9A#c zF{$U2?hjq3#a*P{G+id0D?tpdRfKjt zl^v%}W^RBJ5X&-=?sTzKuO|(A&o2!J(Q;62B_gSpo8v=fFuv%?q{yy*3nKCDXkjrN zJL^(?vd6hb?ZFk9Kk73f6{-O#Ut{=ea)g=|$Cdb~9fC90t2#X7U-o*}l1Rj}tM8jg-IM%GvbuT3KIgM`BZ8Y4kDOUn-+frs! zdp-}EY9*bKBqgZ~y6>+H)pjXP8?M4iM4dF{^g7d0C|*8^Evf{$p6LJLP~(UzY!O#E zQD{D??N|3XU4;Xr9)CC3q2VJxTvm)1C1&(1EBauvOxKt)L+a=1X57rFW2QHu$`VaP zedP&QdTn}9B~x2#Rz zxsduERGO~y>J<{fV>E5(`rZES_mZDy294B?_;VIcuxSB3?DCB+jQKG&9TPX=;~vIL zrnbR5qbVBI(V>q;jrCIt6L+^1HHJ-iV>O)GnQvEuWBdEU1Sp|k|IL77I6#3tM%xah zCSEa0xv-MAvZ?ZoS@BN#*RKJm$$Mz;^1}~UP;y-*E+xu4B)p~4mHG~?OF7*inFMsw z+P@r3Gae^#Zt-lYuf?=9HFoM`$d2JZTwN}>+veoIs$~{A73K-2it+e@!|UL$JvF6! z3pHM5&W|w3C5#B{-DnK;k~nx-1*CFM2UgQWI$yy7T3Vtv57UF1uVTJ+0W@zgePiFX}YvcxYN zY)N?AoC(3kE+v*l9z51&Xr+)DsjqvzRi1f0I^^6#btd__H5v78TzO=1`>UEwAwNys zQdU63xID^u#j43By>ODL*YF_T9^*M$mT(WlPj89N4>@l9F>QtqrQxY?z^}5m<6auc zg<~cC^Wj~ZtBE5K*c{vmRcU<-eWt*HDzmLqx}T3)Mf{*S74>`tIV1bApAJytpiol7h_;VmVxysph7XDykmY)W&XTmWG$1y_lYA)+H%BbnT?{mS#jm*=4ncR zgV~`KW>`HxdU!xtWV+G>M#nJCC*4hdjQC!I1VP6| z3O3uJ-Mwcxb8(ka>-j-xk}6xewcF4ybGrKgx-xlTnLIdv`Mk+#-kcdGXSuVIUv`aq zY&T!ZG$>+No4vTtIjRKTmJ`o!|%2Z$Ym`=+@Vlbt^HMww-& z4JT>w_zcJObDP#4Rka1DU`ipBQBVAQ(=0ct;`pZ(X|_d$SJ?e*+uR}dWQj-@R?F6w z1Cd(-;gIvbJ(RVonPIW1@BQZL{=y0j(wLUss-N=`whQZVB9B;(af5=h6 zWJaE@yqh}f!C7X@+7A32KJd03KYN+z_LHX*958f4^C;DWC4Xx1x{be?rvvg)f+q`J%j}6+;<31qn9&SK$ zYrA*CEMx%=NEn9$HuaW1E~AV?Zil^4tvSY`S44*2-wEj{C`%hNpUlmB*-$ssU1A26 z(g^=G6|N4hm+bIpL1&}|j$ZAq)XAedM@*f(+BRV=DkSSm>Ug`#wHMYL_E9-i8SAQ? z_D!h-zTF3db|gK-9J5=hRh7rFl|wIUw`wix8pAN_*+@c_wrZQ(qqq_1Eb{5jiL^~F_WbqeOEl0@NQU#~O7}1VXlHehW**36w-f829 zs`{Z1_8>mb039o>lJ9JU*){7UAN{HmC(Lpi?zU1N?N{0&2`vRiV}ev-7~rpTOkZiE zYda%q9uL|pX>8=V1-(Ys!spA5lRawGTPOYz7c-2PugrX!E<8CqZzV2Vv;QlKN*5uS2=VnE9ew-M5Je!UYj$ggU z+cX2|UH4Q~p0roKcI=)iHuXKdr*MFMv*BEbGR?X3m5hA4xYv-`?OD(0fa1ZW_-uHs z3ofcJ$DH92%J8Tu#sPbr;SgO>;kd~DoZ=Cr>?3rDOge^P3u(T%UEegh*K&rSrtzDu zqXCSA&dG#+>%9xQl=?<4IPm8J$;zb1Jm_`Dpck2@^-ZSE*A3yK*IRs<$(@=bCe|jE zoKE|*cFZq&5@Li_@;r$-7M0}S0RIYD864nQp=P{6Hbk;NuRoKp|5M$^irIAiIHtoA zC#kre6-ZPVA~&?g%zaiZR9NTIw-yn1UpghQmLaK$oq0jp5JZD}!co>kl{4KKBN<6t zgv$bUg9Fs`n;6H<-|XpRTBfDmAV1SNu!Sj!r4+NWX?}i0A0NkcVcE^2_}E*0t0O>? z`&c!az);f9<5({uT;Xt5JSjVTT>Jtc#6i$47@qN^JcGx9@@KAOTH7#l1<$5o0$-zv zdB$_^V+q9|4XZw(uij_#IK?gp-D$M0=G)M__q*?Noo^O3-kfH-lHx14WA9wPT-uyy z9eXgX9QHBM8-raStk_%>X*jb0OZ)3$K+9h<(bjwL~}+{le+MW_eMI{{~8_Dnvg>dHVQ?fSjt&C^cA zE3)`RGZT65x;DUiW&K!*yW}H?zKXOxrjy`&i9XZZMA^f9r{|b}K+(qUU%SNd zaZ2N*%Y^HNK)>P%NWL52XG4j8^(tTfG{`}BoN!!3T-c?JR-;0$@m1WC3 z7LN=P_r1x=fxZS3$I)9O9?>U^iHWuf5{yNGxYdPQMnz?~W2fIW=DS*QFl5B~134Ai zm06vK0x!UGrWpSlO2ZzOhTQgm^xO51oV^?z-JJp%gAF;7a(`=|*AgYrx*}7TOQzM+vr3 z6U5L>p>BCdZJH$Uj~%y?++UixG~?$Yc!VR(21N>9Ki40;$`kAma#Hwvfjr<6c)r383il-v61o_uZr;H+rl|r2NC&p?D|lC^!U-`4?Yip@UIK zmq`7sU~pZ-arHF>4#@L|3Gl4>G){@8>Sz_l_ownN;KmHmg|;uXTA-wtVf3>nX`KK{ z1d^?f6@gwfdNyGhSyztWy!=8ok^ESjg2g4Rra~;uSY0^4e66QE8XDO~1HxP!J|!q(?sl@WD?s>dJcB0K966h`*1J1mR6~J zlP*_e>KyqXJa99T8#S@G!!GT`CS1y87tFNOr6kX8ANwwKg}Gi*JaHhDdHJAWVxt2V zgMU3E#n#FBF8OO$^w7MwTz)hXrbJ*ARYO=#sCPq3vXQ}%u_oh8$^EwrYP$s49wHjX z$fsc@RKFO{0DwIv7kg14Pti8P>aLQtT}70T#lmTZhu^5U#M*0R#%)vHNV9t|OL7}H zHL&8#&7IcULbl4Oh*YCnrB9un*WRKT$D|hixjz|~tUgyayh>2r#cMt^smMC~9>Hcr zc5&(W!p+(V^TiE8KxQfD#|jm#lUfh!ZzoV!v(&x%U#j{9At9o8o{wv)^X*gCZ*3<^c7QxX5% zmVVf=YHXhi$=LW22R*c=^viocB4<(R+ODeVuydlW6|lRLT{}~H*9EpBIgz=9-Y4G> ze3nBwlJM}8!bBNcqzTCKvI!oezIs7G08o>&pHL7^6hnEF=zKhJRiwvF zshUip$iKhg{jNtdhIN_Jl3C=3Epn8T%Ic{oaTx)jQdFUA`m@6j-QAx)e60h|6>vM^ zslz$l4^XW&%oj)G@1>P?>6(=Y5P3Ft$4hS4GmHDki#nFLWhVuT*4Sl?F@ou%OLt5H zrLk3lf|+{z)7hO9o)R~cBGYp#AmlocWKOBzA*7=g-t=`*ypW@)h*L6h$b@|CuU7Sy z&EJ{b*E9L0#XJRQ%~4lUUa9_bO&14B6MWwYcgXcFTaSqy?(D)(P-^CM!)}wBQjBalsA&y)Of{N?m&WEz%B8STFbu9d*m%x_pPDvyz52#2uDrP=m<8pYrW9cx_Z9q*uJ-Qr8dMSw0ufUT-Lz(HF_a8`y>TlrU=1Qd=sX z$?=o3JM>Zc&#n&pJz6bNr)NxjsuC{>%wh9FRP1UUCmh^cZTita3zh!Yxc^zA O{&frdA9X?Sx&H#mw>5>qTkdjooL`tMZkVfflk)crmk#6Y*85D6481B*c zd7k&a_x`@`#+o_%tl4L;eb!og?KOM<&$*tvUImCXRW(!r2m}Bi;0L%~BYC0{=;8zb zIywL^007tk280&C06Fj~pb4S-hra{i24H{cp#T6S`-=x##kv6SzuH@X=O*w_|Lga^ z&REX@01K>v3yFvbr$GM2L!dCA;O}~2A)x}uzk4iz{@o+Qr~vji4|tVe{?!KT>;FaU}9kYQ~!U>TrYz<<6d_IB-nsBPymCl z0#FhNj0AGs4=Do8#`vS)@8SAGpq`*&SlBqYc*Fn{0)s)}Fbq&%uw}?^r^87wNLhpx zG06<bub+QF;G@S;(J@bApT@D|pgmAH90tewZ4U$*03I+BI0lO_CaIzUmYp{lt4IVkxl&4DeGd+s=p7`5 z{ljrwN_Md|j+5Ue{juo(&Y;NuFN^*$=pTEo7XW-11RO9J2_O$#1MEDD30X8 zb7wVT# zB*jjgEEy6ly*c|sowNCNqork7^XNsp(=&5T#f#p0wp{-s1tzj_Rbdr?+_W=T`sxMx zX0RnnZ^P!94dJCcXX%<=tbxVo`i;{}xEA#>yRd>edx}HIS0T9T+;PNRA_WYJy)aBP zQLy?|K`5^?6TFMa#j`rFcz~+jWx+bylU~Q$M|hFmZp$GlfKgRf2H(7Ca^|!>;^xcz(>{{9`ySchR3i{u&Gk z`CsVRFozWsH`k%(g#daql=SM=HQ-S@m!pJ=@MJe!Swz+Bt!U32yVsVAvd2j`6Y<#! znVnh1;9?5fjAdq&?00-E#Y`$()Oz&j;kuM<1b(XePIy^7j(retQ8h7Br zJ{WgA%&8W|Q+qgFgt`V+Ra0CuXBK`wGnN&ta*^;3r>NTDGh?gu=*xHe)@-#+udxwAg9&0N5H?zYRrA@o!W4F-o!`ZH%A&l*eZcI zf#a~(+x*Y_d1|-g7bsHKrM5XpUO>AYB359DCkx^7EjEk!XuMKT9hR#%Xi=i1FA`Td zvgRj2Hj@w5C!3B@>OV{-eG|0Ius#q}CrYOF(K}p@<^WLKW{leAIlBR)Nj&GfdHW^O{$orz&+Cqxskdz%ilyts1`%~|N1GM6ue1qEI zs|yK~&shPArNG6eIcz|WWRLRKY%E`}Uim=r?wtWXjho9Bb5FNiGGLr{?t8wAyn{$5 zLh;-5*P?FRAF@fvl;Z-uml}Mt3waCir0`~F!aeumt7O%>R`$^KnSFT?9yPrD6@)JE zi5rGog6aab$F*lZSLMFRD7KwTVXR9spME*xceVWyf=|2n7S)NXiU-wyg~T*}dCXCs z;o!AIJsmhCewiEA!-{aA%U?`GDqb3ViuZ^5vkW-yRbOe#XgB}I*ySOoviYc_B~=VpFJIJ|H0FAf%%P})|{&2m`1<3}yRqbWaXg(S?$EgvOb+qXhoTZ+mq*wv`CKxJx;U!??X+%RX- z&*^<7dPd#DL(G2kQZ`(Zw+`!j!o{u7(L5?t+680iY(7;e_T}vp|JYST?D(FDx%0># zJe0Y<9T9l+q!~{3$a3A-94h#tzFmro{z~{wSdX;)YSuMSk2vf>rfuB6q<{^|6CYoy zoHVL1vWz!*#`?G55N5xT!HCxyey3v1>66>!jd9~W4YmE)994Iy2+AFd@YX8?nkn#D zt7AgE8h5DA|7J|aln(t&tB~tM_PUSJj@C4q&qHpUGW7``{m<23FNM8p!g^&VUNyKRh*ySGO!WgFR{a!{Vid6Y!WzR4Um>Z^RKd7B zMUQkbUS**<7J`kCOmWTE0AXO1`n27*iTy7(K0JHSjD63E#JZ=+litx>cS>62!Vr!7 zXzhp%ji+)Ucd2{zI+(@H9qS0|f^YxVle7&wJoQMqcdNS|4wNs8*ll6_sg*biHf^=< zjh{;Doe-U=T;<6-g8!oY`G+@ZgV~V}F9l9w`)QgO!Un^L);c7;1B3K4eHZD25;Ta@ zsjeo47q?Dqp620Sb)eb62#^MoUR|EpRjDi%>6gAYsJ9kPGfi=AQN|M<9-CTkX|f8< z$%;a?Nw`;!G0hRC}VlHZ&-R=Pa^k`h~73%fM1AM~k~^r?JD*@1Dm=r`7oj zgIuxx`)_AFcG}|U ztQ$Ls-ny|-7`_EeCz+7Og7IipP{mt;)5Tdz%-I|7z>Wf7i8@~a(Gx%78q5x-CoAVh#|r)9(X^J@ETyx zdikUO*b^g)#@tvjaEVlU16lk243}tl0@0F;l1k{wCqmfC&oh~!WRPhF@ds#H#Cpgz zFu=3f3Ku-17g7{2VR(YEHJ%X^JjZYYl~F)+J&vY8V_dqSVlSjFZIL@eHqoucIwyiL zf@@Pvi^a5>tZfmkY}88>cKcOAYH)i*7-IER7=4&C7{q+Dm*g^idBmW5o?m()x=@}4 zc*zujkwfOwl*8J2!&)}C4p$p6Wjt3T##z;9Q#?4>zRrF()FkAF{fa)J>J?ldLlfG8 z(&d3W84Ky4tvJJzBnc|ce1OdwXXyYQ`LdeIoIQv$WkZq=+rjjpr{RzfMvQ%;5#n@MpW z4p^}_JX3{;_KzAR@9@$(3r_3#!{BUrXy6*p^iF$B9@W$nB@2?#%K4vH#WHZQHr zKuqMBo73{AHVS;{wN)dj4rN6hpz$QZzW`QL{RRI4qJC(QDV{bfQ5J8Vs39RC7EGiiJ?yZ%}Pf|-w z`*iof;eMkdyFS8bQ$~$@?d_bY@xTvHf?$8t{OTy}P+u4EiGO|R-Y>_UvzS-SrH9A? z&w$CXIFjBH#8DZ+v6XRcLa_0@^}tJa$8u}sssS%S(sK^KcFXX^{v!Q=IGE2{=~164 zeXNo%mky}bvTMLI%sDXMC?8E)lvH*6HXG4|fBzv}RqJs}D_f|S+e|2K=pv#;@^G=x z3T1N^jC!+|WgX$5(GxO|Z z_OuV2W#bAAr`G^QD3^60E>TRD9LrP*afLn`rue2ifSwSRhJuE($;`Khwg-%UerS)8&z*G?d~;LZjJ&9v6S^T2T92#^_aOc0qb6s$VLOVg zk+jk(+0d^P{;XoF!IcpMUZ?t4o*P7Q`6Qt%YIa1)en-j{I=sv7e3`$twIVz{+VIGg zRAq>5?Mwk%Zz~mYeXQ$=KO4P&E$;tb1u7BO+&@WnyHYrKe|R7vN%MZn8vxxjZ({=A}oWToX^grVa038Si{n?oR1u}pq5MzMDpdbhVkpI0^ z|M$hex4pzLVi0#2>;CtZe{nc3Jc?_Fw8|N6!tR#3z1G~mU$-pV4V(I=)eis~s^K^f zSuRivm-qh&LUo7qa`^mzvA3)2MxRMkXqOm#uzXpY$85FtGJ-#8QTpTX>s|e{Wv-;h zR|aegvE!Ks=A%pTC41!d%lfTTgp^6B`#Ml=2w z=2r^sv7t0}_8*y1X?XWuEI)BmU2h7Y#bC;{;_xYStH7A39TcEIApwBPHu_mgRzPNQ zn_T2#-XkAU;)4J5pvJG;|5*VJ!wnGb0r4Iv1`Ojb=mZ9bf-ELE88a~nDTM^9h(4u| zn7AzmI6Bzim_ab0*MO)+tn3+%24yNCwf}l2_I6$ykzL&>U9F~`s6#9WSVUQUH#@{O z6>m};>-b9gxJqJ8Hn$z#=*5kj*&H>Rc82XEOk}p+dqj($9oju3)$Nnax99M}$MA`o z{Ia61o5P~hV8!g?_NP@+q7J&%8NxUHvyC2kyeFC$Cg_;)oj5nnXXrpw*AyH^aBq_y z-ks5!Jv02oR@$ZfTE{)VePt(f_oKo{fG3giQp{jsTSg{FzEQYt8hlA}==M_MxL{sh zym@Zkjj)k|#Y5jO?`#U~+r7vZSjkF_Vx=q71k+|~Xg(L!3iq`n$(cNo37)&r=cku2 z*^FiAZT@;@{^Aw-q45E0{ys(H4c_TFJd!fBT=50(-7a{T&2b>;$Vbdf zzc@emlqWtUiiAMy?98XKzDO9wir`svc($vow5!ubwO?4l)gU@YN`dX#xY1l~Cz(@y zcb2{QQE(dj$P}uLK>QQZ&eO6bbWa)Hu~aga*|K@%I-3>e-qyG0y%FRZ^xVdhX7#9R zf9OGdQ56@=HT<)Dj80%a^DM1h(}N9 zpU!7cWRvN${7x7zwDJjl;;5^_g=z|Oa>@d=maqE{@ZrBT$n$;Z3p>UXy@*CcCt zS$}F;4~B{Qvy#;9&{n)vTT*zEVVg@im!=&pDy<9UbFld6-qCaWMF6ya(4eZWuORZ1 zGvz_G>6)P(IiY{*t?KGGa+;#zG%bE=4W$L96~=L8gYK@&PxvbCOeUqwKwKe{fgEj^ z;%1zCP^PWUN5QFe3xyN5zgm3Ki@BxyZ1I?Oue5cx4F4k z!33M&gdT1-#lJdiR-V4p-$=hB{V5D)egEdU7H6HGis>7J8&G*a=M$1vA(IaEN8-jY zNr_cBD1{dZ83!==n@mTNRA)G{M%sOF=apE+E;18ApSja4Mn#v8i;Z&kf=Zh( zOP@Y4Vb)J{E;fhq+; zWa}iKWYW(ob~SHrs~%4@Xi%(%wCfxlz3Jy}Yy+l_ju#5g_LNE0tON|K)v1+UtH<-N!O39z|VLVXEKOUM=7Umw2kYxC1RL%zd$dqnNz%^ z#7KMQ4>8Lqmu0KX&x2cYiy~i2RVWS1nzdw4*zkV21wpjCG2mbN0#w#>crwNuJGom( zYQKs*q-eIn#y`+fdO)M>7rv}ddG|cmjBV(=)mg9H_A01Cs?d2_v3Tc!IVz8_rtC6UX}k$7Gik$jzQ=X|;k3)z>W`Ig_T#(g@oB5X$e zC(tlL(hnfmd5BBLd3G=0X zw={nsE?><%NMmO`^r$pj(w$y~FNHmU`|}ckidKwcnha(=OvlA3+4b|cvctX^%PSSi z-hu3^gmvd(E-!ivdKz=--YP+N@hzi%zQlvl*$^$}xliiv9?5ZN4YVaqe}Y>riO_D_ z-Fbf8HThv=q=B({&?3{*yoRSJ+v-P+$W)q}(@mRL)r}ORWQli9rM)aiuFF$wNj}2M zmM=|?#2s&{%kcLm-l=|JXy-8XddzilVE5?mew&22Px&SGJVi#!Ns-cas&YeOL*1B1 zV<7Gw-$cxhP0wDICpz7|`I0cUdbj0=EH{;<2!k4lLb@?t!%Acv>w_w8s&Ul;I-W3| zQq;Y{#-XxXr5mc|7KdTjWDa9?Zg;0Rw!fLXyRH0^2h%ApF}TP8AJ|}m3HDzJ7eWI3 z&XjiE5rs%%=9K!Las9OuA=|$fB=RiRK(<=aPt^h4hO?1<<8|W6n-~r;m{7UBrVsY) z_ub3hWZl!Q2$j+P^n4>Er5&Y?Wk0Vb9T<#kL8^A1_(%!UI({M%D@NeEL7o?YCRD#TWJFeB_$X| zVa;0*_7{<_&qFRI`dg^AGNiuZ4~FcSe$pT`L6tM=1b<9QOjg^5PB9Cl4VBV4_b4`gj9c60v*Qmme)o8G^J>!eP?d=% z;{m4YNOjQn&~oW$HrdhOJFxlL$Hq&8O1|9zWx?%>B|<@|V?%GEH7yA}_^KC=GKZ);OwCQHAPYEa1!AlHU%B9Z7@F2SFp~*-=PPC}Yjcy>{Ttn!TrlaF z14DkrbDhfQ2^x$0PH~1JlD-$T^a@VM_OCN)lJfoea?`Z$SNDp@m;5!Ii@FMKuTE4u6?h27VYX1&u!^w>l;#ggY-xq#i7o!Jm3UfF5HS$el9Y?;lRlhz)1eJ=5jjQNoG^Q7BG-7fv(-3W4#0X#8GVwH6o>Qm$Q zEt@V`W~w#?HRFAU?;eh|KCs@C2^YXxDptaO#-hdMAkpHg@s;j{Ul{HV6|0d$*6lBE z1u|BeOLlt%FgyI2o!wE1^GTC|RY|&Hhi=yZ6PQWaeQ`!Be zgo^bk0IT8nfFd^#4n8BtuY<4y@j!7heTsq@Yd4^*KpX+S?|>}?0f-|3Ay7uS?~mYr zRS^9*%gEpBOyd93I`dBh?%$ace9z$b8wdZo)8Cns*gGPH8GOF2j})?9`zLRL;Gucs zO5-D8+A%4v0d^`02fwvgbh;alBfckA-Bc)f>Nqx`pSwQ4l!Abxv|0T7hUof-C*;M7 z_*0XaM1i!aD|P)ZlDKk(H&9wP_?-Q$*psA539YbKrGjPjyNzTPqej0}@On%-*i0&g z5ML($P*flMinU#~L)^@VOr(Pzn%|Bv+`2 z2eA^uA55wb;hm0jEV&Gww!+j}y)rv&e_lZ{xqGVAG@ zG%CCAhco%=RZS;i_t!h1YzQCb^uLNN$`QFQY8#bM6h9F0UMO!YexoaLv&rRzct;DU zI4gukV~#yrwmJ>-M4ay=(LZ(2)eu`R=+D2BWpWMoj0+J`TWU4(w3pbM%+wj$`ke7Byy1NCekSVDkn{;jj~}ArAwuhmB>c;+!OP+GsYx!6W@2ijm2mrwmI2p zHB!wH-`m@ifK;nxXz8*;TQrI7T?=*);pr3M)7F*e*ff#89hLlFB1+t%(jAJ;44(0_ zLj$3+n0SE$?6=9iU(RPmy7Wx(vlOS$`Hqpl4`Ghc=UJU33m*>ox0=iSGlR6V)Jpb^N2CCplfPmsCIoybup zfnmL6C|em#5V;&lZ+OdV0&TK$nhjc>K<}Ys%IO@_Vl@SGFr>( zjHiaq@pSNQ>io%?U`_H$+co;moBIB35*eOPi2T;8$QJ|hPYWMAx3#<#xkBSCbNGR?xrLI6p<5v{qYvvajq1;!Y1%(b5{`fWN>!}-F&M%)OX(nKv#8&3U zP-4AirBWn;XHL4xvOIl#jI~+DqM@-&r}vmVMF|f|IiEJ9&8Khb!|KIfU0k)d122!j zC~-tnl#XqUrc%_@3JA`CG{75;Ip z+=X=e@8-n0Y#MP8!egLs*42soxh`E{r=Dv#Getx9S!G2No{Nldd#<^}KHamGXBsBv zUpPG#Y0ZE-p_0CL+!W+Ds#;T(W|`U3sWrJeEB2i4V0*FHdXkPXbllpEi{{;8bXgV8 z)k$V2B5!+G*AHpv>})Pv(+9A*g~$l+ufL#IA`wawchJl>$*|uV=kI*(|Eg>(5=-G~ z@79wa2&WIjb&gj0IEpOa!(fY~yF^PAtOn!Du8#zD*Rn(i$PC6`>vLWMkS$Ib+>YWK zS4sz|DNG-`k?F_%o#h=hwMdQfgneN`k$KYm*u^$y-Ee_b9xF4W3}3$fy^Mf3NF9J3Tnigod0_d=ih!po1I%?S4>0VLh z_CFm1yew-+ik_1SO!(_C$ndDQ`%m7cB|wDrKPu=U?v~*0PH;^K8r&ri+=2vqGg)h|Bk$Ya zJ?H&>w`Y$rYPMHZYgP5j;>#w0E+-``1wcSR0N&sq@Un$DDd}x%1ptbQ03!eZ@Bj=1 zApir0Ai$3t1kpdR8U#H6^(Pz>03fmchQX;KYys%M)9Zo1P2dMy@}K;Fec|2%030|3 znw^uABN5^+7y=RsDEL=62RnNK#DA1n0Qs*HA#Dnv{slu66~X?U23*)52MYo9cL*%F z!hhNg0r?;8hCuu;?S}ZLG)%~UYQbpyyF3~2_cz{uzAuX}D?sRPT7exT*h4`=LP9}9 zfgLmq^lt|P3-iap{_WuYIQZWV;UD)WHzXu9*dxHg!2T2dKL%dbz->l+=>;(00Y{(! z3W5@V#DIXpfOr{%_yBG%%%2ASZuTDsZYQ{5aPSC-Naz401QZk`G!zWDz2KC7zljcw z0fR}wAqI=3Y6?f`ip?36SO`xgUf+kKHhoUbW#$%)fQXBSPe4dROGnSZ$j!sc$1fl# z@mf+!T1Hk*T|-k#TSr&V+``hz+Q!z--NV!CjknL+kkGL3h{&kuq~w&;wDgS3tPdZH zic3n%$}1Wgo0?l%+uA$&2L^|RM@GlSXJ+TV%`Yr2EpKh_?CyQvKlpKYad~z9^XB&M z{uj6x{Dltq{r|!T1B?$OG&B@6+;4m!Am4x=C=6&A3JzFIF;zHIS1d}-Ab4!?#KQVM z1S&4Ia~w0bX+&IV?k$>&-zfdT=zoS#@c$=_{z2#;d|s9TWGDzQ!=Nw#5b*p={~;Rq zpFA&9|BJNK&rg}b3qyH)FF%Ir zkCnbzbK5vg)(uK%dm=XB4QTz?An>Ag3q}IF67XZ5Esr_TVhOyw$ z0G>F~w+JTO(xTO%wvvEwow^_S?Ub>X%?~|~GB;9}rD@U}0V{S9EjJ=(v^RTq&h`)K z-#6A5MSJYflbbM@lY;1hRxDimt5Zk5u1~Ih#MZikHjm}f2KknM2?Nf@cvE%j&fV^ldU$VL`fqf2NQ}8-*$NSVgT9Q%#qd^-+ z$LteP_X`jb0@{pS_B>wp+ZcHPP^;=*No}&M0HZE0TC2gFg7L!iP&ZQ1k27}&F91v& zc#Ii6&#r?S@3hbPcAUdKdoT`BsTyZLP0cs1j=q(KG$=B%&l}A&U)28L2w-au8VP{2 zBs>oWpaj?}qTS$KOb)Vul8yz;^CZVpcIF$+WzHMv<$;-U zUubY;UtWMdwjSFTAX+b=PdRXdz=I9ru2fTZeNrjp1Y?Z7C2T+4HVxZ2iGFExa!vh* zC|N1C_IN-;?a9Cct^dk0Y2&u?5@vUa_O6jW9^Bwx|u1`5Z2!OH%Tri50)J zkI;t67VQVM$?7M=0xZauu7*Al&s?_^Jvc`>ojGL*@G#Y;4hyCD4qM*WE%q1LPiZZj zfafRZrHeNsplFvz{X066*f)A9;=)Gj-<8R^rva*5crEA#Qv=squ~YjkvO5p56XWF# z@O0RF_Pk*HqcUWjt^AS_V?dYOxSEpcJX@BkpFw*o=qvFo&9-mt ztBEu&hVs~70}bV{I}a^qHq8WjM`vvJ^D<20P`-ye{k9Ogvy+7;(}_+!+2?(Dh}?9 z^;pB{>Rj;R;pg?!Wqfdt=nG)EvG{IBE96{a7}N-Nf`trUJsM+0RF4feZ9NS%<3N3k12w5s= zL@Yz<9)<$WQWeSmvl=z`)gt!qroL;CRK z>x|gDhiS6yBTlx(^*w`z?NRZbJ8RL5Zn`Je@)HfiE_hAz_zju?9c z^v!IXO9w2&-2`5M+1}$kxkxwgwxsz2c1bE|`xP5nINs$7{L1m011iSd!OoLnrpdHv zw2tr|;{^96tf55T@xFMy5w!wax{wiED(&9(KtbPF7m(2>)ER(hmz&!t?tL9}$% zQaEw0BKMNka%8m$IBX~$e0|qOe zj}32PlssVTQ=OkaJ)9&iKtvJ`7Kct2if@z5ItNcpk%aL*7r2TpEZ?egPxg*gmx>(5 zK-oev(*`WPk>XUoE&6-+Vq`yFZG)DRR&E5r$U81Qe=}}wI)m8{dBiX+_YSh4u*z1~ zrw>g#%D;CMUb=iA;1?D^kl&*Hh`zs*|A=Y*jA(q`c?0gh!3K{cU3UZL>+>=-HIHmU zP^MIX z`Rt(G(NFdv70o^dIB(rG2`f5!`_UP<^9ORH6Y{YTC(R;RxCEi{)$3qkYC+~-((-c)Q`;*+l z0j25#Cn#!AV2d8=2<=aQkbOXVUNWxJ4i9k1Utv8mxKNIKrV?IzMAtoJvxdp-(pFn3 zvx@aeR;GGABXfF36dVSP8Q=%9&cEZ+enyCRK=M_JxhcAX+L|z17+S}#pHs@`Ijwc~ zM0WJFgZKjxAEc{E&ASNHzNE~02H&ag!GIUc9DG}_7we0=Rf=c zG_DPJn;YKEZrj`Nkl}o$TiTwzE-PS{$Hqc6OH=1Nvl=!FBWG0*$-)Vxf1MaBl z9*Isa&~75Zg?K;uK4pTOUjX1R0OjYJ_Fzj^e(+B}r6sk~;a}@BG81!1Io+MOGW!al zsr>5gn&~ICcYOO=;GDhBy&xQ?p3N-dmWBQREc+frhgv5VA-)|ZZ^eo5_ZO!ol^!x^ z#RfqgY}-MLtW#B=@!?)(pRa`6w@{cy)-mRMacW?B-xhbJ&D>j5!X&z;QoSE26OOFi zw;m#^OFwZgxQo7(p5Z<^QJY!Ul40HM5>1vu8{OpYjg7Uy=-cioW@P$ljN)tj<<59n z8U}xyt9gfdx#O_vTc_Vz!ho;&ITHPjvbm6->`X3GF{2~x?7azuCD;new z@C)xJ+apR0c*0$n?r9)<#Cq6f=^1)ZCVX_BR?087iDlghe-ka?=~2poOvN`IvG4Gc z9QAvTcUkR}`$jPz`i_gHi)yJ-zTjRLcCzfvZKrKZF4}mT`GMkqfS0*6M;>LfvBg}9 z)#mo{@((fFJwLCvC}Lrdxso`KGXdsd*s7v1|K=fknfqUyZNL35(k_7oS}TAA0G84G z_FzE)SWW-~iwO5yK7)dQ2#<({f{u=cf`*ELg^!JaiHC`bhC_^lM?gSCM1+n_LPkPJ zhEGUD_`47YXc(|~2NoF)4w(=G4TJE1?Oyu90vm`TNSpsGumOm{@}<8d9Dj>z001nj zfq{aC1dDS3BCycoPwYRD0J!~7kPvYHk|Ur)p@RiEJ-z>t;Q!YUYCx@!ceC@=Y2!_V zRa&R%ZI;Bl_U#VEw8~|@PH|naC)W;J>w#NS4~naUe?zEJ2SzGI#ogG@4~L+El*xg zGMsPy<@BeJz6 zsd#kaWj*m@xwA4$UL;Qn$ymX6UP3#s#_3t1A!%M%)#L6q<#tsf-O}o;@pJ3w@}B?7 zEJ$b0YaST06|Hpx3wuUG1ME!Be{1K$Sfj1P1weSy#XbrEb1+mx${; zim%;gWkH(;5J7#ITYZ@OeFLma^|hBP_u81;n#BE66(S>i-ur->!$QS{xvw4c_vfc| zBhtlB1ap?)Emd9LP)@_h zxWeeLyz#>$^V?hP>_;S``0hzTJO%US0tzhGKnpCyw$;VTdR&pX zdTCpT9BzL;idTFLC$kW%(~ArxZeKoU*yaqixdlxw>d<*(7wb}#jV;5|1w``SKKnPT z(xkxUv3I$nFM2372(%@8O%1`dxa@t^Ye#e2nt7Pw?=;Id0W1wQQ{^g8?wScQcxa7? zwhVMO5w}O~EeDIRjK5YgAS+D1AN(RgCn8c`G-=!Ufk)9@vzrLnuW+`~2|2z4O^8AX z{!$c@Cicy`5hGdWR*!sb4H+-`LfdiEPfHZvR4MKCJD0r!L?p)~Lh#~VrI6cDUz0?^ zL(;CAEzx;Cit1$ZO~@R1od-^6qelplUUk*@m3LS9*t#~0aZ)pyGe(OI+Sg3>H+DXn zX^`ExnB9cqAF?E(^If+l#=7x?!}oCbB|L?8VReNt~eWa!SV9-0_{{WJH) z)z#fkV{6}P8J1Fsk5ADk>;Q20+=rg>0EqCf_WI`Q@C$DdBtuP3y?!UGI=J_FB)>ST5PMDZ=k(B_PXr_ zm~vAiiXsU@UH?)Q8Gfl%I7JtUV!dL)u@O2w_2%j2@k|-RiP=_K9|?kOk`wDF}n0GK=lNZ90JIk zZGp<5Wxy>Nn~9SO*A6!wV{HWUv&9vqT8UdSUtx9vS;p?HIW|cqRvZhGeB@c0Wik;h z9UJwR=OuqlYUUb$v-bPw{qIcq8I_*EPz9YDN%K%J?Tm zP-;nloXMJW#?v0Xun2PE1LikO>i8Od1S`lSQ-sP|DR@c-)FC{St+%TLAZyi85%oP? zT@l%MhD2OKmvwbt2=>W2hupA>QHEFBxq6UlU2A&c-3o^%`fAiSuF%1K(z70jKA9aL zT=`_4Y@lwZIbUm(r4JT)6g)q{Ji3Q3!XZb&jmyb7b#CkAhSTPHGPiQdmB^eVi2l+| z`ejy6u{ZWMDZ5;PLJwMLvVjCWOj zVlES9FkT}j%(@WKRp&e#7S~B@P}+ezKaOXmtmvn<1ElskY*bjV(JDTTi%Duk`~vdK}{~mGx{tw^9sh+M%^xqWfU_)$svW&Z+F-vbOAw8&Z!yp$EDK-EP}z` znKdBl!|LbAHcBGv=twwCuhd!WEAG;0Ovg?VCr(*WZ#1imBYQ8AFWzb6HH$&6@}0e- z9sF+OQ87~L=fpU#871CDU|2uav9iE{wjDhWn0vx@1Y2)s^@cd@YpYoLksm}VetPw& z#jpH1(r1o+T6UaP0v};-wI*9`M!QG#dvjw=cJn*blWc;Ek4#u7-w}U8gZeyYcgXac zGrF(MQ)e%Jq^PfyfqJAAJG81D?J)*hL^7ceU$6I;v|>zB;C6-TU1HdeN|qEgJc%r< zfBsS)VawN{A!u1L%mL?v*CN!P%C^9O_FXan8$T%&BSkUtqqTxdXp62vL9j>ZqyBqxL*wRxz1h6c;%735_5@F{Smk)eBeJ++Bt1%F|hqs_nj%3e1&9E{7#_QR!<8&@#v`#WEt8tMrGq( zP=o*2W@pTPShjE(z@7bckDpD5U9^T+mY9=GZ2>#fTe{xlMdGUbDp2YnVgBtaFRE50 zX|F>Cr^S@&b>TVVwSxC%ReT9iMj|v;1wt0feGQ<(aldORi$0n(%9J&`5$GcFBtOy2 z&KKZ4(p=7FPTTeJ+)Qt(Y+`2btKNJSxi9!NPHye(hfS{R{ zf4(1d^V+_&@Mn)hzt#e2O}6B8hhK_}MJP>tR?3<-yh#6$);m1cgaySt-%aogVS~5r zMKAE7&RX2VqtjS!VA5a?wGgjUVIG#~t;8!1*Bw6>@r6(&11W3iSE2I6l`HbUJmwdE z35MA(p1>Y7q9}5Q6P?g9aS?3yRR~#`@hqFD?h+>?AVr^97P8XCM0AT2p}Sc{21^>~@KE52 zzx#p$1mD%!h4PZ`PGCSsx7FE2p+quI5TM6_!R@}v7}y$SLNFMY5F37LOSz@kgNp*r z>T2?BC`3aiR@};F)WTt%o2K~Y*;c35b0Y?}?ES}H`>wl%IJ&>O?Vr9o6EV0p`V1;a zj_NS~FTt;u>!08Qo1d};a2gZ!|LaR4ob_u!Z7X=~WW@j=puv_Ba8NM+(ozGz)$SOW zSR88DI24p(Duw8xffA~&LG|Z*^6mu~62j$gs^T3^Kgl=l88qckteh&d zV(${J?B4xhct$5z?x1fA?On;9b(zM9>t#85uEk*dGyADZ5M{~pEm}`AP>*4E%-exy z+BnsTk{4#w-A03ktP|&W(2OcRp_-!?B5{w{g;Fzta)Z>8t6JFCUh_+sjOkN?^!VAw z>C1R-KZ47~36t|qE{12}aGez{B$C6DN^Uk57JW63)Mh%_-yam+5^o#3c;3O@;d2l} z511P)f4BiY=|lguj{zT$|KbJ~2PTe~8a9PWVZEqo;4~Fy&|k~|8*}_-hQ0hMAvQjb zmK1adXFa2bou~oPcdsiZgn<=@^v_S?d)V)UCr@_zqy8|@VJCu`SyCazjyc;m(5*H znD+E6EJSnDsk7s_IJ0ip=rmee-u5(|CRG1hjOeVhkr@ft{H%z>Yi;;Ehrt=;&qsLly$FxNBgEh{jBr`^5G2Nm~gJ4I+NCUbAaF zZPvY2(~al7z|6a%ib>ex12azsybXnff`UPUhll-ZANrS(3OWS`CMBnsDhw7m6}Pyl zYhvFtHW!auP*U<1yM&rYNa4o@YIPH{;QE4d9P^70e{WG?LEtTFjF)(j>qcG1Zb0Q_ zy!5XgdSo9Cr^5x-7H6@BtE?egnSUaF1HmbpsIe-w2Yh7i(D94<{d--CmFlJX4qnf-JzLH7oZ)?_} zmIeaxM)5rdI1D1L+Eg`+4hS+8Q3dGX?Tb|>>~3hpdt(>P%pZ;DOoD0V>P|l>v8&Zf zLWwiO(1w~KxaYCz{CEze5wXRioaNq}kQ_YkS6!EDSr)yarp-H@Ldvu4-`T$nQ;&gHqJ1vbUNi6j^SV^1>VfiwKLw3(kpC zSC@i6D|}DNwF^gHU0ye-g2U`iYi6QMu`DOBQyU66#V8sXh@%xDw&ghAkxhueJA*sl zPQJ8JS)M7s$KRkf94|>yV%Xvz-ur^9W3&5>c&7$^Ct2PP*r)Kuw=)gVT2oTYFoA&T2iE@w)oO8v|y{yhiO7)2{ zme&U9D!9TQ##eX8A6NV?z}W&*oEcSnVGF}1sNVU>zziPqI-}%qtiHvvBXGtc9hFdA zO^Fswb=YT9u9(u|FFDgzmW-Egso_9x$(hzEVoB#ta$HaIp07C~rXv#lh-%&Ztzm%r z$Bt#wTh(5r{e~Kjht&hgJe7k=%%wHEZx*|~eIpvunof1>jl<(P2^ZfSzAV=}3Uo7m zL)81KmSU%v&ttC!U)3dn+2k+OI@@ifW7hHYrjCUNL;1-i-iqvP^faFmj)IFM^A6iiu^noD!1zBH7uwMYSeX|^wX$Bnj*7UEv4!HLcA;J@+K}enx*pd1*w)FG8 zn2d`3m>VFRybJ-jIK#1vP{H*i1&aA=4JJs-`keK2%`#8HrtoP5JruP<)jm}lEi+?b zl2?+k6Da65_057?th{dT3pEm}bau{i(pHqzVuiVHSc$^>%0%U0)lzJLaLeaL+1X}Q z!Dkp0#@x5n_RQ0zcg45DJnMYHF^jUfLQod$;j1V#ly*Wl`3^#yYd+IrHj3sCGge| zZjc_TRwt(9oZaaKmfZrFXucy8xfhEJ0^SeCOlO(Drb5s*J_Za4@}Ip^YB~Q zM7Ktgqd5no?_E_3T+2uE%(+yUknc2@4F-bC(-hT@NXfv&87AuHygYz+B>iF*j3wM5 z^Gt`k0w&0&bIt%nxtaEBB%9oh3N#u1iI*$WyQ?bLF(bB(8h?2urLrv zhGGz@yBAFr+o6pE4Vs(Vt9(orGV5seSLGirSyv&^jN&RE=+1ZkI$TVR*LC4zX9&Tc z55ISKi?v1EZM5x6m0r239P0M-UDSCr!OsWG#5I_av{GU%M%8lm6yBubvT1VrC?E3@ z80{k>#56axL^TCrj-PK?&`#)7lPii2VMgvlMgmLj z7mKd@5V5I92}z{+u0q>lW(rjyEg`R0KULa%9BOwxXk>wkaRXb6K^JIb2ZlCr?30v6 z^fd8DWtDj_8OA!9bjiP?N+OB*JwL22)cm5`{>K0%@tg{w@&RXP;|RTKUBvo%xEA?`;vr_j_oYUTyd;*Ky{!>QdsT zSW7m|RvqyQ9$xFqs@{mN8YRtXmP33t3u;LoAj>vvxpEXEX1h`v2 zEm2q7NZkYsAK0IQ)#-MP?kVk78XuXLs9QuvsMssvP!Tkcckz7S;$&kZUo=f4Tdu8O zHZl-_?<-&4ij`E@y_nkKrj@LbSV53U=2)@w)8=}rr*Mf*s3kiNn~`d z=R*Sge|rUdrZ+SdwS4AtuS0O49^Tu!c<3aagqMImZ$~as3oso&j@Y58me%d zz>%^qD^>8B-}%}~>e+TxJ(BArbmWo>_a2_7I@9(lZe6fiKC4b!1BUk8XEw`05 zY1m@-tPdaZ@ViOR#!$2R?gYo=MrvE^0-YV!=)%F2G7pK%njhQ^d2JAU$$?R}Ms67S zNA~sh_;uUbYa153t(U7vCuH?-6)lys0rGs4cm>m;?nkH(_o1zf+N)F!dFjX z&{%DiN_sn}veUN)RaQ=Np5($zn_DslU9nBG0=I_C=5Z^GXZ8#3v>uByX4Dg3G}FO? zo;V$=-T?WvE&L2d5tWS8`#1_hi5A2h9l)bM5*vDX{r>py7vBgVnaJ8~Z8WjzOzlxY1=N^;zn=i`)j@>Mbo5m-TE{WGb7 zMcI*pJJZKE;>nc#_=qFa<_H?smCogqfo0MkmyQEdfRva(VxWCVD^I7BLHzFeo$&jj z(YkQR`Z4@|O<~5BLz#j<)lj`nwqLnsVA(MrMGQl|=?r(!uN2sAJf-_FR`SO$cGRp|?P~%zLjhSW2EvbD z1Wu(h-B%Mh_uEM=C*6cT=k5BW3y)s0zDxX0d`td{ogNEpWs>mwpvUz4tN;xS_eV+k zN5~BUV1N$_VydRDLFB)!zRxeF*-f?z{*hRNC|-cHP4_lmh$vo#;WTr36EQ)0$55iX z2rT_%j^d1&Ra-ME(!zRJuw42qByh+i4<9;84j_RcGDvh1$NCgNtPH&RGM%UYk@^9a z>K7LG?7=7 zuP9~QYP?lMIkaK2Km)IJdiSMvQ4kq2Y?Yrb$caa($h^AVL>ZWdyo(IdsB{pETwKLp4sg28{kS%>kO;OQO{VI1$Z&(kJ(3RHLXcCVhn7OhXc_z9 z>V;qeiP@KPq_{r$LB`+~FzA&y*|?MqPY z9P)Dh3a?5ON<$ZHA$$mHsP(D>AG#=xUf+GsL*-wQHqiLnqcZ72UOF335gk<4lGM5gh@2m%yKjQPWUDACZ$ zq;S!|MGN_RZJku5lKX~;FG|Y|YLRbfJF4tt)Q6oVJ^#k2!ki-7fGn3Ui_8Sj;?nG> zvJxMd@+MGBf3i%nIFuUDD5s;TqERFgd4-W=y}Z`} zavDpMi%qtE0T_}Q_8?81puxuIZCkZABd~|?{N)a(Q(?@8^vSSj(X8??$=y=sD2*?uqlBiRQi)Z;?yL_fhStE0`3(fJ~7)6bSEsTEHC=8L{!60ZDLJ7 zl(kLvOoPXft*{zpu_`*BckY+SR#_1xg>b}hYZ>k6)HyQc&cg6Mqo={UbvmeW$q-Nn z`0SFt8|7&8Q*R8H%pcJKbU2r{qsUqCUS{4l%Jr70s{SvAS$8!s*eXFd=n#>X zuZbmd$m2L^U(4tR56QSJCvMH}MJi;f>$|!0ZXcCwUcSaOr_3)1A9@?gTo|wNIz*dz zHW}`M0NOS2^<~c*($wq;DIRNtT zt)5$ml4ls7lrHxoR$}aU?{S2FJjNHh276KHey0NNjnQ}u4C3pM0y_5Eu2Gs`NK0FyvYRi7NcW z{(ekTr_hcFO_70%s99&s#*^~3JNZNJW>^+J2~NUS@Xmn%H?QV?nA6@tx@QVbqtV!}d_iWJ*Vy+EQiWA$o3H>^O7T~7 z4_&x6ELh|alkX~hpWdPx3>0IS?2+)Xf0r+<$W9Dj+^NsQ$Og-#5Z1?`zK}FLZd!{{ z!pFh^$wl2~{kJ`c&7yU3&xetC9z_odFr$6Eh^4pXjs@@IO=#sz#OErpvgS7WoX#maAq_jL${YYGpYkgf9{vHg%?QaV&Q(;9 zXkH<=U|NW`v5^2<0@@4%{tYbV<2-$A2w&h&s3Ow*6ue`%iZY!HwCtckc5Nyt7}@4c zy<#)M?L&u16Lt+!j2KLI4@6FKQPIW_b0}Wm`tx^K7yC2;Vlh2fwO#^Cj$Ua>mW63! zNk5!x?P+5v`rz}BSZHJ{TJeq&!z|+==lV>)hV(^I0{C_DK*R@2Td>mOHAH6b@&`(| zJeo-Fz=wK%WJEdvg9FE*QpOYVK)hLUKXiydd=~^p&ajFH3lHAy*;KDUQGB)*vHnX7 zv8iq;g&Dtw2g{q!C+RB;feo&B>v&<>vurKK4xg435XCa5&@89)jh)xkbrq2 zEl6MbtJuHlJ5H`L&nwz_0YtzizJFiX`g5-kY-{y@-ERs&Uxo2D1pXUZ;L*O4=lnO3 z5crU>t`*1tx=A3qc`&ev2#ARTreF(ifI3K#y#P(z{*@RwYmgW= zFzCSm#|Sc%i6aL9O>Y*&-y2CE2+bXg%HAsuK!{=kFhVcDuZYWDQ2-t~B$^p)fkFYk z)*1-Z^#Uql=V(ae=;Jr9_D{iXl+IJ4VQ#!qMA*vEskN%JL3;nUzMGArht2gi) z5;lDXf?EYV5}s&F?{RR{wy4c0d_3AzD=947WDHYYZVl-H5u8;5 zDf62dgV-i=X4|+M{(?eZ?tz+k&Zj%a~K@E+ipZ}JG->G z84^Z7kT)^;gnCiq7zP}@*Q+L`3LjoXqpeSL>Nv@Pr~{aRA)#OS#j+xdd>)}kkWAGJ z#8?9RUVpLI>}pjih$Ue3+{@xThdZfYh@5K*d_N^C8Ot=?)FN&K<&!dZh2)F+>FG{~ z_cTu-5(;8G{DMlWm-|6H76^RA`so@Kglfw_kSCkhWvDUb>cK&chsWsgMG&L}hmeBb zdc6DmsBnNn?-d?dCIS1num^K5_R6Kmg)&##f>-IaiTzFJq=A8kkH&e{lSIuyC_8ug zkF((@nY~>VcCU+HT0;J*31mh194VVN79U8s%q3>8rWoL{Vh9uHW4@1-i2!lEaOiJ_ z3w>UtC%5~&O0L_nT!S2jcZepXcD?XJ7sF#eY40qQ3xpZ5E6sEFE>PfO>=P|qkCDG{&Oc!CS)S%wyOP3DI9mE66;?T)vY6`dZG1x!}u4=|3- zSK{1o?N||EQctf4Nb}E?SzDyohZ_j@Nkx`=969O5&41Lr4YJl90AElf&OfJ~&U`cu z*Q9pRma93DOQ@+~WQQ@8NjOI)VdOX=b-Y87E0*!q$CS8dfOKSgzrClwy#N-bNe?Z6 z+AmyeNLe;HfU^OcXZTa@Dw)&Cjd-ReWY-p|PZ;GAKTyLB z!VVus8g{?)1NesjwT1;}BuqcNp7-!iG*r>Z1b~9XCI{*8;f^X=^3frV(oI=hG*98} z_eFizf~HaNLJpG+O^~KSd}7BF)8O;bzx)9)A2h>|13?})2F@Z z)e+Ns?Xe7m?zmO0#)Tt%l0x(FO2M&=Bp3kgx>K$LkJN*WZ%r#`osv*tP{^|?!Mut;|Tqs zG9|I-hALZ*#SfK3rawCagYlY5*@K!3X8)6LQ}=FAe!YItTUclz9sUz($&Z8^nA93drOAT{H4_M z%RADz_wG&+lgb(kTkOYSU7vzfVjS;!LW2CUTkI}qH3f{-sDg64Q7**9EjVdWp_uCB zA1v!#yb6x*gOLFOJzxb^{kvhu*D!+gV+1=h{z#mN!g7amz}WJ!!Y^=u$#lY`d`%lF z&<%+Y&lDml7D7zCU+{DoH5IjM+u6C-s=${nf^e644wo4B$n%TETg}~V>z*|~SUDO{~<0J@+}I6-#$u0W0_h=3pG4(A_?R>Aq9cAU5Gx@8zFUWfggR~QK^xNqBDGL4*eQ3(hx9ooj*y!*_k)p|dS&&`JrOC= zTUFJeFeuiI#5XqhX1*o8uv#&=r4blcB%Tg(Lh`r!s_(n8LCo|Q*gzmJh;%!2RuW2GLaM6$H^&3ztkBt$=dHIt z#U>$UeD~UE6T>NeKaw0AWVx&*#&(|d&d4WWh%yMb9%eCZ4GzXgB-0soMJ~ryAV(#Z zL4GfJGhk^#Y)u^>I5$(PaC}2LWGFQ!%X*v3G$T$5QoMNqAcUp*ix2NQkODv>DllRO zBc%OrY(t1B%Unu4@1GNSVw~(I|Ac?ZH@Aveh!) z7XpIPRt~b9G$<+d8QP`FEA${8-atUzop_r7cN>(-p z&XoN``;jR%o7vpRS>NT23rxPIL038yzP`~x?7idW;gV+F2t(3n9-u!UBwsD3riWR3 z;$_yUbA;;5>nnFMe-uM(7CZ|rgGjY|OmLI*WkQvTyhIHuR_av?;)E$_J6`=Y#RTIV zrX>4wO{*xkjCECLj*c^~3U6tIUfi?jnrM$xgM}5=r4yoR8#G1MjcfD00uF4{PqaIR zx;ft}i|ysx3;SAuMjBlO*ueez=!c-T)pD^kFrVAcBBM*DgU0l(rA4V&0;Mgeas*xR z#>;#~^>dyJ_dCH?A-B38%^pM8aey$aH}hQbn=Sb#Bvpjqy}CA}9fJs32L^V-!&P;w)Sj0bh7e)_}>XmA=%z!O{-atc3{zXa89pZkg{MRm7o*tk?Heh_JKl~$-p+K#> z)G3+3MxcFOL(tfL88;1`DO&%Go{mlbAPA|YVW5EEcw}TlCTI*w?j}Z9{K8muRPaU? zM?crgfg@p1<|=w8_P$;-U-c+UCLUna>|e zUvD;o-Ae%-7Q_}Hd7%Q88;#PTwXM$~GevGnC?M=aR_BtM082E(Ke^S?Q2&XePay6n zit|kYg7Hk{FU%vj6KCF%KUn$vfMo3i(J3DeJCu4x~A-z+f=EZwND*PCP4(V z5fXk7ep2dl*UlToY`uyDm)Fqpd);uZHiKFf=)69nxi+A(Ow~YC7C|vwynyZ^`29V) z&1rVotElv|!|d_zU-6~=T_sfdw#|cDk#cPrgkGDH(lA{%ww$uI=lt*|j_T${O7c$& za6T`gef`S3ckd^oOM&tt!mCuUdFf{S>(Pv&wUl-PGOop%rnRaZrN4;Ojj;DEEV;tj`B+fhLAGs-dEVnWy%Kayna$g$pGpx2}w}7LT8&{hmTKzH=vdp(d$k$F(GV~1vldo}h zjXm&j4CzvbuH~e0_x%+>-vIyxbt9=81zi*dtTGAWD95nA5ifowhf+bbQ5fBD!#&Ob j-{uta>xU+a^FjOOJ^9}~IM7)sX7QuR3$VZOvh@D|{nB3i diff --git a/README.md b/README.md index cdff6f4b..10536406 100644 --- a/README.md +++ b/README.md @@ -21,45 +21,7 @@ ## Sponsors / 赞助商 - - - - - - - -
- - VMRack - -
- VMRack 洛杉矶 VPS｜CN2 GIA + AS9929 + CMIN2 三网精品线路，AMD + NVMe 高性能！ -

- 👉 vmrack - 		- - 99CDN - -
- 🚀 99CDN｜自建 CDN，不止是加速！ -
- 边缘缓存 + 智能调度 + 自动故障切换 + 商业化运营 -

- 🌐 99cdn - 		- - AxisNow - -
- AxisNow｜自建私有部署 CDN｜订阅式高防 CDN -
- 自主可控、灵活组合的 CDN 网络 -

- 🌐 AxisNow - 		- 🤝

- 期待您的支持 -
+> 🤝 Sponsorship slot available / 赞助位待认领 ## User Guide From 9881c47f0cc92256ba203919da5a25fab79a6bad Mon Sep 17 00:00:00 2001 From: naiba Date: Sun, 4 Oct 2026 20:03:52 +0800 Subject: [PATCH 41/48] chore: upgrade frontend --- service/singleton/frontend-templates.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/singleton/frontend-templates.yaml b/service/singleton/frontend-templates.yaml index c9bbd8ec..6b83d827 100644 --- a/service/singleton/frontend-templates.yaml +++ b/service/singleton/frontend-templates.yaml @@ -2,7 +2,7 @@ name: "OfficialAdmin" repository: "https://github.com/nezhahq/admin-frontend" author: "nezhahq" - version: "v2.3.7" + version: "v2.3.8" is_admin: true is_official: true - path: "user-dist" From 8721a15b1290eba7451e12236ce239e8e48bb576 Mon Sep 17 00:00:00 2001 From: Sam5440 <47051833+Sam5440@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:25:13 +0800 Subject: [PATCH 42/48] feat(server): online session history with configurable stats window (#1243) Add persisted online sessions, configurable retention and online history statistics with lifecycle, recovery, timezone and concurrency regression coverage. Co-authored-by: naiba/CloudCode --- cmd/dashboard/controller/controller.go | 1 + cmd/dashboard/controller/online_history.go | 121 ++++++ .../controller/online_history_test.go | 356 ++++++++++++++++++ cmd/dashboard/controller/setting.go | 30 +- .../controller/setting_single_commit_test.go | 49 +++ cmd/dashboard/http_header_timeout.go | 14 + cmd/dashboard/http_header_timeout_test.go | 107 ++++++ cmd/dashboard/main.go | 19 +- model/config.go | 11 + model/online_history.go | 211 +++++++++++ model/online_history_test.go | 220 +++++++++++ model/server.go | 15 +- model/server_runtime_ownership_test.go | 27 ++ model/setting_api.go | 2 + service/rpc/nezha.go | 13 +- service/singleton/config.go | 39 ++ service/singleton/online_history.go | 348 +++++++++++++++++ .../singleton/online_history_failure_test.go | 224 +++++++++++ .../singleton/online_history_restore_test.go | 73 ++++ .../online_history_retention_test.go | 63 ++++ service/singleton/online_history_test.go | 316 ++++++++++++++++ service/singleton/singleton.go | 3 +- 22 files changed, 2242 insertions(+), 20 deletions(-) create mode 100644 cmd/dashboard/controller/online_history.go create mode 100644 cmd/dashboard/controller/online_history_test.go create mode 100644 cmd/dashboard/controller/setting_single_commit_test.go create mode 100644 cmd/dashboard/http_header_timeout.go create mode 100644 cmd/dashboard/http_header_timeout_test.go create mode 100644 model/online_history.go create mode 100644 model/online_history_test.go create mode 100644 service/singleton/online_history.go create mode 100644 service/singleton/online_history_failure_test.go create mode 100644 service/singleton/online_history_restore_test.go create mode 100644 service/singleton/online_history_retention_test.go create mode 100644 service/singleton/online_history_test.go diff --git a/cmd/dashboard/controller/controller.go b/cmd/dashboard/controller/controller.go index a3dd50b0..1668e192 100644 --- a/cmd/dashboard/controller/controller.go +++ b/cmd/dashboard/controller/controller.go @@ -100,6 +100,7 @@ func routers(r *gin.Engine, frontendDist fs.FS) { optionalAuth.GET("/service/:id/history", restScopeMiddleware(model.ScopeServiceRead), commonHandler(getServiceHistory)) optionalAuth.GET("/server/:id/service", restScopeMiddleware(model.ScopeServiceRead), commonHandler(listServerServices)) optionalAuth.GET("/server/:id/metrics", restScopeMiddleware(model.ScopeServerRead), commonHandler(getServerMetrics)) + optionalAuth.GET("/server/:id/online-history", restScopeMiddleware(model.ScopeServerRead), commonHandler(getServerOnlineHistory)) // CSRF middleware applies group-wide. Safe methods short-circuit and // PAT bearer requests bypass — so the only callers gated are diff --git a/cmd/dashboard/controller/online_history.go b/cmd/dashboard/controller/online_history.go new file mode 100644 index 00000000..302f1029 --- /dev/null +++ b/cmd/dashboard/controller/online_history.go @@ -0,0 +1,121 @@ +package controller + +import ( + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/service/singleton" +) + +// Get server online history +// @Summary Get server online history +// @Security BearerAuth +// @Schemes +// @Description Get online periods and aggregated uptime statistics of a specific server +// @Tags common +// @Param id path uint true "Server ID" +// @Param days query int false "Query window in days (default: configured online_stats_days, clamped to it)" +// @Produce json +// @Success 200 {object} model.CommonResponse[model.OnlineHistoryResponse] +// @Router /server/{id}/online-history [get] +func getServerOnlineHistory(c *gin.Context) (*model.OnlineHistoryResponse, error) { + idStr := c.Param("id") + serverID, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + return nil, err + } + + server, ok := singleton.ServerShared.Get(serverID) + if !ok || server == nil { + return nil, singleton.Localizer.ErrorT("server not found") + } + if !userCanViewServer(c, server) { + return nil, singleton.Localizer.ErrorT("unauthorized") + } + _, isMember := c.Get(model.CtxKeyAuthorizedUser) + + maxDays := singleton.Conf.OnlineHistoryRetentionDays() + if maxDays < model.MinOnlineStatsDays || maxDays > model.MaxOnlineStatsDays { + maxDays = model.DefaultOnlineStatsDays + } + days := maxDays + if daysStr := c.Query("days"); daysStr != "" { + d, err := strconv.Atoi(daysStr) + if err != nil || d < 1 { + return nil, singleton.Localizer.ErrorT("invalid days parameter") + } + // 查询窗口收敛到后台设置的统计范围内 + if d > maxDays { + d = maxDays + } + days = d + } + + if !isMember && days > 1 { + return nil, singleton.Localizer.ErrorT("unauthorized: only 1d data available for guests") + } + + now := time.Now() + windowEnd := now + windowStart := model.OnlineHistoryWindowStart(now, days, singleton.Loc) + + sessions, err := singleton.OnlineHistoryShared.Sessions(serverID, windowStart, windowEnd) + if err != nil { + return nil, newGormError("%v", err) + } + + periods := model.ClipOnlineSessions(sessions, windowStart, windowEnd) + daily := model.DailyOnlineBreakdown(periods, windowStart, windowEnd, singleton.Loc) + // 总时长取各日之和，与 daily[] 严格对账（每日累计原始时长后统一取整， + // 口径见 DailyOnlineBreakdown） + var totalSeconds uint64 + for _, d := range daily { + totalSeconds += d.OnlineSeconds + } + windowSeconds := uint64(windowEnd.Sub(windowStart).Seconds()) + onlineRate := 0.0 + if windowSeconds > 0 { + onlineRate = float64(totalSeconds) / float64(windowSeconds) * 100 + } + + runtime := server.RuntimeSnapshot() + currentlyOnline := !runtime.LastActive.IsZero() && now.Sub(runtime.LastActive) <= model.OnlineActiveWindow + var lastActive *time.Time + if !runtime.LastActive.IsZero() { + lastActiveUTC := runtime.LastActive.UTC() + lastActive = &lastActiveUTC + } + + // 响应统一以 UTC 序列化，避免窗口时间与库中读回的会话时间混用 + // 本地偏移与 Z 两种写法（时段为值拷贝，不影响调用方） + utcPeriods := make([]model.OnlinePeriod, len(periods)) + for i, p := range periods { + utcPeriods[i] = model.OnlinePeriod{Start: p.Start.UTC(), End: p.End.UTC(), Ongoing: p.Ongoing} + } + + return &model.OnlineHistoryResponse{ + ServerID: serverID, + ServerName: server.Name, + Days: days, + MaxDays: maxDays, + Timezone: singleton.Loc.String(), + TzOffset: tzOffsetSeconds(now, singleton.Loc), + WindowStart: windowStart.UTC(), + WindowEnd: windowEnd.UTC(), + TotalOnlineSeconds: totalSeconds, + OnlineRate: onlineRate, + CurrentlyOnline: currentlyOnline, + LastActive: lastActive, + Periods: utcPeriods, + Daily: daily, + }, nil +} + +// tzOffsetSeconds 返回指定时区在给定时刻相对 UTC 的偏移秒数。 +func tzOffsetSeconds(at time.Time, loc *time.Location) int { + _, offset := at.In(loc).Zone() + return offset +} diff --git a/cmd/dashboard/controller/online_history_test.go b/cmd/dashboard/controller/online_history_test.go new file mode 100644 index 00000000..544703a4 --- /dev/null +++ b/cmd/dashboard/controller/online_history_test.go @@ -0,0 +1,356 @@ +package controller + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/nezhahq/nezha/model" + "github.com/nezhahq/nezha/pkg/i18n" + "github.com/nezhahq/nezha/service/singleton" +) + +func setupOnlineHistoryControllerTest(t *testing.T) (visibleServer, hiddenServer *model.Server) { + t.Helper() + originalDB := singleton.DB + originalLocalizer := singleton.Localizer + originalServer := singleton.ServerShared + originalConf := singleton.Conf + originalLoc := singleton.Loc + if singleton.Localizer == nil { + singleton.Localizer = i18n.NewLocalizer("en_US", "nezha", "translations", i18n.Translations) + } + + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&model.User{}, &model.Server{}, &model.OnlineSession{})) + + singleton.DB = db + singleton.ServerShared = singleton.NewEmptyServerClassForTest() + singleton.Conf = &singleton.ConfigClass{Config: &model.Config{}} + singleton.Conf.OnlineStatsDays = 30 + singleton.Loc = time.FixedZone("TEST", 8*3600) + + visibleServer = &model.Server{ + Common: model.Common{ID: 1}, + Name: "visible", + UUID: "online-history-visible", + LastActive: time.Now(), + } + hiddenServer = &model.Server{ + Common: model.Common{ID: 2}, + Name: "hidden", + UUID: "online-history-hidden", + HideForGuest: true, + } + singleton.ServerShared.InsertForTest(visibleServer) + singleton.ServerShared.InsertForTest(hiddenServer) + + t.Cleanup(func() { + _ = sqlDB.Close() + singleton.DB = originalDB + singleton.Localizer = originalLocalizer + singleton.ServerShared = originalServer + singleton.Conf = originalConf + singleton.Loc = originalLoc + }) + return visibleServer, hiddenServer +} + +func onlineHistoryCtx(t *testing.T, serverID string, query string) *gin.Context { + t.Helper() + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/server/"+serverID+"/online-history?"+query, nil) + c.Params = gin.Params{{Key: "id", Value: serverID}} + return c +} + +func onlineHistoryCtxAsMember(t *testing.T, serverID string, query string) *gin.Context { + t.Helper() + c := onlineHistoryCtx(t, serverID, query) + c.Set(model.CtxKeyAuthorizedUser, &model.User{Common: model.Common{ID: 100}, Role: model.RoleMember}) + return c +} + +func seedOnlineSessions(t *testing.T, serverID uint64, now time.Time) []model.OnlineSession { + t.Helper() + now = now.Truncate(time.Second) + sessions := []model.OnlineSession{ + {ServerID: serverID, StartedAt: now.Add(-48 * time.Hour), EndedAt: now.Add(-47 * time.Hour), LastActiveAt: now.Add(-47 * time.Hour)}, + {ServerID: serverID, StartedAt: now.Add(-25 * time.Hour), EndedAt: now.Add(-23*time.Hour - 30*time.Minute), LastActiveAt: now.Add(-23 * time.Hour)}, + {ServerID: serverID, StartedAt: now.Add(-10 * time.Minute), Ongoing: true, LastActiveAt: now}, + // 超出默认 7 天窗口、但在 30 天窗口内的会话 + {ServerID: serverID, StartedAt: now.Add(-20 * time.Hour * 24), EndedAt: now.Add(-20*time.Hour*24 + time.Hour), LastActiveAt: now.Add(-20 * time.Hour * 24)}, + } + for i := range sessions { + require.NoError(t, singleton.DB.Create(&sessions[i]).Error) + } + return sessions +} + +func TestGetServerOnlineHistoryDefaultsToConfiguredWindow(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + // 不带 days 参数：使用后台设置的统计范围（30 天） + c := onlineHistoryCtxAsMember(t, "1", "") + resp, err := getServerOnlineHistory(c) + require.NoError(t, err) + require.Equal(t, 30, resp.Days) + require.Equal(t, 30, resp.MaxDays) + // 4 个会话全部落在 30 天窗口内 + require.Len(t, resp.Periods, 4) + require.True(t, resp.CurrentlyOnline) +} + +func TestGetServerOnlineHistoryDefaultIsSevenDays(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + // 出厂默认统计范围为 7 天 + singleton.Conf.OnlineStatsDays = model.DefaultOnlineStatsDays + c := onlineHistoryCtxAsMember(t, "1", "") + resp, err := getServerOnlineHistory(c) + require.NoError(t, err) + require.Equal(t, 7, resp.Days) + // 20 天前的会话不进入 7 天窗口 + require.Len(t, resp.Periods, 3) +} + +func TestGetServerOnlineHistoryAggregatesAndClampsWindow(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + c := onlineHistoryCtxAsMember(t, "1", "days=9999") + resp, err := getServerOnlineHistory(c) + require.NoError(t, err) + // 请求超出后台设置窗口时收敛到设置值 + require.Equal(t, 30, resp.Days) + + // 聚合正确性：三个已结束/进行中会话都计入，进行中会话按当前时间截断 + var total uint64 + for _, p := range resp.Periods { + total += uint64(p.End.Sub(p.Start).Seconds()) + } + require.Equal(t, resp.TotalOnlineSeconds, total) + require.GreaterOrEqual(t, resp.TotalOnlineSeconds, uint64(3600+5400+599)) + + // 每日拆分合计等于总时长 + var dailySum uint64 + for _, d := range resp.Daily { + dailySum += d.OnlineSeconds + } + require.Equal(t, resp.TotalOnlineSeconds, dailySum) + + // 在线率按窗口秒数计算 + windowSeconds := resp.WindowEnd.Sub(resp.WindowStart).Seconds() + require.InDelta(t, float64(resp.TotalOnlineSeconds)/windowSeconds*100, resp.OnlineRate, 0.001) +} + +func TestGetServerOnlineHistoryInvalidDays(t *testing.T) { + setupOnlineHistoryControllerTest(t) + for _, q := range []string{"days=abc", "days=0", "days=-1"} { + c := onlineHistoryCtxAsMember(t, "1", q) + _, err := getServerOnlineHistory(c) + require.EqualError(t, err, singleton.Localizer.ErrorT("invalid days parameter").Error(), "query %q should be rejected", q) + } +} + +func TestGetServerOnlineHistoryUnknownServer(t *testing.T) { + setupOnlineHistoryControllerTest(t) + c := onlineHistoryCtx(t, "424242", "") + _, err := getServerOnlineHistory(c) + require.Error(t, err) +} + +func TestGetServerOnlineHistoryGuestLimitedToOneDay(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + now := time.Now() + seedOnlineSessions(t, visible.ID, now) + + // 游客不带 days 参数时默认为配置窗口（>1 天），与 metrics 的游客语义一致被拒绝 + _, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "")) + require.Error(t, err) + + // 游客显式请求多天被拒绝，1 天可用 + _, err = getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=3")) + require.Error(t, err) + resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1")) + require.NoError(t, err) + require.Equal(t, 1, resp.Days) + // 与 1 天窗口相交的两个会话：一个被裁剪到窗口起点，一个进行中 + require.Len(t, resp.Periods, 2) + + // 登录用户不受游客窗口限制 + resp, err = getServerOnlineHistory(onlineHistoryCtxAsMember(t, "1", "days=30")) + require.NoError(t, err) + require.Equal(t, 30, resp.Days) + require.Len(t, resp.Periods, 4) +} + +func TestGetServerOnlineHistoryPermission(t *testing.T) { + visible, _ := setupOnlineHistoryControllerTest(t) + + // 游客可以查看未隐藏服务器（窗口受限于 1 天） + guest := onlineHistoryCtx(t, "1", "days=1") + resp, err := getServerOnlineHistory(guest) + require.NoError(t, err) + require.Equal(t, visible.ID, resp.ServerID) + + // 游客不能查看 HideForGuest 服务器 + guestHidden := onlineHistoryCtx(t, "2", "days=1") + _, err = getServerOnlineHistory(guestHidden) + require.EqualError(t, err, singleton.Localizer.ErrorT("unauthorized").Error()) + + // 成员不能查看他人隐藏服务器 + member := onlineHistoryCtx(t, "2", "days=7") + member.Set(model.CtxKeyAuthorizedUser, &model.User{Common: model.Common{ID: 100}, Role: model.RoleMember}) + _, err = getServerOnlineHistory(member) + require.Error(t, err) +} + +func patchOnlineStatsDays(t *testing.T, days any) error { + t.Helper() + gin.SetMode(gin.TestMode) + body, err := json.Marshal(map[string]any{"user_template": "user-dist", "online_stats_days": days}) + require.NoError(t, err) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPatch, "/api/v1/setting", bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + _, err = updateConfig(c) + return err +} + +func TestUpdateConfigOnlineStatsDaysRange(t *testing.T) { + originalLocalizer := singleton.Localizer + if singleton.Localizer == nil { + singleton.Localizer = i18n.NewLocalizer("en_US", "nezha", "translations", i18n.Translations) + } + t.Cleanup(func() { singleton.Localizer = originalLocalizer }) + + templates := []model.FrontendTemplate{{Path: "user-dist", IsAdmin: false}} + originalFrontendTemplates := singleton.FrontendTemplates + singleton.FrontendTemplates = templates + t.Cleanup(func() { singleton.FrontendTemplates = originalFrontendTemplates }) + conf := &singleton.ConfigClass{Config: &model.Config{}} + configPath := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, conf.Read(configPath, templates)) + originalConf := singleton.Conf + singleton.Conf = conf + t.Cleanup(func() { singleton.Conf = originalConf }) + + require.Equal(t, model.DefaultOnlineStatsDays, conf.OnlineStatsDays) + + // 合法边界值 + for _, d := range []int{model.MinOnlineStatsDays, 30, model.MaxOnlineStatsDays} { + require.NoError(t, patchOnlineStatsDays(t, d), "days=%d should be accepted", d) + require.Equal(t, d, conf.OnlineStatsDays) + } + + // 越界值整体拒绝且不修改现值 + for _, d := range []int{6, model.MaxOnlineStatsDays + 1, 0, -5} { + require.Error(t, patchOnlineStatsDays(t, d), "days=%d should be rejected", d) + require.Equal(t, model.MaxOnlineStatsDays, conf.OnlineStatsDays) + } + + // 非法值必须整体拒绝：校验先于任何对 Conf 的写入完成，其他字段不得 + // 半提交（评审复现场景：错误返回后 SiteName 不能被请求内容覆盖） + conf.SiteName = "existing-site" + require.Error(t, patchOnlineStatsDays(t, 6)) + require.Equal(t, "existing-site", conf.SiteName) + conf.SiteName = "" + + // 字段缺省时保留现值（部分 PATCH 语义） + require.NoError(t, patchOnlineStatsDays(t, nil)) + require.Equal(t, model.MaxOnlineStatsDays, conf.OnlineStatsDays) + + // 重启持久化断言：updateConfig 显式 Save 后，重新从磁盘读取的配置 + // 必须带上刚写入的值（持久化不再依赖 MCP 过渡块的副作用） + reloaded := &singleton.ConfigClass{Config: &model.Config{}} + require.NoError(t, reloaded.Read(configPath, templates)) + require.Equal(t, model.MaxOnlineStatsDays, reloaded.OnlineStatsDays) +} + +func TestSettingFormOnlineStatsDaysDecode(t *testing.T) { + var absent model.SettingForm + require.NoError(t, json.NewDecoder(bytes.NewReader([]byte(`{"site_name":"x"}`))).Decode(&absent)) + require.Nil(t, absent.OnlineStatsDays, "omitted key must decode to nil") + + var explicit model.SettingForm + require.NoError(t, json.NewDecoder(bytes.NewReader([]byte(`{"online_stats_days":3650}`))).Decode(&explicit)) + require.NotNil(t, explicit.OnlineStatsDays) + require.Equal(t, 3650, *explicit.OnlineStatsDays) +} + +func TestOnlineHistoryReadsPendingCloseBoundary(t *testing.T) { + setupOnlineHistoryControllerTest(t) + previous := singleton.OnlineHistoryShared + t.Cleanup(func() { singleton.OnlineHistoryShared = previous }) + r := singleton.NewOnlineHistoryRecorder() + singleton.OnlineHistoryShared = r + start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + r.RecordOnline(1, start) + require.NoError(t, singleton.DB.Exec(`CREATE TRIGGER reject_close BEFORE UPDATE ON online_sessions WHEN NEW.ongoing = 0 BEGIN SELECT RAISE(ABORT, 'close failure'); END`).Error) + r.RecordOffline(1, start.Add(time.Minute)) + r.RecordOnline(1, start.Add(30*time.Minute)) + resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1")) + require.NoError(t, err) + require.Len(t, resp.Periods, 2) + require.False(t, resp.Periods[0].Ongoing) + require.True(t, resp.Periods[0].End.Equal(start.Add(time.Minute))) + require.Less(t, resp.TotalOnlineSeconds, uint64(1900)) +} + +func TestOnlineHistoryQueryWithNonUTCSession(t *testing.T) { + setupOnlineHistoryControllerTest(t) + now := time.Now() + offset := time.FixedZone("UTC+12", 12*3600) + session := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Hour).In(offset), LastActiveAt: now.Add(-time.Minute).In(offset), Ongoing: true} + require.NoError(t, singleton.DB.Create(&session).Error) + resp, err := getServerOnlineHistory(onlineHistoryCtx(t, "1", "days=1")) + require.NoError(t, err) + require.Len(t, resp.Periods, 1) + require.True(t, resp.Periods[0].Start.Equal(session.StartedAt)) + require.InDelta(t, 3600, resp.TotalOnlineSeconds, 2) +} + +func TestUpdateConfigFailedSavePreservesRetention(t *testing.T) { + server, _ := setupOnlineHistoryControllerTest(t) + require.NoError(t, singleton.DB.Create(server).Error) + templates := []model.FrontendTemplate{{Path: "user-dist"}} + previous := singleton.FrontendTemplates + singleton.FrontendTemplates = templates + t.Cleanup(func() { singleton.FrontendTemplates = previous }) + path := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, singleton.Conf.Read(path, templates)) + singleton.Conf.OnlineStatsDays = 30 + require.NoError(t, singleton.Conf.Save()) + // Replacing the destination with a directory deterministically fails Save even as root. + require.NoError(t, os.Remove(path)) + require.NoError(t, os.Mkdir(path, 0700)) + now := time.Now().UTC() + old := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)} + require.NoError(t, singleton.DB.Create(&old).Error) + require.Error(t, patchOnlineStatsDays(t, 7)) + require.Equal(t, 30, singleton.Conf.OnlineStatsDays) + singleton.PruneOnlineSessions() + var retained model.OnlineSession + require.NoError(t, singleton.DB.First(&retained, old.ID).Error) + require.Equal(t, old.ID, retained.ID) +} diff --git a/cmd/dashboard/controller/setting.go b/cmd/dashboard/controller/setting.go index 0b6b70e6..4b5395b0 100644 --- a/cmd/dashboard/controller/setting.go +++ b/cmd/dashboard/controller/setting.go @@ -2,6 +2,7 @@ package controller import ( "errors" + "fmt" "log" "strings" @@ -29,7 +30,7 @@ func listConfig(c *gin.Context) (*model.SettingResponse, error) { isAdmin = user.Role.IsAdmin() } - config := *singleton.Conf + config := singleton.Conf.SettingsSnapshot() config.Language = strings.ReplaceAll(config.Language, "_", "-") conf := model.SettingResponse{ @@ -92,6 +93,12 @@ func updateConfig(c *gin.Context) (any, error) { if !userTemplateValid { return nil, errors.New("invalid user template") } + // 全部输入校验必须先于任何对 singleton.Conf 的写入：任一字段非法都应 + // 整体拒绝，避免"请求以错误收场、先处理的字段却已生效"的半提交状态 + if sf.OnlineStatsDays != nil && + (*sf.OnlineStatsDays < model.MinOnlineStatsDays || *sf.OnlineStatsDays > model.MaxOnlineStatsDays) { + return nil, fmt.Errorf("online_stats_days must be between %d and %d", model.MinOnlineStatsDays, model.MaxOnlineStatsDays) + } singleton.Conf.Language = strings.ReplaceAll(sf.Language, "-", "_") @@ -114,18 +121,23 @@ func updateConfig(c *gin.Context) (any, error) { singleton.Conf.AgentRealIPHeader = sf.AgentRealIPHeader singleton.Conf.AgentTLS = sf.AgentTLS singleton.Conf.UserTemplate = sf.UserTemplate - mcpWasEnabled := singleton.Conf.MCPEnabled() - mcpNext := resolveSettingEnableMCP(sf.EnableMCP, mcpWasEnabled) - if err := applyEnableMCPTransition( - mcpWasEnabled, mcpNext, - singleton.Conf.SetMCPEnabled, - singleton.Conf.Save, - fireMCPKillSwitch, - ); err != nil { + if err := singleton.Conf.CommitOnlineHistoryRetention(sf.OnlineStatsDays, func() error { + mcpWasEnabled := singleton.Conf.MCPEnabled() + mcpNext := resolveSettingEnableMCP(sf.EnableMCP, mcpWasEnabled) + return applyEnableMCPTransition( + mcpWasEnabled, mcpNext, + singleton.Conf.SetMCPEnabled, + singleton.Conf.Save, + fireMCPKillSwitch, + ) + }); err != nil { return nil, newGormError("%v", err) } + // applyEnableMCPTransition already persisted every setting, even when MCP + // is unchanged. A second save could fail after irreversible cleanup. + singleton.OnUpdateLang(singleton.Conf.Language) return nil, nil } diff --git a/cmd/dashboard/controller/setting_single_commit_test.go b/cmd/dashboard/controller/setting_single_commit_test.go new file mode 100644 index 00000000..ee200526 --- /dev/null +++ b/cmd/dashboard/controller/setting_single_commit_test.go @@ -0,0 +1,49 @@ +package controller + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" + + "github.com/stretchr/testify/require" +) + +// The transition helper owns Save (including unchanged MCP settings). A direct +// second Save after it can fail after persistence and irreversible revocation. +func TestUpdateConfigHasSinglePersistenceOwner(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "setting.go", nil, 0) + require.NoError(t, err) + transitions, directSaves := 0, 0 + found := false + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != "updateConfig" { + continue + } + found = true + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "applyEnableMCPTransition" { + transitions++ + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok && selector.Sel.Name == "Save" { + directSaves++ + } + return true + }) + } + require.True(t, found) + require.Equal(t, 1, transitions) + require.Zero(t, directSaves) +} + +func TestUnchangedMCPTransitionStillSavesOnce(t *testing.T) { + saves := 0 + err := applyEnableMCPTransition(true, true, func(bool) {}, func() error { saves++; return nil }, func() { t.Error("unchanged MCP must not revoke streams") }) + require.NoError(t, err) + require.Equal(t, 1, saves) +} diff --git a/cmd/dashboard/http_header_timeout.go b/cmd/dashboard/http_header_timeout.go new file mode 100644 index 00000000..3db55188 --- /dev/null +++ b/cmd/dashboard/http_header_timeout.go @@ -0,0 +1,14 @@ +package main + +import ( + "net/http" + "time" +) + +const dashboardReadHeaderTimeout = 5 * time.Second + +func newDashboardHTTPServer(handler http.Handler) *http.Server { + // ReadHeaderTimeout protects incomplete HTTP/1 requests and is cleared after + // headers. Unlike ReadTimeout it does not impose a lifetime on HTTP/2 streams. + return &http.Server{Handler: handler, ReadHeaderTimeout: dashboardReadHeaderTimeout} +} diff --git a/cmd/dashboard/http_header_timeout_test.go b/cmd/dashboard/http_header_timeout_test.go new file mode 100644 index 00000000..d643afaa --- /dev/null +++ b/cmd/dashboard/http_header_timeout_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "crypto/tls" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDashboardRejectsIncompleteHTTPHeaders(t *testing.T) { + for _, useTLS := range []bool{false, true} { + name := "http" + if useTLS { + name = "https" + } + t.Run(name, func(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("incomplete headers reached handler") })) + server.Config = newDashboardHTTPServer(server.Config.Handler) + require.Positive(t, server.Config.ReadHeaderTimeout) + server.Config.ReadHeaderTimeout = 50 * time.Millisecond + if useTLS { + server.StartTLS() + } else { + server.Start() + } + defer server.Close() + var conn net.Conn + var err error + if useTLS { + conn, err = tls.Dial("tcp", server.Listener.Addr().String(), &tls.Config{InsecureSkipVerify: true}) + } else { + conn, err = net.Dial("tcp", server.Listener.Addr().String()) + } + require.NoError(t, err) + defer conn.Close() + require.NoError(t, conn.SetDeadline(time.Now().Add(2*time.Second))) + _, err = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: dashboard\r\nX-Incomplete: ") + require.NoError(t, err) + _, err = io.ReadAll(conn) + if networkErr, ok := err.(net.Error); ok { + require.False(t, networkErr.Timeout(), "server did not close incomplete request") + } + }) + } +} + +func TestDashboardHeaderTimeoutDoesNotCancelHTTP2Stream(t *testing.T) { + for _, useTLS := range []bool{false, true} { + name := "h2c" + if useTLS { + name = "h2" + } + t.Run(name, func(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ProtoMajor != 2 { + t.Errorf("expected HTTP/2, got %s", r.Proto) + } + require.NoError(t, http.NewResponseController(w).Flush()) + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("long-lived request body: %v", err) + return + } + _, _ = w.Write(body) + }) + server := httptest.NewUnstartedServer(handler) + server.Config = newDashboardHTTPServer(handler) + server.Config.ReadHeaderTimeout = 250 * time.Millisecond + server.Config.Protocols = new(http.Protocols) + server.Config.Protocols.SetHTTP1(true) + server.Config.Protocols.SetHTTP2(true) + server.Config.Protocols.SetUnencryptedHTTP2(true) + server.EnableHTTP2 = true + if useTLS { + server.StartTLS() + } else { + server.Start() + } + defer server.Close() + transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, Protocols: new(http.Protocols)} + transport.Protocols.SetHTTP2(true) + transport.Protocols.SetUnencryptedHTTP2(true) + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 10 * time.Second} + reader, writer := io.Pipe() + defer reader.Close() + defer writer.Close() + request, err := http.NewRequest(http.MethodPost, server.URL, reader) + require.NoError(t, err) + response, err := client.Do(request) + require.NoError(t, err) + defer response.Body.Close() + time.Sleep(4 * server.Config.ReadHeaderTimeout) + _, err = writer.Write([]byte("still streaming")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.Equal(t, "still streaming", string(body)) + }) + } +} diff --git a/cmd/dashboard/main.go b/cmd/dashboard/main.go index 4c41ae0a..d636dc4c 100644 --- a/cmd/dashboard/main.go +++ b/cmd/dashboard/main.go @@ -12,7 +12,6 @@ import ( "os" "runtime/debug" "strings" - "time" _ "time/tzdata" "github.com/gin-gonic/gin" @@ -78,6 +77,16 @@ func initSystem(bus chan<- *model.Service) error { return err } + // 在线会话兜底清扫：状态流异常断开（失联）时由这里关闭进行中的会话 + if _, err := singleton.CronShared.AddFunc("@every 30s", singleton.SweepOnlineSessions); err != nil { + return err + } + + // 在线会话保留清理：超出后台设置的统计窗口后删除 + if _, err := singleton.CronShared.AddFunc("0 30 4 * * *", singleton.PruneOnlineSessions); err != nil { + return err + } + if err := singleton.StartJWTSessionGC(); err != nil { return err } @@ -172,10 +181,8 @@ func main() { wireNATDashboardCredentialGate() muxHandler := newHTTPandGRPCMux(httpHandler, grpcHandler) - muxServerHTTP := &http.Server{ - Handler: muxHandler, - ReadHeaderTimeout: time.Second * 5, - } + muxServerHTTP := newDashboardHTTPServer(muxHandler) + muxServerHTTP.Protocols = new(http.Protocols) muxServerHTTP.Protocols.SetHTTP1(true) muxServerHTTP.Protocols.SetUnencryptedHTTP2(true) @@ -183,9 +190,9 @@ func main() { var muxServerHTTPS *http.Server if singleton.Conf.HTTPS.ListenPort != 0 { muxServerHTTPS = &http.Server{ + ReadHeaderTimeout: dashboardReadHeaderTimeout, Addr: fmt.Sprintf("%s:%d", singleton.Conf.ListenHost, singleton.Conf.HTTPS.ListenPort), Handler: muxHandler, - ReadHeaderTimeout: time.Second * 5, TLSConfig: &tls.Config{ InsecureSkipVerify: singleton.Conf.HTTPS.InsecureTLS, }, diff --git a/model/config.go b/model/config.go index fde4bd19..6dc42b89 100644 --- a/model/config.go +++ b/model/config.go @@ -79,6 +79,10 @@ type ConfigDashboard struct { IgnoredIPNotification string `koanf:"ignored_ip_notification" json:"ignored_ip_notification,omitempty"` // 特定服务器IP（多个服务器用逗号分隔） DNSServers string `koanf:"dns_servers" json:"dns_servers,omitempty"` + + // OnlineStatsDays 设备在线时间段统计的保留与查询窗口（天）。 + // 取值范围 [MinOnlineStatsDays, MaxOnlineStatsDays]，默认 DefaultOnlineStatsDays。 + OnlineStatsDays int `koanf:"online_stats_days" json:"online_stats_days,omitempty"` } type Config struct { @@ -232,6 +236,13 @@ func (c *Config) Read(path string, frontendTemplates []FrontendTemplate) error { c.JWTTimeout = 1 } + if c.OnlineStatsDays == 0 { + c.OnlineStatsDays = DefaultOnlineStatsDays + } else if c.OnlineStatsDays < MinOnlineStatsDays || c.OnlineStatsDays > MaxOnlineStatsDays { + log.Printf("NEZHA>> online_stats_days %d out of range [%d, %d], reset to %d", c.OnlineStatsDays, MinOnlineStatsDays, MaxOnlineStatsDays, DefaultOnlineStatsDays) + c.OnlineStatsDays = DefaultOnlineStatsDays + } + c.jwtIPChangeAllowed.Store(c.AllowJWTIPChange) if c.AgentSecretKey == "" { diff --git a/model/online_history.go b/model/online_history.go new file mode 100644 index 00000000..79ba766c --- /dev/null +++ b/model/online_history.go @@ -0,0 +1,211 @@ +package model + +import ( + "gorm.io/gorm" + "sort" + "time" +) + +// 在线统计窗口的取值边界（天）。后台管理端可设置统计范围，查询窗口不允许 +// 超过该设置，历史数据按该设置做保留清理。 +const ( + DefaultOnlineStatsDays = 7 + MinOnlineStatsDays = 7 + MaxOnlineStatsDays = 3650 +) + +// OnlineActiveWindow 服务器状态上报距该窗口内视为在线。与 MCP server.list +// 的 onlineWindow 口径保持一致；超过该窗口未上报的进行中会话由定时清扫关闭。 +const OnlineActiveWindow = 30 * time.Second + +// OnlineSession 记录一台服务器的一段连续在线时间。Ongoing=true 表示会话 +// 仍在进行（此时 EndedAt 为零值）；会话结束时写入 EndedAt 并置 Ongoing=false。 +// LastActiveAt 由上报侧节流持久化，用于 dashboard 异常停机后把进行中的会话 +// 结转到最后一次已知活跃时刻。 +type OnlineSession struct { + ID uint64 `gorm:"primaryKey" json:"id,omitempty"` + CreatedAt time.Time `gorm:"index;<-:create" json:"created_at,omitempty"` + UpdatedAt time.Time `gorm:"autoUpdateTime" json:"updated_at,omitempty"` + ServerID uint64 `gorm:"index:idx_online_sessions_server_started;index:idx_online_sessions_server_closed,priority:1;index:idx_online_sessions_server_open,priority:1" json:"server_id,omitempty"` + StartedAt time.Time `gorm:"index:idx_online_sessions_server_started;index:idx_online_sessions_server_open,priority:3" json:"started_at,omitempty"` + EndedAt time.Time `gorm:"index;index:idx_online_sessions_server_closed,priority:3" json:"ended_at,omitempty"` // 会话结束时刻，进行中为零值 + // 上报侧节流持久化的最后活跃时刻（≥OnlineActiveWindow 间隔写一次） + LastActiveAt time.Time `json:"last_active_at,omitempty"` + Ongoing bool `gorm:"index:idx_online_sessions_server_closed,priority:2;index:idx_online_sessions_server_open,priority:2" json:"ongoing"` +} + +// BeforeSave keeps SQLite's lexicographic timestamp comparisons chronological. +// Query parameters and map-based updates must use UTC as well. +func (s *OnlineSession) BeforeSave(_ *gorm.DB) error { + s.StartedAt = s.StartedAt.UTC() + s.EndedAt = s.EndedAt.UTC() + s.LastActiveAt = s.LastActiveAt.UTC() + return nil +} + +// OnlinePeriod 是裁剪到查询窗口后的在线时间段，供前端时间轴直接渲染。 +type OnlinePeriod struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` + Ongoing bool `json:"ongoing"` +} + +// DailyOnlineStat 是单个自然日（面板时区）在查询窗口内的在线秒数。 +// 窗口首尾为非整日时，只统计与窗口重叠的部分。 +type DailyOnlineStat struct { + Date string `json:"date"` + OnlineSeconds uint64 `json:"online_seconds"` +} + +// OnlineHistoryResponse 是单台服务器在线时间段统计的聚合结果。 +type OnlineHistoryResponse struct { + ServerID uint64 `json:"server_id"` + ServerName string `json:"server_name"` + Days int `json:"days"` + MaxDays int `json:"max_days"` + // Daily 拆日所用面板时区：IANA 名与窗口末端相对 UTC 的偏移秒数， + // 前端按同一时区渲染，保证跨时区访客的图形与数字一致 + Timezone string `json:"timezone"` + TzOffset int `json:"tz_offset"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + TotalOnlineSeconds uint64 `json:"total_online_seconds"` + OnlineRate float64 `json:"online_rate"` + CurrentlyOnline bool `json:"currently_online"` + // 从未上报过的服务器该值为 nil；指针避免把 Go 零值时间序列化成 0001-01-01 + LastActive *time.Time `json:"last_active,omitempty"` + Periods []OnlinePeriod `json:"periods"` + Daily []DailyOnlineStat `json:"daily"` +} + +// ClipOnlineSessions 把与窗口 [windowStart, windowEnd) 相交的会话裁剪成 +// OnlinePeriod 列表。进行中的会话以 windowEnd 为右边界（保留 Ongoing 标记）。 +// 输入需按 StartedAt 升序；输出与输入同序。 +func ClipOnlineSessions(sessions []OnlineSession, windowStart, windowEnd time.Time) []OnlinePeriod { + periods := make([]OnlinePeriod, 0, len(sessions)) + for _, s := range sessions { + start := s.StartedAt + if start.Before(windowStart) { + start = windowStart + } + end := s.EndedAt + ongoing := s.Ongoing || end.IsZero() + if ongoing || end.After(windowEnd) { + end = windowEnd + } + if !start.Before(end) { + // 与窗口零交集（理论上不会由查询条件选出）或空会话 + continue + } + periods = append(periods, OnlinePeriod{Start: start, End: end, Ongoing: ongoing}) + } + return periods +} + +// DailyOnlineBreakdown 把在线时间段按面板时区的自然日拆分，返回窗口覆盖的 +// 每一天的在线秒数（含在线为 0 秒的日期），按日期升序。 +// 取整口径：每天先累计原始时长（日界切分不损失精度），仅在每日汇总时向下 +// 取整一次；调用方以各日之和作为总时长，保证响应内部 total 与 daily[] 严格 +// 对账，不与"逐段截断再累加"的口径混用（亚秒会话会让后者系统性偏小）。 +// 时段二分定位起始日界后逐日推进，复杂度 O(P·log D + D)；单服务器的会话 +// 互不重叠，内层推进总量受窗口天数约束，长窗口（如 days=3650）下不会出现 +// 时段数×天数的双重循环。 +func DailyOnlineBreakdown(periods []OnlinePeriod, windowStart, windowEnd time.Time, loc *time.Location) []DailyOnlineStat { + first := windowStart.In(loc) + // Advance a timezone-independent calendar, not a normalized local midnight: + // midnight DST gaps can normalize into the previous day and repeat dates. + date := time.Date(first.Year(), first.Month(), first.Day(), 0, 0, 0, 0, time.UTC) + var days []DailyOnlineStat + var dayStarts, dayEnds []time.Time + for { + start := onlineCalendarBoundary(date, loc) + if !start.Before(windowEnd) { + break + } + nextDate := date.AddDate(0, 0, 1) + end := onlineCalendarBoundary(nextDate, loc) + if start.Before(end) { + dayStarts = append(dayStarts, start) + dayEnds = append(dayEnds, end) + days = append(days, DailyOnlineStat{Date: date.Format("2006-01-02")}) + } + date = nextDate + } + + if len(days) == 0 { + return days + } + + dayDurations := make([]time.Duration, len(days)) + for _, p := range periods { + // 该时段覆盖的第一天：不晚于时段开始的最后一个日界 + i := sort.Search(len(dayStarts), func(i int) bool { + return dayStarts[i].After(p.Start) + }) - 1 + if i < 0 { + i = 0 + } + // 逐日推进，累加该时段与每一天 [dayStart, nextDay) 的交集 + for ; i < len(dayStarts) && dayStarts[i].Before(p.End); i++ { + overlapStart := maxTime(p.Start, dayStarts[i]) + overlapEnd := minTime(p.End, dayEnds[i]) + if overlapStart.Before(overlapEnd) { + dayDurations[i] += overlapEnd.Sub(overlapStart) + } + } + } + for i := range days { + days[i].OnlineSeconds = uint64(dayDurations[i].Seconds()) + } + return days +} + +func maxTime(a, b time.Time) time.Time { + if a.After(b) { + return a + } + return b +} + +func minTime(a, b time.Time) time.Time { + if a.Before(b) { + return a + } + return b +} + +// A missing midnight (or an entirely skipped date) starts at the timezone +// transition, not at the previous date returned by time.Date normalization. +func onlineCalendarBoundary(date time.Time, loc *time.Location) time.Time { + boundary := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, loc) + if boundary.Format("2006-01-02") < date.Format("2006-01-02") { + _, end := boundary.ZoneBounds() + if !end.IsZero() { + return end + } + } + // time.Date may choose the second midnight in a rollback. Recover the + // earlier occurrence using the offset on the preceding side of the transition. + zoneStart, _ := boundary.ZoneBounds() + if !zoneStart.IsZero() { + _, currentOffset := boundary.Zone() + _, previousOffset := zoneStart.Add(-time.Nanosecond).Zone() + if previousOffset > currentOffset { + earlier := boundary.Add(-time.Duration(previousOffset-currentOffset) * time.Second) + if earlier.Format("2006-01-02 15:04:05") == boundary.Format("2006-01-02 15:04:05") { + boundary = earlier + } + } + } + return boundary +} + +// OnlineHistoryWindowStart defines the shared query and retention calendar window. +func OnlineHistoryWindowStart(now time.Time, days int, loc *time.Location) time.Time { + // Calendar days follow the dashboard timezone, not the host timezone; + // DST days can contain 23 or 25 hours. Queries and pruning must agree. + if loc == nil { + loc = time.UTC + } + return now.In(loc).AddDate(0, 0, -days).UTC() +} diff --git a/model/online_history_test.go b/model/online_history_test.go new file mode 100644 index 00000000..5403ef98 --- /dev/null +++ b/model/online_history_test.go @@ -0,0 +1,220 @@ +package model + +import ( + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var testZone = time.FixedZone("TEST", 8*3600) + +func ts(y int, mo time.Month, d, h, mi, s int) time.Time { + return time.Date(y, mo, d, h, mi, s, 0, testZone) +} + +func TestClipOnlineSessions(t *testing.T) { + windowStart := ts(2026, 10, 1, 0, 0, 0) + windowEnd := ts(2026, 10, 8, 0, 0, 0) + + t.Run("session fully inside window", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 10, 2, 8, 0, 0), EndedAt: ts(2026, 10, 2, 9, 30, 0)}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.Equal(t, ts(2026, 10, 2, 8, 0, 0), periods[0].Start) + require.Equal(t, ts(2026, 10, 2, 9, 30, 0), periods[0].End) + require.False(t, periods[0].Ongoing) + }) + + t.Run("session started before window is clipped", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 9, 28, 12, 0, 0), EndedAt: ts(2026, 10, 1, 5, 0, 0)}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.Equal(t, windowStart, periods[0].Start) + require.Equal(t, ts(2026, 10, 1, 5, 0, 0), periods[0].End) + }) + + t.Run("ongoing session ends at window end", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 10, 7, 10, 0, 0), Ongoing: true}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.Equal(t, windowEnd, periods[0].End) + require.True(t, periods[0].Ongoing) + }) + + t.Run("zero ended_at treated as ongoing", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 10, 7, 10, 0, 0)}, + } + periods := ClipOnlineSessions(sessions, windowStart, windowEnd) + require.Len(t, periods, 1) + require.True(t, periods[0].Ongoing) + require.Equal(t, windowEnd, periods[0].End) + }) + + t.Run("session outside window yields no period", func(t *testing.T) { + sessions := []OnlineSession{ + {ServerID: 1, StartedAt: ts(2026, 9, 1, 0, 0, 0), EndedAt: ts(2026, 9, 2, 0, 0, 0)}, + } + require.Empty(t, ClipOnlineSessions(sessions, windowStart, windowEnd)) + }) +} + +func TestDailyOnlineBreakdown(t *testing.T) { + loc := time.FixedZone("UTC+8", 8*3600) + windowStart := time.Date(2026, 10, 1, 10, 0, 0, 0, loc) + windowEnd := time.Date(2026, 10, 3, 14, 0, 0, 0, loc) + + periods := []OnlinePeriod{ + // 跨午夜：2 日 23:00 ~ 3 日 01:00 + {Start: time.Date(2026, 10, 2, 23, 0, 0, 0, loc), End: time.Date(2026, 10, 3, 1, 0, 0, 0, loc)}, + // 窗口前就开始的会话，裁剪后从窗口起点开始 + {Start: windowStart, End: time.Date(2026, 10, 1, 12, 0, 0, 0, loc)}, + } + + daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + require.Len(t, daily, 3) + require.Equal(t, "2026-10-01", daily[0].Date) + require.Equal(t, "2026-10-02", daily[1].Date) + require.Equal(t, "2026-10-03", daily[2].Date) + + // 1 日：12:00-10:00 = 2h；2 日：23:00-24:00 = 1h；3 日：0:00-1:00 + 窗口外无 = 1h + require.Equal(t, uint64(7200), daily[0].OnlineSeconds) + require.Equal(t, uint64(3600), daily[1].OnlineSeconds) + require.Equal(t, uint64(3600), daily[2].OnlineSeconds) + + // 每日合计即总时长（口径：每日累计原始时长后统一取整） + var sum uint64 + for _, d := range daily { + sum += d.OnlineSeconds + } + require.Equal(t, uint64(7200+3600+3600), sum) +} + +// 评审回归：取整必须按日累计原始时长后统一转换。逐段截断再累加会让亚秒 +// 会话系统性偏小——同一天两个 1.5s 的会话应得 3s，而不是 1+1=2s +func TestDailyOnlineBreakdownSubSecond(t *testing.T) { + loc := testZone + windowStart := ts(2026, 10, 1, 0, 0, 0) + windowEnd := ts(2026, 10, 2, 0, 0, 0) + periods := []OnlinePeriod{ + {Start: windowStart.Add(100 * time.Millisecond), End: windowStart.Add(1600 * time.Millisecond)}, + {Start: windowStart.Add(2 * time.Second), End: windowStart.Add(3500 * time.Millisecond)}, + } + daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + require.Len(t, daily, 1) + require.Equal(t, uint64(3), daily[0].OnlineSeconds) +} + +// 跨日时段按日界切分不损失精度：各日分别累计原始时长后各自取整 +func TestDailyOnlineBreakdownCrossDayFraction(t *testing.T) { + loc := testZone + windowStart := ts(2026, 10, 1, 0, 0, 0) + windowEnd := ts(2026, 10, 3, 0, 0, 0) + periods := []OnlinePeriod{{ + Start: time.Date(2026, 10, 1, 23, 59, 59, 250000000, loc), + End: time.Date(2026, 10, 2, 0, 0, 2, 750000000, loc), + }} + daily := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + require.Len(t, daily, 2) + require.Equal(t, uint64(0), daily[0].OnlineSeconds) // 0.75s → 0 + require.Equal(t, uint64(2), daily[1].OnlineSeconds) // 2.75s → 2 +} + +func TestDailyOnlineBreakdownZeroOnline(t *testing.T) { + loc := time.FixedZone("UTC+8", 8*3600) + windowStart := time.Date(2026, 10, 1, 0, 0, 0, 0, loc) + windowEnd := time.Date(2026, 10, 4, 0, 0, 0, 0, loc) + daily := DailyOnlineBreakdown(nil, windowStart, windowEnd, loc) + require.Len(t, daily, 3) + for _, d := range daily { + require.Zero(t, d.OnlineSeconds) + } +} + +func TestOnlineStatsDaysBounds(t *testing.T) { + require.Equal(t, 7, DefaultOnlineStatsDays) + require.Equal(t, 7, MinOnlineStatsDays) + require.Equal(t, 3650, MaxOnlineStatsDays) +} + +// F1 回归保护：指针推进实现必须与逐日朴素实现对任意不相交时段集合等价 +// （单服务器会话互不重叠的约束一致），长窗口大量时段下不再随时段数平方放大。 +func TestDailyOnlineBreakdownEquivalence(t *testing.T) { + loc := testZone + windowStart := ts(2026, 1, 1, 0, 0, 0) + windowEnd := windowStart.AddDate(0, 0, 90) + + rng := rand.New(rand.NewSource(42)) + var periods []OnlinePeriod + cursor := windowStart + for cursor.Before(windowEnd) { + // 起止带亚秒抖动：让逐段截断与按日累计两种口径可区分 + start := cursor.Add(time.Duration(rng.Intn(4*3600))*time.Second + time.Duration(rng.Int63n(int64(time.Second)))) + end := start.Add(time.Duration(1+rng.Intn(6*3600))*time.Second + time.Duration(rng.Int63n(int64(time.Second)))) + if end.After(windowEnd) { + end = windowEnd + } + periods = append(periods, OnlinePeriod{Start: start, End: end}) + cursor = end.Add(time.Duration(1+rng.Intn(3600)) * time.Second) + } + + days := DailyOnlineBreakdown(periods, windowStart, windowEnd, loc) + + // 参考实现与生产同一口径：每日累计原始时长、汇总时向下取整一次 + reference := map[string]time.Duration{} + for _, p := range periods { + for d := windowStart; d.Before(windowEnd); d = d.AddDate(0, 0, 1) { + overlapStart := maxTime(p.Start, d) + overlapEnd := minTime(p.End, d.AddDate(0, 0, 1)) + if overlapStart.Before(overlapEnd) { + reference[d.Format("2006-01-02")] += overlapEnd.Sub(overlapStart) + } + } + } + require.Len(t, days, 90) + for _, day := range days { + require.Equal(t, uint64(reference[day.Date].Seconds()), day.OnlineSeconds, "date %s", day.Date) + } +} + +func TestDailyOnlineBreakdownMidnightDSTTransition(t *testing.T) { + loc, err := time.LoadLocation("America/Sao_Paulo") + require.NoError(t, err) + start := time.Date(2018, 11, 3, 0, 0, 0, 0, loc) + end := time.Date(2018, 11, 6, 0, 0, 0, 0, loc) + days := DailyOnlineBreakdown([]OnlinePeriod{{Start: start, End: end}}, start, end, loc) + require.Equal(t, []DailyOnlineStat{{Date: "2018-11-03", OnlineSeconds: 86400}, {Date: "2018-11-04", OnlineSeconds: 82800}, {Date: "2018-11-05", OnlineSeconds: 86400}}, days) +} + +func TestDailyOnlineBreakdownRepeatedMidnight(t *testing.T) { + loc, err := time.LoadLocation("Asia/Amman") + require.NoError(t, err) + start := time.Date(2021, 10, 28, 21, 0, 0, 0, time.UTC) + end := time.Date(2021, 10, 29, 22, 0, 0, 0, time.UTC) + days := DailyOnlineBreakdown([]OnlinePeriod{{Start: start, End: end}}, start, end, loc) + require.Equal(t, []DailyOnlineStat{{Date: "2021-10-29", OnlineSeconds: 90000}}, days) +} + +func TestOnlineHistoryWindowUsesDashboardCalendar(t *testing.T) { + loc, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + for _, test := range []struct { + month time.Month + day, hours int + }{{time.March, 9, 23}, {time.November, 2, 25}} { + end := time.Date(2026, test.month, test.day, 0, 30, 0, 0, loc) + start := OnlineHistoryWindowStart(end.UTC(), 1, loc) + require.True(t, start.Equal(time.Date(2026, test.month, test.day-1, 0, 30, 0, 0, loc))) + require.Equal(t, time.Duration(test.hours)*time.Hour, end.Sub(start)) + require.Len(t, DailyOnlineBreakdown(nil, start, end, loc), 2) + require.True(t, OnlineHistoryWindowStart(end.UTC(), 7, loc).Equal(end.In(loc).AddDate(0, 0, -7))) + } +} diff --git a/model/server.go b/model/server.go index ada61db9..3783ffe8 100644 --- a/model/server.go +++ b/model/server.go @@ -211,10 +211,16 @@ func (lease StateStreamLease) updateState(receiver *Server, state *HostState, la } func (lease StateStreamLease) Clear() bool { - return lease.clear(nil) + return lease.clear(nil, nil) } -func (lease StateStreamLease) clear(receiver *Server) bool { +// ClearWithSideEffect serializes disconnect persistence with replacement streams. +// The callback must not acquire the runtime holder again. +func (lease StateStreamLease) ClearWithSideEffect(sideEffect func()) bool { + return lease.clear(nil, sideEffect) +} + +func (lease StateStreamLease) clear(receiver *Server, sideEffect func()) bool { if lease.holder == nil { return false } @@ -226,6 +232,9 @@ func (lease StateStreamLease) clear(receiver *Server) bool { lease.holder.stream = nil lease.holder.lastActive = time.Time{} lease.holder.canonical.LastActive = time.Time{} + if sideEffect != nil { + sideEffect() + } return true } @@ -329,7 +338,7 @@ func (s *Server) UpdateStateIfCurrentWithSideEffect(lease StateStreamLease, stat } func (s *Server) ClearStateStreamIfCurrent(lease StateStreamLease) bool { - return lease.clear(s) + return lease.clear(s, nil) } // RuntimeSnapshot is a deep copy of the mutable runtime state. diff --git a/model/server_runtime_ownership_test.go b/model/server_runtime_ownership_test.go index 970ac7df..b29f5146 100644 --- a/model/server_runtime_ownership_test.go +++ b/model/server_runtime_ownership_test.go @@ -367,3 +367,30 @@ func TestServerRuntimeOwnership_failedRestartAllowsHourlyRecordThenRetry(t *test require.Equal(t, uint64(0), result.Transfer.In) require.Equal(t, uint64(0), result.Transfer.Out) } + +func TestStateStreamDisconnectSideEffectSerializesReplacement(t *testing.T) { + server := &Server{} + old := server.AttachStateStream(runtimeOwnershipStream{}) + entered := make(chan struct{}) + release := make(chan struct{}) + cleared := make(chan bool, 1) + go func() { cleared <- old.ClearWithSideEffect(func() { close(entered); <-release }) }() + <-entered + attached := make(chan StateStreamLease, 1) + go func() { attached <- server.AttachStateStream(runtimeOwnershipStream{}) }() + select { + case <-attached: + close(release) + t.Fatal("replacement attached before disconnect persistence finished") + case <-time.After(20 * time.Millisecond): + } + close(release) + require.True(t, <-cleared) + next := <-attached + require.True(t, next.UpdateState(&HostState{}, time.Now())) + called := false + require.False(t, old.ClearWithSideEffect(func() { called = true })) + require.False(t, called) + require.False(t, old.UpdateStateWithSideEffect(&HostState{}, time.Now(), func() error { called = true; return nil })) + require.False(t, called) +} diff --git a/model/setting_api.go b/model/setting_api.go index bd0510a0..0a530f54 100644 --- a/model/setting_api.go +++ b/model/setting_api.go @@ -21,6 +21,8 @@ type SettingForm struct { EnablePlainIPInNotification bool `json:"enable_plain_ip_in_notification,omitempty" validate:"optional"` AllowJWTIPChange *bool `json:"allow_jwt_ip_change,omitempty" validate:"optional"` EnableMCP *bool `json:"enable_mcp,omitempty" validate:"optional"` + + OnlineStatsDays *int `json:"online_stats_days,omitempty" validate:"optional"` // 在线统计范围（天），nil=保留现值 } type Setting struct { diff --git a/service/rpc/nezha.go b/service/rpc/nezha.go index e332aa74..30308c46 100644 --- a/service/rpc/nezha.go +++ b/service/rpc/nezha.go @@ -202,7 +202,15 @@ func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStat return errors.New("server not found") } lease := server.AttachStateStream(stream) - defer lease.Clear() + defer func() { + // Persist while owning the lease; a replaced stream must never close a + // newer stream's session between Clear and RecordOffline. + lease.ClearWithSideEffect(func() { + if singleton.OnlineHistoryShared != nil { + singleton.OnlineHistoryShared.RecordOffline(clientID, time.Now()) + } + }) + }() var state *pb.State var stateCount uint64 for { @@ -253,6 +261,9 @@ func (s *NezhaHandler) ReportSystemState(stream pb.NezhaService_ReportSystemStat log.Printf("NEZHA>> Failed to write server metrics to TSDB: %v", err) } } + if singleton.OnlineHistoryShared != nil { + singleton.OnlineHistoryShared.RecordOnline(clientID, lastActive) + } return nil }) if !accepted { diff --git a/service/singleton/config.go b/service/singleton/config.go index 65f19687..6dcd1072 100644 --- a/service/singleton/config.go +++ b/service/singleton/config.go @@ -4,6 +4,7 @@ import ( "log" "strconv" "strings" + "sync" "github.com/nezhahq/nezha/model" "github.com/nezhahq/nezha/pkg/utils" @@ -14,6 +15,8 @@ var Conf *ConfigClass type ConfigClass struct { *model.Config + onlineHistoryRetentionMu sync.RWMutex + IgnoredIPNotificationServerIDs map[uint64]bool `json:"ignored_ip_notification_server_ids,omitempty"` Oauth2Providers []string `json:"oauth2_providers,omitempty"` } @@ -59,3 +62,39 @@ func (c *ConfigClass) updateIgnoredIPNotificationID() { } } } + +// OnlineHistoryRetentionDays returns the retention policy visible to queries and pruning. +func (c *ConfigClass) OnlineHistoryRetentionDays() int { + c.onlineHistoryRetentionMu.RLock() + defer c.onlineHistoryRetentionMu.RUnlock() + return c.OnlineStatsDays +} + +// CommitOnlineHistoryRetention applies a retention change with the settings save. +func (c *ConfigClass) CommitOnlineHistoryRetention(days *int, persist func() error) error { + // Readers must not capture a shorter cutoff while Save can still fail. + // Serialize assignment, persistence and rollback with the policy snapshot. + c.onlineHistoryRetentionMu.Lock() + defer c.onlineHistoryRetentionMu.Unlock() + previous := c.OnlineStatsDays + if days != nil { + c.OnlineStatsDays = *days + } + if err := persist(); err != nil { + c.OnlineStatsDays = previous + return err + } + return nil +} + +// SettingsSnapshot copies response fields without copying the retention mutex. +func (c *ConfigClass) SettingsSnapshot() model.Setting { + c.onlineHistoryRetentionMu.RLock() + defer c.onlineHistoryRetentionMu.RUnlock() + return model.Setting{ + ConfigForGuests: c.ConfigForGuests, + ConfigDashboard: c.ConfigDashboard, + IgnoredIPNotificationServerIDs: c.IgnoredIPNotificationServerIDs, + Oauth2Providers: c.Oauth2Providers, + } +} diff --git a/service/singleton/online_history.go b/service/singleton/online_history.go new file mode 100644 index 00000000..9f61a864 --- /dev/null +++ b/service/singleton/online_history.go @@ -0,0 +1,348 @@ +package singleton + +import ( + "fmt" + "log" + "sort" + "sync" + "time" + + "github.com/nezhahq/nezha/model" +) + +// OnlineHistoryRecorder 维护每台服务器当前进行中的在线会话。会话的打开与 +// 关闭由 agent 状态流的建立/断开驱动（service/rpc），并由定时清扫兜底关闭 +// 失联（未正常断流）的会话；dashboard 重启时通过数据库中 Ongoing 会话恢复。 +type OnlineHistoryRecorder struct { + mu sync.Mutex + restored bool + open map[uint64]*onlineSessionRuntime + // closing 是关库失败后的待关闭状态：断线边界已被观察到，会话 ID 与 + // 结束时间在此固定、不可被后续上报覆盖；重试只为旧会话完成持久化， + // 重连不得把它续接成活跃会话（同服务器可能同时有多条待关闭会话）。 + closing map[uint64][]*onlineSessionClosing + // Preserve the first disconnect while startup reads are unavailable. + recoveryDisconnects map[uint64]time.Time +} + +type onlineSessionRuntime struct { + id uint64 + lastObserved time.Time + revision uint64 + // lastPersist 是 LastActiveAt 最近一次落库时间，用于节流写库。 + lastPersist time.Time +} + +type onlineSessionClosing struct { + id uint64 + // end 是已观察到的会话结束时刻，重试持久化时原样使用。 + end time.Time +} + +var OnlineHistoryShared *OnlineHistoryRecorder + +func NewOnlineHistoryRecorder() *OnlineHistoryRecorder { + r := &OnlineHistoryRecorder{ + open: make(map[uint64]*onlineSessionRuntime), + closing: make(map[uint64][]*onlineSessionClosing), + recoveryDisconnects: make(map[uint64]time.Time), + } + r.mu.Lock() + r.restoreOngoingSessions() + r.mu.Unlock() + return r +} + +// restoreOngoingSessions 每台服务器仅允许最新且仍在在线窗口内的会话续接， +// 其余会话按最后持久化的活跃时间结转。崩溃会丢失内存中的待关闭边界， +// 恢复时采用 LastActiveAt 保守结转，不推测未持久化的断线时间。 +func (r *OnlineHistoryRecorder) restoreOngoingSessions() bool { + if DB == nil { + return false + } + var ongoing []model.OnlineSession + if err := DB.Where("ongoing = ?", true). + Order("server_id ASC").Order("started_at DESC").Order("id DESC").Find(&ongoing).Error; err != nil { + log.Printf("NEZHA>> Failed to load ongoing online sessions: %v", err) + return false + } + now := time.Now() + seenServers := make(map[uint64]bool) + for _, s := range ongoing { + // 关闭失败后重连可留下同服务器多条 Ongoing 行；直接赋值 open 会覆盖 + // 旧 ID，导致其永久脱管。按开始时间（同时间按 ID）选最新行，其余 + // 必须关闭或进入 closing 重试，即使最新行本身已经过期也不能复活旧行。 + duplicate := seenServers[s.ServerID] + seenServers[s.ServerID] = true + disconnect, observedDisconnect := r.recoveryDisconnects[s.ServerID] + observedDisconnect = observedDisconnect && !duplicate && !disconnect.Before(s.StartedAt) + if observedDisconnect || duplicate || s.LastActiveAt.Before(now.Add(-model.OnlineActiveWindow)) { + end := s.LastActiveAt + if observedDisconnect && disconnect.After(end) { + end = disconnect + } + if end.Before(s.StartedAt) { + end = s.StartedAt + } + if err := closeOnlineSessionRow(s.ID, end); err != nil { + log.Printf("NEZHA>> Failed to close recovered online session %d: %v", s.ID, err) + // 关库失败不能跳过，否则会留下永不闭合、也无人重试的 + // Ongoing 行；进入待关闭状态交由清扫周期重试。过期或 + // 被更新会话替代的行均不得被重连续接为活跃会话 + r.closing[s.ServerID] = append(r.closing[s.ServerID], + &onlineSessionClosing{id: s.ID, end: end}) + } + continue + } + r.open[s.ServerID] = &onlineSessionRuntime{id: s.ID, lastPersist: s.LastActiveAt, lastObserved: s.LastActiveAt} + } + clear(r.recoveryDisconnects) + r.restored = true + return true +} + +// flushClosing 尽力重试关闭该服务器的待关闭会话：按入列时固定的结束时间 +// 持久化，成功才移除；失败的条目原样保留，结束时间不得被后续上报改写。 +func (r *OnlineHistoryRecorder) flushClosing(serverID uint64) { + pendings, ok := r.closing[serverID] + if !ok { + return + } + kept := pendings[:0] + for _, pc := range pendings { + if err := closeOnlineSessionRow(pc.id, pc.end); err != nil { + log.Printf("NEZHA>> Failed to retry closing online session %d: %v", pc.id, err) + kept = append(kept, pc) + } + } + if len(kept) == 0 { + delete(r.closing, serverID) + } else { + r.closing[serverID] = kept + } +} + +// RecordOnline 记录一次状态上报：无进行中会话时开启新会话（重启遗留且仍在 +// 窗口内的 Ongoing 会话直接续接）；已有会话时节流落库 LastActiveAt。 +func (r *OnlineHistoryRecorder) RecordOnline(serverID uint64, at time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + // A failed startup read is not an empty history: retry before accepting reports. + if !r.restored && !r.restoreOngoingSessions() { + return + } + at = at.UTC() + r.flushClosing(serverID) + + // A report can arrive after expiry but before the next sweep. Close at the + // previous observation before overwriting it, preserving failed-close retries. + if rt := r.open[serverID]; rt != nil && at.Sub(rt.lastObserved) > model.OnlineActiveWindow { + r.recordOfflineLocked(serverID, rt.lastObserved) + } + if rt, ok := r.open[serverID]; ok { + rt.lastObserved = at + rt.revision++ + if at.Sub(rt.lastPersist) >= model.OnlineActiveWindow { + if err := DB.Model(&model.OnlineSession{}).Where("id = ?", rt.id).Update("last_active_at", at).Error; err != nil { + log.Printf("NEZHA>> Failed to persist online session last_active_at: %v", err) + } else { + // Only committed writes advance the throttle; failures retry on the next report. + rt.lastPersist = at + } + } + return + } + + // All ongoing rows are owned by open or closing after successful recovery. + + s := model.OnlineSession{ServerID: serverID, StartedAt: at, LastActiveAt: at, Ongoing: true} + if err := DB.Create(&s).Error; err != nil { + log.Printf("NEZHA>> Failed to create online session: %v", err) + return + } + r.open[serverID] = &onlineSessionRuntime{id: s.ID, lastPersist: at, lastObserved: at} +} + +// RecordOffline 关闭该服务器当前进行中的会话。at 是会话结束时刻：正常断流 +// 为当前时间，清扫/结转为最后一次已知活跃时间。 +// 关库失败时把会话转入待关闭状态而非保留为活跃条目：会话 ID 与已观察到的 +// 结束时刻（at 与 lastPersist 的较大者）就此固定，后续上报不得覆盖；重试 +// 只为旧会话完成持久化，重连则开启新会话——断线后的离线间隔不得计作在线。 +func (r *OnlineHistoryRecorder) RecordOffline(serverID uint64, at time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + // Disconnect may be the first event after startup storage recovers. + if !r.restored && !r.restoreOngoingSessions() { + // Retrying SELECT alone loses confirmed offline gaps on a quick reconnect. + // Keep the first boundary; subsequent disconnected streams cannot extend it. + if r.recoveryDisconnects == nil { + r.recoveryDisconnects = make(map[uint64]time.Time) + } + if _, exists := r.recoveryDisconnects[serverID]; !exists { + r.recoveryDisconnects[serverID] = at.UTC() + } + return + } + r.recordOfflineLocked(serverID, at) +} + +func (r *OnlineHistoryRecorder) recordOfflineLocked(serverID uint64, at time.Time) { + rt, ok := r.open[serverID] + if !ok { + return + } + end := at + if end.Before(rt.lastPersist) { + end = rt.lastPersist + } + if err := closeOnlineSessionRow(rt.id, end); err != nil { + log.Printf("NEZHA>> Failed to close online session %d: %v", rt.id, err) + // 结束时刻已观察到，不能因写库失败丢失，也不能把行继续当活跃 + // 会话：转待关闭状态，由 RecordOnline 冲洗与清扫周期重试 + r.closing[serverID] = append(r.closing[serverID], &onlineSessionClosing{id: rt.id, end: end}) + delete(r.open, serverID) + return + } + delete(r.open, serverID) +} + +// SweepStale 关闭超过在线窗口仍无状态上报的进行中会话。lastActiveFn 返回 +// 服务器的运行时最后活跃时间与是否存在。 +func (r *OnlineHistoryRecorder) SweepStale(now time.Time, lastActiveFn func(serverID uint64) (time.Time, bool)) { + // Never call the runtime observer while holding mu: reports acquire the + // runtime lease before this mutex. Snapshot identities, then revalidate them. + r.mu.Lock() + if !r.restored && !r.restoreOngoingSessions() { + r.mu.Unlock() + return + } + candidates := make(map[uint64]onlineSessionRuntime, len(r.open)) + for id, rt := range r.open { + candidates[id] = *rt + } + for id := range r.closing { + r.flushClosing(id) + } + r.mu.Unlock() + for id, candidate := range candidates { + lastActive, ok := lastActiveFn(id) + r.mu.Lock() + rt := r.open[id] + if rt != nil && rt.id == candidate.id && rt.revision == candidate.revision { + if lastActive.Before(rt.lastObserved) { + lastActive = rt.lastObserved + } + if (!ok || lastActive.Before(now.Add(-model.OnlineActiveWindow))) && rt.lastObserved.Before(now.Add(-model.OnlineActiveWindow)) { + r.recordOfflineLocked(id, lastActive) + } + } + r.mu.Unlock() + } + +} + +// SweepOnlineSessions 是定时清扫入口，接上服务器运行时的最后活跃时间。 +func SweepOnlineSessions() { + if OnlineHistoryShared == nil || ServerShared == nil || DB == nil { + return + } + OnlineHistoryShared.SweepStale(time.Now(), onlineServerLastActive) +} + +func onlineServerLastActive(serverID uint64) (time.Time, bool) { + server, ok := ServerShared.Get(serverID) + if !ok || server == nil { + return time.Time{}, false + } + lastActive := server.RuntimeSnapshot().LastActive + return lastActive, !lastActive.IsZero() +} + +// PruneOnlineSessions 清理超出统计窗口的已结束会话与已删除服务器的残留记录。 +// 时间列直接比较（与写入时的绑定格式同构）以走 ended_at 索引，并用分批 +// DELETE 替换单条巨型删除，避免大表上长时间占用 sqlite 写锁、阻塞上报路径。 +func PruneOnlineSessions() { + if DB == nil { + return + } + keepDays := Conf.OnlineHistoryRetentionDays() + if keepDays < model.MinOnlineStatsDays || keepDays > model.MaxOnlineStatsDays { + keepDays = model.DefaultOnlineStatsDays + } + cutoff := model.OnlineHistoryWindowStart(time.Now(), keepDays, Loc) + pruneOnlineSessionsBatch("ongoing = ? AND ended_at < ?", false, cutoff) + pruneOnlineSessionsBatch("server_id NOT IN (SELECT `id` FROM servers)") +} + +// pruneOnlineSessionsBatch 按主键分批删除匹配的行，每批之间释放写锁。 +func pruneOnlineSessionsBatch(where string, args ...any) { + const batchSize = 10000 + for { + var ids []uint64 + if err := DB.Model(&model.OnlineSession{}).Unscoped(). + Where(where, args...).Limit(batchSize).Pluck("id", &ids).Error; err != nil { + log.Printf("NEZHA>> Failed to select online sessions for pruning: %v", err) + return + } + if len(ids) == 0 { + return + } + if err := DB.Unscoped().Delete(&model.OnlineSession{}, ids).Error; err != nil { + log.Printf("NEZHA>> Failed to prune online sessions: %v", err) + return + } + if len(ids) < batchSize { + return + } + } +} + +func closeOnlineSessionRow(id uint64, end time.Time) error { + return DB.Model(&model.OnlineSession{}).Where("id = ?", id).Updates(map[string]any{ + "ended_at": end.UTC(), + "ongoing": false, + }).Error +} + +// Sessions snapshots rows and volatile close boundaries under the same lock. +// Otherwise a failed close can count confirmed offline time as ongoing, or a +// successful retry can remove the boundary after the reader fetched a stale row. +func (r *OnlineHistoryRecorder) Sessions(serverID uint64, start, end time.Time) ([]model.OnlineSession, error) { + if r != nil { + r.mu.Lock() + defer r.mu.Unlock() + if !r.restored && !r.restoreOngoingSessions() { + return nil, fmt.Errorf("online history recovery unavailable") + } + } + var sessions []model.OnlineSession + // Separate branches use (server_id, ongoing, ended_at) and + // (server_id, ongoing, started_at), avoiding a scan of retained closed rows. + for _, ongoing := range []bool{false, true} { + var rows []model.OnlineSession + q := DB.Where("server_id = ? AND ongoing = ?", serverID, ongoing) + if ongoing { + q = q.Where("started_at < ?", end.UTC()) + } else { + q = q.Where("ended_at > ?", start.UTC()) + } + if err := q.Find(&rows).Error; err != nil { + return nil, err + } + sessions = append(sessions, rows...) + } + if r != nil { + endings := make(map[uint64]time.Time) + for _, pending := range r.closing[serverID] { + endings[pending.id] = pending.end + } + for i := range sessions { + if end, ok := endings[sessions[i].ID]; ok { + sessions[i].Ongoing = false + sessions[i].EndedAt = end + } + } + } + sort.Slice(sessions, func(i, j int) bool { return sessions[i].StartedAt.Before(sessions[j].StartedAt) }) + return sessions, nil +} diff --git a/service/singleton/online_history_failure_test.go b/service/singleton/online_history_failure_test.go new file mode 100644 index 00000000..8f22a7e8 --- /dev/null +++ b/service/singleton/online_history_failure_test.go @@ -0,0 +1,224 @@ +package singleton + +import ( + "errors" + "testing" + "time" + + "github.com/nezhahq/nezha/model" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestOnlineHistoryRetriesFailedHeartbeatImmediately(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().UTC().Truncate(time.Second) + r.RecordOnline(1, start) + id := r.open[1].id + require.NoError(t, DB.Exec(`CREATE TRIGGER reject_heartbeat BEFORE UPDATE OF last_active_at ON online_sessions BEGIN SELECT RAISE(ABORT, 'heartbeat failure'); END`).Error) + r.RecordOnline(1, start.Add(30*time.Second)) + require.NoError(t, DB.Exec("DROP TRIGGER reject_heartbeat").Error) + r.RecordOnline(1, start.Add(31*time.Second)) + require.True(t, getOnlineSession(t, id).LastActiveAt.Equal(start.Add(31*time.Second))) +} + +func TestOnlineHistoryRetriesStartupReadBeforeNewReport(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + require.NoError(t, DB.Callback().Query().Remove("reject_history_restore")) + r.RecordOnline(1, time.Now()) + stored := getOnlineSession(t, old.ID) + require.False(t, stored.Ongoing) + require.True(t, stored.EndedAt.Equal(start)) + require.NotEqual(t, old.ID, r.open[1].id) + require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true)) +} + +func TestOnlineHistorySweepKeepsSessionReconnectedDuringObservation(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().Add(-time.Hour) + r.RecordOnline(1, start) + observed := make(chan struct{}) + release := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { close(observed); <-release; return start, true }) + }() + <-observed + reconnected := make(chan struct{}) + go func() { r.RecordOffline(1, start.Add(time.Second)); r.RecordOnline(1, time.Now()); close(reconnected) }() + // The observer must not hold the recorder lock: RPC updates take the runtime + // lock first and then enter the recorder. Holding both in reverse order deadlocks. + select { + case <-reconnected: + case <-time.After(time.Second): + close(release) + <-done + <-reconnected + t.Fatal("sweep holds recorder lock while consulting runtime state") + } + id := r.open[1].id + close(release) + <-done + require.Contains(t, r.open, uint64(1)) + require.Equal(t, id, r.open[1].id) + require.True(t, getOnlineSession(t, id).Ongoing) +} + +func TestOnlineHistoryQueryNormalizesOffsetsAndUsesWindowIndexes(t *testing.T) { + setupOnlineHistoryTestDB(t) + east := time.FixedZone("UTC+8", 8*3600) + start := time.Date(2025, 1, 1, 12, 0, 0, 0, east) + r := NewOnlineHistoryRecorder() + r.RecordOnline(1, start) + r.RecordOffline(1, start.Add(time.Hour)) + rows, err := r.Sessions(1, start.UTC().Add(-time.Minute), start.UTC().Add(2*time.Hour)) + require.NoError(t, err) + require.Len(t, rows, 1) + require.True(t, rows[0].StartedAt.Equal(start)) + require.True(t, rows[0].EndedAt.Equal(start.Add(time.Hour))) + for _, test := range []struct { + query, index string + args []any + }{ + {"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND ended_at > ?", "idx_online_sessions_server_closed", []any{1, false, start.UTC()}}, + {"EXPLAIN QUERY PLAN SELECT * FROM online_sessions WHERE server_id = ? AND ongoing = ? AND started_at < ?", "idx_online_sessions_server_open", []any{1, true, start.Add(2 * time.Hour).UTC()}}, + } { + var plan []struct{ Detail string } + require.NoError(t, DB.Raw(test.query, test.args...).Scan(&plan).Error) + require.NotEmpty(t, plan) + require.Contains(t, plan[0].Detail, test.index) + } +} + +func TestOnlineHistorySweepKeepsReportReceivedDuringObservation(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().Add(-time.Hour) + r.RecordOnline(1, start) + id := r.open[1].id + r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { + r.RecordOnline(1, time.Now()) + return start, true + }) + require.Contains(t, r.open, uint64(1)) + // A report after an hour starts a new session; the stale sweep must not close it. + require.NotEqual(t, id, r.open[1].id) + require.False(t, getOnlineSession(t, id).Ongoing) + require.True(t, getOnlineSession(t, r.open[1].id).Ongoing) +} + +func TestOnlineHistoryStartupReadFailureBlocksQueryAndSweepRetries(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().Add(-time.Hour) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + _, err := r.Sessions(1, start, time.Now()) + require.ErrorContains(t, err, "recovery unavailable") + r.RecordOnline(1, time.Now()) + require.Empty(t, r.open) + require.NoError(t, DB.Callback().Query().Remove("reject_history_restore")) + r.SweepStale(time.Now(), func(uint64) (time.Time, bool) { return time.Time{}, false }) + require.False(t, getOnlineSession(t, old.ID).Ongoing) + require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true)) +} + +func TestOnlineHistoryDisconnectRetriesStartupRead(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().Add(-10 * time.Second) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("reject_history_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + require.NoError(t, DB.Callback().Query().Remove("reject_history_restore")) + end := start.Add(time.Second) + r.RecordOffline(1, end) + require.False(t, getOnlineSession(t, old.ID).Ongoing) + require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end)) + r.RecordOnline(1, time.Now()) + require.NotEqual(t, old.ID, r.open[1].id) +} + +func TestOnlineHistoryReportAfterTimeoutSplitsBeforeSweep(t *testing.T) { + for _, failClose := range []bool{false, true} { + t.Run(map[bool]string{false: "persisted", true: "retry_close"}[failClose], func(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + start := time.Now().UTC().Truncate(time.Second) + r.RecordOnline(1, start) + r.RecordOnline(1, start.Add(time.Second)) + first := r.open[1].id + r.SweepStale(start.Add(30*time.Second), func(uint64) (time.Time, bool) { return start.Add(time.Second), true }) + require.Equal(t, first, r.open[1].id) + if failClose { + injectCloseUpdateFailure(t) + } + r.RecordOnline(1, start.Add(45*time.Second)) + require.NotEqual(t, first, r.open[1].id) + second := r.open[1].id + rows, err := r.Sessions(1, start, start.Add(46*time.Second)) + require.NoError(t, err) + require.Len(t, rows, 2) + require.False(t, rows[0].Ongoing) + require.True(t, rows[0].EndedAt.Equal(start.Add(time.Second))) + require.True(t, rows[1].StartedAt.Equal(start.Add(45*time.Second))) + if failClose { + require.Len(t, r.closing[1], 1) + dropCloseUpdateFailure(t) + } + r.SweepStale(start.Add(46*time.Second), func(uint64) (time.Time, bool) { return start.Add(45 * time.Second), true }) + require.Empty(t, r.closing) + require.False(t, getOnlineSession(t, first).Ongoing) + require.True(t, getOnlineSession(t, first).EndedAt.Equal(start.Add(time.Second))) + require.True(t, getOnlineSession(t, second).Ongoing) + }) + } +} + +func TestOnlineHistoryDisconnectSurvivesUnavailableRecovery(t *testing.T) { + for _, failClose := range []bool{false, true} { + t.Run(map[bool]string{false: "close_success", true: "close_retry"}[failClose], func(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().UTC().Add(-10 * time.Second) + old := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + require.NoError(t, DB.Create(&old).Error) + require.NoError(t, DB.Callback().Query().Before("gorm:query").Register("unavailable_disconnect_restore", func(db *gorm.DB) { db.AddError(errors.New("restore unavailable")) })) + r := NewOnlineHistoryRecorder() + end := start.Add(time.Second) + r.RecordOffline(1, end) + // Later disconnects while recovery is unavailable must not extend the first session. + r.RecordOffline(1, end.Add(time.Second)) + require.NoError(t, DB.Callback().Query().Remove("unavailable_disconnect_restore")) + if failClose { + injectCloseUpdateFailure(t) + } + reconnect := start.Add(5 * time.Second) + r.RecordOnline(1, reconnect) + require.NotEqual(t, old.ID, r.open[1].id) + rows, err := r.Sessions(1, start, reconnect.Add(time.Second)) + require.NoError(t, err) + require.Len(t, rows, 2) + require.False(t, rows[0].Ongoing) + require.True(t, rows[0].EndedAt.Equal(end)) + require.True(t, rows[1].StartedAt.Equal(reconnect)) + if failClose { + require.Len(t, r.closing[1], 1) + dropCloseUpdateFailure(t) + } + r.SweepStale(reconnect.Add(time.Second), func(uint64) (time.Time, bool) { return reconnect, true }) + require.Empty(t, r.closing) + require.False(t, getOnlineSession(t, old.ID).Ongoing) + require.True(t, getOnlineSession(t, old.ID).EndedAt.Equal(end)) + require.True(t, getOnlineSession(t, r.open[1].id).Ongoing) + }) + } +} diff --git a/service/singleton/online_history_restore_test.go b/service/singleton/online_history_restore_test.go new file mode 100644 index 00000000..ebafdaaf --- /dev/null +++ b/service/singleton/online_history_restore_test.go @@ -0,0 +1,73 @@ +package singleton + +import ( + "testing" + "time" + + "github.com/nezhahq/nezha/model" + "github.com/stretchr/testify/require" +) + +func TestRestoreOngoingSessionsReconcilesRecentDuplicates(t *testing.T) { + for _, failClose := range []bool{false, true} { + name := "successful_close" + if failClose { + name = "retry_failed_close" + } + t.Run(name, func(t *testing.T) { + setupOnlineHistoryTestDB(t) + base := time.Now().Add(-10 * time.Second).Truncate(time.Second) + // Insert out of chronological order: choosing by ID alone would keep the wrong session. + newest := model.OnlineSession{ServerID: 1, StartedAt: base.Add(4 * time.Second), LastActiveAt: base.Add(5 * time.Second), Ongoing: true} + oldest := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: base.Add(time.Second), Ongoing: true} + middle := model.OnlineSession{ServerID: 1, StartedAt: base.Add(2 * time.Second), LastActiveAt: base.Add(3 * time.Second), Ongoing: true} + for _, session := range []*model.OnlineSession{&newest, &oldest, &middle} { + require.NoError(t, DB.Create(session).Error) + } + if failClose { + injectCloseUpdateFailure(t) + } + r := NewOnlineHistoryRecorder() + require.Len(t, r.open, 1) + require.Equal(t, newest.ID, r.open[1].id) + if failClose { + require.Len(t, r.closing[1], 2) + require.Equal(t, int64(3), countOnlineSessions(t, "ongoing = ?", true)) + dropCloseUpdateFailure(t) + } + r.SweepStale(base.Add(6*time.Second), func(uint64) (time.Time, bool) { return newest.LastActiveAt, true }) + require.Empty(t, r.closing) + require.True(t, getOnlineSession(t, newest.ID).Ongoing) + for _, previous := range []model.OnlineSession{oldest, middle} { + stored := getOnlineSession(t, previous.ID) + require.False(t, stored.Ongoing) + require.True(t, stored.EndedAt.Equal(previous.LastActiveAt)) + } + r.RecordOnline(1, base.Add(7*time.Second)) + require.Equal(t, newest.ID, r.open[1].id) + require.Equal(t, int64(3), countOnlineSessions(t, "server_id = ?", 1)) + r.SweepStale(base.Add(2*model.OnlineActiveWindow), func(uint64) (time.Time, bool) { return time.Time{}, false }) + require.Empty(t, r.open) + require.Empty(t, r.closing) + require.Equal(t, int64(0), countOnlineSessions(t, "ongoing = ?", true)) + }) + } +} + +func TestRestoreOngoingSessionsUsesIDToBreakStartTimeTies(t *testing.T) { + setupOnlineHistoryTestDB(t) + start := time.Now().Add(-5 * time.Second) + first := model.OnlineSession{ServerID: 1, StartedAt: start, LastActiveAt: start, Ongoing: true} + second := first + other := model.OnlineSession{ServerID: 2, StartedAt: start, LastActiveAt: start, Ongoing: true} + for _, session := range []*model.OnlineSession{&first, &second, &other} { + require.NoError(t, DB.Create(session).Error) + } + r := NewOnlineHistoryRecorder() + require.Len(t, r.open, 2) + require.Equal(t, second.ID, r.open[1].id) + require.Equal(t, other.ID, r.open[2].id) + require.False(t, getOnlineSession(t, first.ID).Ongoing) + require.True(t, getOnlineSession(t, first.ID).EndedAt.Equal(start)) + require.Equal(t, int64(2), countOnlineSessions(t, "ongoing = ?", true)) +} diff --git a/service/singleton/online_history_retention_test.go b/service/singleton/online_history_retention_test.go new file mode 100644 index 00000000..f505ba2f --- /dev/null +++ b/service/singleton/online_history_retention_test.go @@ -0,0 +1,63 @@ +package singleton + +import ( + "errors" + "testing" + "time" + + "github.com/nezhahq/nezha/model" + "github.com/stretchr/testify/require" +) + +func TestOnlineHistoryPruneWaitsForRetentionCommit(t *testing.T) { + for _, saveFails := range []bool{true, false} { + t.Run(map[bool]string{true: "failed_save", false: "successful_save"}[saveFails], func(t *testing.T) { + setupOnlineHistoryTestDB(t) + previous := Conf + Conf = &ConfigClass{Config: &model.Config{}} + Conf.OnlineStatsDays = 30 + t.Cleanup(func() { Conf = previous }) + now := time.Now().UTC() + row := model.OnlineSession{ServerID: 1, StartedAt: now.AddDate(0, 0, -15), EndedAt: now.AddDate(0, 0, -14)} + require.NoError(t, DB.Create(&row).Error) + saveEntered := make(chan struct{}) + releaseSave := make(chan struct{}) + saved := make(chan error, 1) + days := 7 + go func() { + saved <- Conf.CommitOnlineHistoryRetention(&days, func() error { + close(saveEntered) + <-releaseSave + if saveFails { + return errors.New("retention save unavailable") + } + return nil + }) + }() + <-saveEntered + pruneStarted := make(chan struct{}) + pruned := make(chan struct{}) + go func() { close(pruneStarted); PruneOnlineSessions(); close(pruned) }() + <-pruneStarted + premature := false + select { + case <-pruned: + premature = true + case <-time.After(100 * time.Millisecond): + } + close(releaseSave) + err := <-saved + <-pruned + require.False(t, premature, "pruning must not read an uncommitted retention policy") + if saveFails { + require.ErrorContains(t, err, "retention save unavailable") + require.Equal(t, 30, Conf.OnlineHistoryRetentionDays()) + require.Equal(t, int64(1), countOnlineSessions(t, "id = ?", row.ID)) + } else { + require.NoError(t, err) + require.Equal(t, 7, Conf.OnlineHistoryRetentionDays()) + require.Equal(t, int64(0), countOnlineSessions(t, "id = ?", row.ID)) + } + }) + } +} diff --git a/service/singleton/online_history_test.go b/service/singleton/online_history_test.go new file mode 100644 index 00000000..285f8f31 --- /dev/null +++ b/service/singleton/online_history_test.go @@ -0,0 +1,316 @@ +package singleton + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + "github.com/nezhahq/nezha/model" +) + +func setupOnlineHistoryTestDB(t *testing.T) { + t.Helper() + + previousDB := DB + var err error + DB, err = gorm.Open(openSQLiteDialector(filepath.Join(t.TempDir(), "dashboard.sqlite")), &gorm.Config{}) + require.NoError(t, err) + sqlDB, err := DB.DB() + require.NoError(t, err) + t.Cleanup(func() { + DB = previousDB + if err := sqlDB.Close(); err != nil { + t.Errorf("close online history test database: %v", err) + } + }) + + require.NoError(t, DB.AutoMigrate(&model.Server{}, &model.OnlineSession{})) + require.NoError(t, DB.Exec("INSERT INTO servers (id, name, uuid) VALUES (1, 'server-1', 'online-history-test-1')").Error) +} + +func getOnlineSession(t *testing.T, id uint64) model.OnlineSession { + t.Helper() + var s model.OnlineSession + require.NoError(t, DB.First(&s, id).Error) + return s +} + +func countOnlineSessions(t *testing.T, cond string, args ...any) int64 { + t.Helper() + var count int64 + require.NoError(t, DB.Model(&model.OnlineSession{}).Where(cond, args...).Count(&count).Error) + return count +} + +// injectCloseUpdateFailure 用 SQLite 触发器注入"SELECT 正常、仅关闭 UPDATE 失败"： +// 只拦截把 ongoing 置 0 的更新，LastActiveAt 节流写与 INSERT 均不受影响， +// 使后续 RecordOnline/恢复/清扫全部走真实执行路径。 +func injectCloseUpdateFailure(t *testing.T) { + t.Helper() + require.NoError(t, DB.Exec(`CREATE TRIGGER fail_online_session_close +BEFORE UPDATE ON online_sessions +WHEN NEW.ongoing = 0 +BEGIN + SELECT RAISE(ABORT, 'injected close failure'); +END`).Error) + t.Cleanup(func() { dropCloseUpdateFailure(t) }) +} + +func dropCloseUpdateFailure(t *testing.T) { + t.Helper() + require.NoError(t, DB.Exec("DROP TRIGGER IF EXISTS fail_online_session_close").Error) +} + +// 评审回归（P2：关库失败丢边界/重连续接待关闭会话）：RecordOffline 关库失败时 +// 必须转入待关闭状态——会话 ID 与已观察到的断线时刻固定、后续上报不得覆盖； +// 重连开启的是新会话而非续接旧行；失败恢复后由清扫周期按固定边界完成持久化， +// 离线间隔不计入在线。时间自单一基准单调推进。 +func TestRecordOfflineCloseFailurePreservesBoundary(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + base := time.Now().Add(-time.Hour).Truncate(time.Second) + atStart := base + atLastActive := atStart.Add(model.OnlineActiveWindow) // 恰好达到节流窗口，触发落库 + atOffline := atLastActive.Add(15 * time.Second) // 观察到的断线时刻 + atReconnect := atOffline.Add(10 * time.Minute) + + r.RecordOnline(1, atStart) + r.RecordOnline(1, atLastActive) + require.Len(t, r.open, 1) + first := getOnlineSession(t, r.open[1].id) + require.WithinDuration(t, atLastActive, first.LastActiveAt, time.Second) + + // 注入关闭失败：行保持 Ongoing 等待重试，会话转入待关闭而非继续当活跃会话 + injectCloseUpdateFailure(t) + r.RecordOffline(1, atOffline) + require.NotContains(t, r.open, uint64(1)) + stuck := getOnlineSession(t, first.ID) + require.True(t, stuck.Ongoing) + require.True(t, stuck.EndedAt.IsZero(), "关库失败时旧行必须保持未闭合等待重试") + + // 失败未恢复时 agent 重连：旧行不得被续接，必须开启新会话 + r.RecordOnline(1, atReconnect) + require.Contains(t, r.open, uint64(1)) + require.NotEqual(t, first.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话") + afterRetry := getOnlineSession(t, first.ID) + require.True(t, afterRetry.Ongoing, "失败未恢复时重试仍失败，旧行保持待关闭") + second := getOnlineSession(t, r.open[1].id) + require.True(t, second.Ongoing) + require.WithinDuration(t, atReconnect, second.StartedAt, time.Second) + + // 失败恢复后清扫：旧行按观察到的断线时刻闭合，新会话仍活跃不受影响 + dropCloseUpdateFailure(t) + r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) { + return atReconnect, true + }) + + closed := getOnlineSession(t, first.ID) + require.False(t, closed.Ongoing) + require.WithinDuration(t, atOffline, closed.EndedAt, time.Second, "旧行必须按观察到的断线时刻闭合") + require.WithinDuration(t, atStart, closed.StartedAt, time.Second) + stillOngoing := getOnlineSession(t, second.ID) + require.True(t, stillOngoing.Ongoing) + + // 离线间隔不计入在线：两行在 [atStart, atReconnect) 内的在线合计止于断线时刻 + var rows []model.OnlineSession + require.NoError(t, DB.Where("server_id = ?", 1).Order("id").Find(&rows).Error) + require.Len(t, rows, 2) + var online time.Duration + for _, s := range rows { + end := s.EndedAt + if s.Ongoing { + end = atReconnect + } + online += end.Sub(s.StartedAt) + } + require.Equal(t, atOffline.Sub(atStart), online, "离线间隔不得计作在线") +} + +// 评审回归（P2 启动恢复路径）：重启恢复对超窗会话结转关库失败时，条目必须 +// 进入待关闭状态（经真实 NewOnlineHistoryRecorder 恢复路径验证，而非手工 +// 构造内存状态）；重连不得续接该行，清扫周期按固定边界时刻完成持久化。 +func TestRestoreStaleCloseFailureEntersPendingClose(t *testing.T) { + setupOnlineHistoryTestDB(t) + + // 预置一条超窗 Ongoing 行（dashboard 停机前遗留），时间自单一基准单调推进 + base := time.Now().Add(-3 * time.Hour).Truncate(time.Second) + staleAt := base.Add(time.Hour) + atReconnect := base.Add(2 * time.Hour) + legacy := model.OnlineSession{ServerID: 1, StartedAt: base, LastActiveAt: staleAt, Ongoing: true} + require.NoError(t, DB.Create(&legacy).Error) + + injectCloseUpdateFailure(t) + r := NewOnlineHistoryRecorder() + + stuck := getOnlineSession(t, legacy.ID) + require.True(t, stuck.Ongoing, "恢复结转关库失败时行必须保持 Ongoing 等待重试") + + // 重连：待关闭的超窗会话不得被续接，开启新会话 + r.RecordOnline(1, atReconnect) + require.Contains(t, r.open, uint64(1)) + require.NotEqual(t, legacy.ID, r.open[1].id, "待关闭会话不得被续接为活跃会话") + + dropCloseUpdateFailure(t) + r.SweepStale(atReconnect.Add(time.Second), func(uint64) (time.Time, bool) { + return atReconnect, true + }) + + closed := getOnlineSession(t, legacy.ID) + require.False(t, closed.Ongoing) + require.WithinDuration(t, staleAt, closed.EndedAt, time.Second, "旧行必须按恢复时观察到的边界闭合") + require.Equal(t, int64(2), countOnlineSessions(t, "server_id = ?", 1)) + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) +} + +func TestRecordOnlineOpensSingleSession(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + now := time.Now() + r.RecordOnline(1, now) + r.RecordOnline(1, now.Add(time.Second)) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1)) + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) +} + +func TestRecordOnlinePersistsLastActiveThrottled(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + now := time.Now() + r.RecordOnline(1, now) + // 窗口内的重复上报不应触发落库 + r.RecordOnline(1, now.Add(5*time.Second)) + s := r.open[1] + stored := getOnlineSession(t, s.id) + require.WithinDuration(t, now, stored.LastActiveAt, time.Second) + + // 超过窗口后落库 + r.RecordOnline(1, now.Add(model.OnlineActiveWindow+time.Second)) + stored = getOnlineSession(t, s.id) + require.WithinDuration(t, now.Add(model.OnlineActiveWindow+time.Second), stored.LastActiveAt, time.Second) +} + +func TestRecordOfflineClosesSession(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + start := time.Now() + r.RecordOnline(1, start) + r.RecordOffline(1, start.Add(10*time.Minute)) + // 重复关闭应为无操作 + r.RecordOffline(1, start.Add(11*time.Minute)) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1)) + require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) + var s model.OnlineSession + require.NoError(t, DB.First(&s).Error) + require.WithinDuration(t, start.Add(10*time.Minute), s.EndedAt, time.Second) + require.WithinDuration(t, start, s.StartedAt, time.Second) +} + +func TestRecordOnlineContinuesSessionAfterRestart(t *testing.T) { + setupOnlineHistoryTestDB(t) + + // 模拟重启前遗留的进行中会话（最后活跃时间在在线窗口内：dashboard 只是短暂重启） + now := time.Now() + legacy := model.OnlineSession{ServerID: 1, StartedAt: now.Add(-time.Minute), LastActiveAt: now.Add(-5 * time.Second), Ongoing: true} + require.NoError(t, DB.Create(&legacy).Error) + + // 新 recorder（构造时从库恢复）：新近会话应保留并续接，不产生新会话 + r := NewOnlineHistoryRecorder() + r.RecordOnline(1, now) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ?", 1)) + continued := getOnlineSession(t, legacy.ID) + require.True(t, continued.Ongoing) + require.WithinDuration(t, now.Add(-time.Minute), continued.StartedAt, time.Second) +} + +func TestRestoreOngoingSessionsClosesStaleSessions(t *testing.T) { + setupOnlineHistoryTestDB(t) + + // 上次活跃时间远早于在线窗口：重启恢复时按最后已知活跃时刻结转 + staleStart := time.Now().Add(-time.Hour) + stale := model.OnlineSession{ServerID: 1, StartedAt: staleStart, LastActiveAt: staleStart.Add(30 * time.Minute), Ongoing: true} + require.NoError(t, DB.Create(&stale).Error) + + r := NewOnlineHistoryRecorder() + r.RecordOnline(1, time.Now()) + + closed := getOnlineSession(t, stale.ID) + require.False(t, closed.Ongoing) + require.WithinDuration(t, staleStart.Add(30*time.Minute), closed.EndedAt, time.Second) + // 结转后新的上报开启新会话 + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) +} + +func TestSweepStaleClosesOnlyStaleSessions(t *testing.T) { + setupOnlineHistoryTestDB(t) + r := NewOnlineHistoryRecorder() + + now := time.Now() + r.RecordOnline(1, now.Add(-time.Hour)) + // 第二台服务器：会话在 DB 但没有 servers 记录（服务器已被删除），手工放进内存 map + require.NoError(t, DB.Exec("INSERT INTO online_sessions (server_id, started_at, last_active_at, ongoing, created_at, updated_at) VALUES (2, ?, ?, 1, ?, ?)", now.Add(-time.Hour), now.Add(-time.Hour), now, now).Error) + var second model.OnlineSession + require.NoError(t, DB.Where("server_id = ?", 2).First(&second).Error) + r.mu.Lock() + r.open[2] = &onlineSessionRuntime{id: second.ID, lastPersist: now.Add(-time.Hour)} + r.mu.Unlock() + + // 服务器 1 活跃，服务器 2 不存在 → 仅关闭服务器 2 的会话 + lastActive := func(serverID uint64) (time.Time, bool) { + if serverID == 1 { + return now, true + } + return time.Time{}, false + } + r.SweepStale(now, lastActive) + + require.Equal(t, int64(1), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) + require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 2, true)) + + // 服务器 1 失联后也会被关闭，结束时刻为最后已知活跃时间 + r.SweepStale(now.Add(model.OnlineActiveWindow*2), func(uint64) (time.Time, bool) { + return now, true + }) + require.Equal(t, int64(0), countOnlineSessions(t, "server_id = ? AND ongoing = ?", 1, true)) + var closedFirst = getOnlineSession(t, func() uint64 { + var s model.OnlineSession + require.NoError(t, DB.Where("server_id = ?", 1).First(&s).Error) + return s.ID + }()) + require.WithinDuration(t, now, closedFirst.EndedAt, time.Second) +} + +func TestPruneOnlineSessions(t *testing.T) { + setupOnlineHistoryTestDB(t) + previousConf := Conf + Conf = &ConfigClass{Config: &model.Config{}} + Conf.OnlineStatsDays = 7 + t.Cleanup(func() { Conf = previousConf }) + + now := time.Now() + old := now.AddDate(0, 0, -30) + rows := []model.OnlineSession{ + {ServerID: 1, StartedAt: old, EndedAt: old.Add(time.Hour), LastActiveAt: old.Add(time.Hour)}, // 超窗口 → 清理 + {ServerID: 1, StartedAt: old, EndedAt: now.Add(-time.Hour), LastActiveAt: now.Add(-time.Hour)}, // 跨窗口边界 → 保留 + {ServerID: 1, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now}, // 窗口内 → 保留 + {ServerID: 1, StartedAt: now, Ongoing: true, LastActiveAt: now}, // 进行中 → 保留 + {ServerID: 99, StartedAt: now.Add(-time.Hour), EndedAt: now, LastActiveAt: now}, // 服务器已删除 → 清理 + } + for i := range rows { + require.NoError(t, DB.Create(&rows[i]).Error) + } + + PruneOnlineSessions() + + require.Equal(t, int64(3), countOnlineSessions(t, "1 = 1")) + require.Equal(t, int64(1), countOnlineSessions(t, "ongoing = ?", true)) +} diff --git a/service/singleton/singleton.go b/service/singleton/singleton.go index 8d41b70b..03885020 100644 --- a/service/singleton/singleton.go +++ b/service/singleton/singleton.go @@ -63,6 +63,7 @@ func LoadSingleton(bus chan<- *model.Service) (err error) { ServerShared = NewServerClass() CronShared = NewCronClass() ServerTransferShared = NewServerTransferClass() + OnlineHistoryShared = NewOnlineHistoryRecorder() // 最后初始化 ServiceSentinel ServiceSentinelShared, err = NewServiceSentinel(bus) return @@ -94,7 +95,7 @@ func InitDBFromPath(path string) error { model.Cron{}, model.Transfer{}, model.ServerGroupServer{}, model.NAT{}, model.DDNSProfile{}, model.NotificationGroupNotification{}, model.WAF{}, model.Oauth2Bind{}, model.ServerTransfer{}, model.JWTSession{}, - model.APIToken{}, model.MCPAuditLog{}) + model.APIToken{}, model.MCPAuditLog{}, model.OnlineSession{}) if err != nil { return err } From 05baa2f331f23e65d51bffbecc9b72c4c3596bfa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 16 Oct 2026 19:25:38 +0800 Subject: [PATCH 43/48] update contributors[no ci] --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 10536406..fdb652ec 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ add your theme to [service/singleton/frontend-templates.yaml](service/singleton/ Chra Mubelotix Ryrenz +Sam5440 zhdsmy ## Special Thanks From 29f1568988750c67e460034e85e8d35c3ed73e42 Mon Sep 17 00:00:00 2001 From: naiba Date: Fri, 16 Oct 2026 14:31:38 +0000 Subject: [PATCH 44/48] docs(security): add vulnerability reporting email Co-authored-by: naiba/CloudCode --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 9314b333..d932e53f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,4 +6,4 @@ Code in `master` branch. ## Reporting a Vulnerability -Thank you for your contribution to open source security, please submit on https://github.com/nezhahq/nezha/security +Thank you for your contribution to open source security. Please report vulnerabilities privately at https://github.com/nezhahq/nezha/security or by email to [hi@nai.ba](mailto:hi@nai.ba). From b329755836511da48d7b2af24968f275b0e41b32 Mon Sep 17 00:00:00 2001 From: naiba Date: Sat, 17 Oct 2026 07:13:01 +0000 Subject: [PATCH 45/48] docs: add feepay sponsor Co-authored-by: naiba/CloudCode --- .github/sponsor/feepay.jpg | Bin 0 -> 24717 bytes README.md | 14 +++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 .github/sponsor/feepay.jpg diff --git a/.github/sponsor/feepay.jpg b/.github/sponsor/feepay.jpg new file mode 100644 index 0000000000000000000000000000000000000000..cab5c15429a5ab4f4ba562ad05ea5191b7784f81 GIT binary patch literal 24717 zcmb5Vb97}v^Dew|V%xTD+qSIQfM5 z;b7p95m8W(5s{Hl(Qzth;i`n@&3~Z7z89F6eJV^ zG&BMpDl#hG|8My>1farz0zrY`U}OLY6$~5|?B57L@ZX_=z`=n3D_|ff=+BYDeEy5~ zpLu|Pe*T|;fgm8kq5f?F2%l|0DBvgn0Cu%vIbUQJtX7#yLKY;`sH>t$lOP@Q>UsYa zS@rh44(aF7u6bM>lf#`q?@ zOQSe2ut_yNw3spm6bvWHkWOS|K18H1`0^+$-~r9wwsDPIAYYMH)}s0N5*=RI0;5G1H5uU`ws}r@E*KzWcww zJ@BD-bQz02YuUL`ealQy3G8Z&L1X|hG|N)Ir=HKo2{RMKkIx#c8PU^)V5-Scs+v$1 zilm|xbJ78kl{7^b=|QOQ$!ep3h-7kKf-d`2yz0w(hnKPVw55&PJtsotR1_y%=~TL{ zs#Vp67vI-?_0gNsU%n$v1$lJxB9yS0A`tQ>7^PxiVbNK3TxqD5g=sP%T@-^rkUS&| zEGOpa4aG3-@iB^U*JO(TS4II2|-s5^j#>8QVV1HWU!%Z~aTs@a6Etsh) zj6SVQ(3iyKD9Eldj7G|2(4u81XbMG5rw|MRw6mJ^Dl8b$;qRxEaFx%!>{VX={b%+h zRUsrz$3n}I+kW4@PXIf7Q!yGw45tZ5Rs^Jrl?uR`6j8>Z+8xfAm9l; zZiF?V2?RdhH;LM^#x~2q<^h^Om=tk#FP*aVFB1s?z5f6q*F2(ryV?^!i0mYWvYVGq z;)VpMKkeUy6d9)HF0?qv;6OT9i?SG!WGWCkt*8<}A=(s>jw(?ICu2Jm7$pG*Wm(+6cVFln(MN>!SRq%7H@Wo2DW zs=nw%vvb1sP8N9iT~+>$r(Qj+cvCO(dld9=Uwv7HcP*zi$)J>MN0BQ`pVowo9ZN1& z8&HX1gfZBVW0VU-#X3pVOD7Qd8nWggAe|kDb@f|^ z?EiZB_Nt#}r6bj%C@E7+6B?bWQ+&cWv68r?0?$xHHfT0fR-MWqlV-w^1~5?B>(*{2 zNvv$+l!~?FO=hf|=JIFUJv>t`uj%T-;&txYdA{&kj(+FeKIhEIE}~&f{{bnZLpP$) zPRfu$8Nz6XXD(|gY9X(S0iRT7wpNN(q&BjmDF?E$m4|h)kvwhXPN`$4s{;wfZq6h+!xO-MY$Hf4Y&7$(O!%6| zxS!#**4FEZpMNn2S#Lc2{?6R==o2tpwYqQD+E%o4A;kR(J~V#s=%ZfKZ7h7m{kuwN zo%9QSH|e2wfk zaPF_YfS2XmKfj(%)t=VJS1p(EZrsH^krn2jss&m)-aEt?!1MyZT4m$Lv1z z?4wWl3`g7xF|YmcCT@N;|C*P>tbA%i(XP=QgOWg)mbXSvNK#pqXX(hCL(pLA`BA~G zED&&XyVJ)z9cpLJFzG( zUYFhVi60SmL;pkrrjM;l!?#e}`?=ox+posR&>jyN9!LHGrv(fB`R@ z78eh1@`rx=VUw?O;M<1s+y34C%=g!4%=6m@`-WA^FYDIs8J4#0{L(Ql@ywI2{*h08 z);>S_CEtbWeluZjx(j1m#nX1o{z9he{GMQ3>-obEdSt5WlA*F#RfRczI$kcFp|G93 zvisB+I{Uu&vxn}*wS%5hqVVy4(8GI!epy?#RbSk?&_5vj`m8_hmmd=KroYo8|8^;WYpC@AraP_iHcXjQW$W*x-mbsDeB&o`Z@}+-l2&-#BT9hs>UA|2Z|{0ODd?HM`u1?~5BREH zv+=z__mFMzk8 z;$GM5+dSFd_34*>$(@cmzW~C$$v=6b{SHk3fFlCplcNI6bocLuj~~)zo>xy5Nq2t& z%-4Le=P#eX(1ByO1;)egnUqq3-iX1?tvI>;Ub10g;BL4H>o&Ecl|DL|G-#2_O2p^X}t(!GdqpS*xEz?p} zFi{bV|G4zu{rN6lp6n@Z8y8$<827#k^UROM)w`);z)3}=7vbTg*?E~~ zxce2%KZ|#5$#pSQxtg6e@*umPKk{O0KP3#GdU&(z&D$Mgu?<23Vs#1|9I5b- zWP_GugJLT309%=gTne2|VF?|L2&hsNYoJgB05j*bm4|Y4et6yey}lbW@hWT8#z#my z_LfubBh1OrV{pk(S{Z|=tEMQXCZ{J)0|)kbr;mXKDUVeqnHnA0D4*TIrH2AhqopAa zV5tF+sf{AKoGC)%A-Q%}9?Xk8{n|{9yhJ=QbAA{; zF7|!RyV3o|>yupAPD?2ldx4_YJQ|G670Q4Yh)Id|#j=X7TDDAV8e|o%f(2+`Bw30n zREky8M9VViDt}%5P3UlCczr4GwK6eFo)fUE??EQY7tY;DiTC)>vg2MJy5y5fAAo?Q zL;=$2%2E|&DH)?lFl8l3?a3G*KeaX*f^m{#(!PqGIbRA>+w&U)?49 zErUxlDy~XhZ>;9T?dzE%W!v0&Pp|ujL+vGlLrk0qAT})m20%6!>yUu9VJ*YN$xr|+ zmEgig0L)|wtxz7zYQ}Zl)|YAZ^qURud#X$+T$G%yyj6V9rSf8)`NGlTdwd48{na5> zG6qT+ibk=TBs{#U2_T836BbzmucFZm3CIDopwz%9vq_7oJ7N8DC^pjOsEO}K8 z5)2S5+V`F=cfO8uYVtGOuB;?TRS|%_C{OGhwsy-s!}ufi;2lX=qjwJ}16fwbCQk803qOe`T%ZJMz`O1Oq@%iCTb8SMhsYsGSIE)wo3!Z_>9n&&~ zt~g38i4Gn}R+F{$Gw|~x8hQJhlecsGLM~ocJJezV8gx`}2rG=_0MLeXGPD@B$(;P% z{RH|2y^RaIw@XG2w)PWTv+7JU@_$CDbm&wSnZN^pjzl^em?`5#6}|mRW%P`G_Bij) zx4x`~{$YORZCCzw>i8*$bOyBAO3GAts30)4&@eG@x>V35Cnw1tKldGVbM5w2hrOlLKl^i)^#}mBI;irZk*vxll9=%2q ztNSQuurej|SUPkXdSub5$*F-T!DT8m46^j(_FJnH51tW1-hbQzM&7aSo?dogelxMR z6pa*!FFdcvx(>@IO3<^qJ8?~N%ti+h*<^%0dr_y)8^68xJU@C1n$Ntzs~17X&n7b_ z%j=fSCUMeQgoQ!GS-VUAhg4uf0l*-@z@VT$DT4paEI@#Pz##!B6jU^H5fuzf5*7{< zEK*i>GB#0)PbvZClVJdZ1YH_(*iT+TMNqEHD5R*){YpN_VjY$Hd*pKR_{5XO+9KFF zjYd9kJ!K_bwUY%^7_1PxK{;ahzGqGv2D9SK4%3%!@1@qyeSMBkzL}_xiiBL{0`ELA z;%c~KOiw=4P!s{zOJ?s|pX6O?xFMK@V$vN&n23vTzD-5OgEA|%c8GIL&iz{kd~k9qj~6pRi%eE`rDW2Z$lUFqDfPe zIUBNctoo)t6o2ZqneC9qObH-IBrA-~opB;nld(Q_#=ALd`*xEiHt|=xCPrF&$GeRB z#=5se325zU^Awt!v9=6&BEE^tCQ7ewO4;$b$eJ{5Y)zQIwW9ZaThnW7cPqmfG;eD1 zYPd#KI3w1yU)o$u7MF}48$6lZD^6SZUcFk@Qk1PVNLZE=oAOV=(*p@fr%blwnTbI~#^}-Kdx*64l*^}ewlBtGpu+fZ@``Ra z)S6C2#cb{WtFKY_=_-o`X$D%j&w#fM!XWex{1X$H$zk38~p zrNV+t_|JG5SjCgyHn}66iKD4X*qdmtydq3Oqd94fVIwc#3e~;4r2lyrb z17iLG*XxeI+XBuTyS6cu46mNZ!|o~{B&Oq*;ku}@l31dfp=b9(2QOjB$(NZum@@Q~+u(9(b>=b2mRBma;=P3#sI;wQyH=-C3loqF|{)I$_oE z9#{F^8H_1|#$I-KPwKDb$6mX?lM^PtnANUaIXT`2dLsKG!k0ZayG}lGm%((|d}85|SEQ;skDLy|Xb@<3md~{cgy;M_!u~lU!B9 zW)dt(qD5tLlBG-S+n7f)mbALaoy%#FCP>__ajfF!MMqpnHGdd#aARayD2L%`f4AW= z%w?Jtvip(2a40)sVIUEPOkw%C1Dz-H$F>G9VeKA#tp(LU#v14a2` zQtN9hF(=9-N$c?ajIC2~{m-6uw`O~kjwnX2F9x^)H|$D;(?&$caM6 zITOUMeN^w;q+!bfqADJOllX=Xc+5HD`3^*0JuJl<5hvN!Fj)+WX#%~zH{hp%#CQY@ z21(qFUs7gs{s9I~EWEzJ%K^ze7?d=+nZhdKKj3sZfFB_xU0_aoxKVVlBPN{SCr+&xusBg#5gPY~CyUjBeB!+y*j4o&U2y8u3 z-dd~{PNIdshfC6}UL?AH-W`I|%stoyf8%G6yfse(kH|17>K#v0=Gmw_RNbG>_up1KpgQ6CBFd;j(G}e}l7mCv}N!{CUjStt= z1A#p%xs?OKxBm}5M4oi6Y_9}e&K=J-5#5#ik$@~Ar@pG?1BM#Xq@}Lm8A2GAGpvoO z>8dJDvQ0+uV0nNU7)qQg-p>`3I5eyLmffgiiA_DL9Q$}hPOh8Q2Pb;HSh0hkU)&i# zIQHi@DHPL6&o|0+9_Igs)J957QpjXEeT}88*W$|_EFDZ9+0;|Yjh9SN_!zDWRqXf& zfLr}qwsxkP7W3t)|Azl++yA&NFdG!AbiRL4+vdrP@+aDH|3qh7f*hW9R6f=<^CD+# zVr?Qpv3X`9`Nz5S{QGi&FjYzhj#d)(S-O?jc22$jLY$}C`j&(_*EGASgI_iwlhI$o zwy!=2*1x+d<4zQpR|CqaWvKT!Yq6((k38{Z!d6v;qpGwz zcjr6wa7{~l`r0r*c{%Sr`VT1QJ;zsMlnK!MTsq9#<9~os zG)}sW2q82vCuz@mWz9c8(g=fR&#t?V_r9?qf%)}8&xy%CQ)3c>Vy;yA+VLqcud#Qz zE6;)8S71K}K5Pr2Y05LT*vQ_+rW8T|D>3tLd~(4T{9)=ZdtH6rO5E0XKYs0A>q$nB3!CKo42^~S~aFBc3;YY6&F_W`!a@)S645z$AZZ=KnSxNJe zQ`oBjmrMWPCxp!UQKm%E13s+$V0xfN1!IASY#14-b1%_2`Je}P1@*@L*lR3V1*VYA zMPdJBG?JEHNR#3R2V*FZHrUD|*fMOBm;xT2aRw88TvGe9W4E)`Mzt>nzvt;!Y0ar1 zdYIs4cfNqe?tXPmu0@hm68pYZN%BdNPTvh+WM3RRXntm};^SzG1^um1%quw@A0nZ| z2Tf{wnSp)}4D3V(#~w{6ZC1LhjjqigI%e1Il7jU|;k&TU7gHI}}Q7*H|4V?s8|vMQ4G_fJ7JS zo2@zKRYQ?)HP2$~-iVA!YB~mTl+n0U-zScVRb`gJ<)f;8nw9Qh6ZHe^RaiBOo%kPJQ2=c-q)dI3W!UKCAW;Sk4fe<&t}UV-zAtT2*Q z=j4$l3&vHc#AcE>L_J=J%;}j@!XUfT>n4tC++qbnIvG+xr>dveUfEw z_?Ix)qalO+mHH!8LYvUvh-UXF!n+Qh;-kZ~w9fK*avzNCJEZgM5M?BVeD0!z1cxaK zqzk)R>+PxJUkQ>UuSFv?zhy=-PkJIqZVty=^S8-=l{LR$Y54`J&i#4Agk1O_2%2y7 zjf?e@8y&jBi&v)J3WMxmx!;s#xBPvn$Zv{EXNQzo@_WdlKK7<&%f6Rb`BbE8e;teRifBV8b`Uc^?uuE1YPTp)=SjD zv0m=fU;l7e1*NID!`~Bk-?S^XxDpl@3v9N0lPAbc>7+Ays2?0Wki=@3wS@8SEq)P~ zRdCDx1D=PZf7!Ss?MkD+syJ!d@Fj)$z3F#6JW!n4_E5d}F);W{5I4Q35zYyT#Dw_N z*J$h3)UK}CUWDyGs1rFKo6FK5AbX(Qo3yRZL3zx7m~xGddub;lkxaa`7(Y&kr20(lO~@+Ru-Ki`^B(2 z$C)0l+IcMbxw^=i5}Y0U9H^lj#7_3Xhv7CO`;pphuVAHV>#Q_W(;7<7sH1W5xI2}j z98HByevaknuZ;*N`;9bou}eL|XIKkDLER{`=Un5eZ*F)sGqfp9tkEsBAk54$h7G-b z&Y()49f8-D1)AG?IRLJSCoD@^Ekl4V{q--VH!6jfo;~qS4SFPgsynT$ro-bHT+^TE z<S4a7#N45E2Mx3Hkv86cK;c6~XaWdmqO2L>Y|(J(}lf9=IHm1%e+3B&t+u4Wz;{K zEuhMQTDzuKEimUw;8DWix%-7XHD9Oq64W`=_VsUpkS{beZFc?`RA8-=JwNqA>uj^n z-&fQQ*Ty!Se?a3qiw615-x}0w)6b^woHS3;^I;nv!Vhgn7XlDaPy&qNt8u6yuPm4_ zTpiB00yT;D{K4aAVl}8qXNV=>lMH5Bhcd(XjR9g~jy!O+gj8O2>lfTEp?|l2a z(;r?}n)53w(%&nzy`~evaULAhKV~G%`+I9tCR~0l;IRlY9qLvTR8_XdBX`#*^k_UG*{{oi>a06IXFMP^N6D6p>{4zc ze@2p*#zXUGTPRwGxnUIG8_&wd8Ke;f)iV7o9tU^na#)rHTznm^@pxihs3gcQj0SGX zLR&J>XkwT6$^2Xt4rl;)0C6QWVPAVdlj)L$J!c*-R=o@_8;nc4$!5NjHM*Q*g1N}@%a8GEd69xSKVZ>b*Dlk z`<`A)n^QK8nX|uq%x#_WAo#Sqq>PH{Dj#ZP7XYN=j*4!t4g9$qT%eWSK2-@D0q73kVrQwO?fF! zIh~Xw%1#EXQYqrWuk^9gJS%v5ZYuI;UMNnxJI2euQ62Ml$BU9kt}s-wJ(oAAlRX+o zJF4d%luh^$%E3<0V%mE3HfAM1^L2iW?UvVFDy_MceNvu4`iie9$NnG!l`9pWieM4D7_jtXDwJiX9V3qxA3 za@Z#ebE-RTvbuBRJ$R)Pm$=m|T75NmDaWEG6I2>Yyi64*EUABfqF1mntEKOXnwQnW zq_sQ8>k~YYppve_-ijK|6s|%U$CxDXjHktT+aI&>@AbIqe;5A7(U2lw`6VAkIHHbw zDvo)nijhmuvVL{yjVS-od9T{Y!qAW%&dfm`dyo}-G%Jl0Yab(uQ93G^QQ?hKA!o%u zT<4*lh!eP?p|G$=@itKy9{%zAeC)eM?IzvcOFvAfD3#9Oo#zXc^OQO0Q7@4BVcLcv zWDHRfRk>SPayRNk>QVWscevg+mfd8$eST-CSO(wP1>W|~CI5R5=X!1HRoAdkk81Xh z$UD-jK^DV$0$E{-N<&H3D~M|L4d)$dnXuA$p>yrRhO@v8?Tqdh>AH*h^qcncb+GM5 zp}qU^Y^mDmd74vleTYn{*=wY2AM4PXtc|2wB2yG%K>kd3+x8d zUk||~0FY>QkN+(nn5rv{>#Qb^%iH6e^_VKoks#l@EU67!IY!|cY zlqWyX%UwJBKXHK-1^Co-NN8|y$p2&oeF_)|Km|u3C1XV+VG|KmF+nF+WifSzU{^Cs z5_1XqKfE54FzD2^6CE0nCeliQika1w)tpmU9Eu&9ojxn7_8Dbn(@I~Ig}mp~nbtCP zI^C+3ok*P=l_y3{pJA|yE&Ca-N6p78cEUrVCi~t~p)RznF@HOLTVD#KHl&&Pgja$} z>kyfx1HNT@l43QCt1l&(-fk5R*)c;UHQszgla*~nfwZj8CIR~zrOyT3+QvxgwzY$ZxNNXW~UxR~v`CX!#Q<_(6K-~H@;7K!n@k+x?PO(&AZMCQ{j6q2@ z!L6DLT~CeQlbKFsM0okn zeAwj_UE@Vkmb^WrnFA0K*)AX7;n5QU6MmR=Kfab}#}3V|5A@-y2W9$kdBn~hxZ_1N zp0d39CliDWYOc{Y7#C|em2ad6%ZA96v!z!{zF~=#JBiJdAop&9$~D6v ze&p}5^Ia`MTFR8pEN_qbR9iRG%MNnYVZ*kW(es#E8Bp?F`OxbQ2yYmss`$P5m8bC; zKJ8N1tq`z7R_&C)4cnLobN~SK>w31r|*j0G=$JFYNU&GtUy<& z5j>P_v$FeEW5vSly;gGTv07AG7QAMqE1p(w@nvGcSc@Z7z17N8u)VY9i)jUHAhQl( zxU0Ltf*kEuRz>^JbI^LJ*5$YA?wH|#iBy!P^G}Cj;#fCZ#c2n~wCf2k*BG{y%3ya7 z+aR!p&Bi_F^iZ5@4W56KP+c&vPwyf7VhtOViZ{www^xWa=+)Fr6@2q#5iPRoxP3wP z7gm%WF)fRq?B+~C8(VNMh5&JM}=T^#=iD#szbK)Fk6w*+wdQ-UHPNs zY2>}_A3*&(N>c=JjQD-fU2jHXep~m=%zqPrHb^zGy$pKIpm@qEtXBYcJx`~ z~cm%0c@J^6*`JK|2=iCdeoh=(c!CPp3h?yPo2X`NezQ(s|% z>D;@nr3uM8duU2)H=t>bZ+AZy(s#9W2oX1;IVP;Tmm*tOYA2P|uBBE!m^PhN;dSv8 ztXAvIX-Z%en|VE!APO%2ne=b3_q&Br4BADVc`NKZ>oX}f@v35KXGy~>c zmpn@+=D71lb~a_u$XraVf2IH4*@o0=_ADCNR>(GYy9Qq>j|9w9Ip~+ooR)qQRA$%C z;nH(0szZc$A0&Xrld71fpvUAIwZ4Kq-{>G2K5D(Ns!098uFg81=zM>8;5+z5KoEM5 zIO*_YVazy*Q0N1XMX{EE1MQ&%fet4h;?O7Ef5B1I2g;5CQ{l zwi_HvM;fqvu#c8rqqHtLGO+dtvfpelf5yUU9JRzzZ#m>B9+25`oQ*vME3?06D1q}( zz*ut7w_=N?;a6hEEJ|x0Q8+3XP^2N@>TE3r_`h`~w^KQ-C=A5gk>To_i}HxFBOOQ9 z&ZKPevm-3C;U-X~$>u^RyO-1JCW)N<13oP-J<6x&4SezxAh1u9`(KL-prVlc*AG`U zbq-1@x>ac!T)a6n`JWh?FzHx#K&{Z_iT__*Q_};>3!+nm$F9kJ^}ja>G=9@w%3c2e z*NBx`CjH|G7{56fzp43+yz;3aW3-58@lV8X;v_#D>MzH=;B#4@`AcKagFkI9-X^hH z;C#G#FI2IIr^s5DW6i@N3^xapF*HYORm|6;#uruxsWt43WxvQ7FQ zOJ2-7l%4~6!6<>Lk0>2g9?CeSE@~%iY@+xUEmvr;s2A#OC`kFG02{T&qoTfKT%lvg zLcx(aZ_DW+DW=}5wfk<^qOFsv%N;=U=1BL-h{`tCL5n`vrU>W7%_M9$Ls<8+@vSpE>;3y-+vY?w8J1qR!irCYL_@b-ABhkNonnP>k4z;gPu(bv3)Z2^&Pe z*q%BxVhj?P%=t5ST(rE?a-Oe1PbtWtOEsu1^>a|>vP{PpLs&0n zN{cG}w%kdzb~>%mp=yaQfq47A!n)O3a&wJgcnE4+&|)v~4YdgL;NWGobQ-nc+WgF> zu*@(bf#0r~@0;!60#UQTgZ*2yj^!o}%)RKe)IH<%26-q);MC$22SfClie_i$- zu_m|YKF2*^rr$cTt@u>-$z&!tBRba-C?L>zeB9MM!3_6(n7>YmN8nvXx=D$2D%5Qy z714ND=cZ*2s?sh|@s0iiwCdW4;R2aMLUISE4wjB6R?lni{{e#}=N(@-lyjgt#YpH& zC6Aki+6yi)3nC*)_{M?B4ZS{TH0wi(=FBr}M*2gCant8> z)wj-7dyu0ZW{|BuN;0ONIqOVl&+}(#1UUJKUDe(w&T$sbuNh1RUFBcLT#LS^ZJGx+ z{;}@H^-OL~*Y?FAnoeRDjx88cK4cj*2{@8q;Ld-(#t-N2(T`VJ{yss-K@|uU=qz~H zko$F~7m;^5U(s`+K59+1+Xqz-`I0ZRt<}EF8V3A^L&bluH4QHo!J^9w-?Z`9h zrNR?^;fj*B@LBlexIee!F5^S}gW>jLVIMj^`Hm;khKhtvJ~b0`)obF1f?KvC zSA*j{y;UQN$2B;8%6AX;>~HyGG+-l7;$ib6|A4N7XZ8-e;nqgZvsC6~(Kcb0Rn3?5 zsggcLEI#&F-j{SY&PUohY_~9fjLP*cIXhu&B=SNgrh+ULlr=`At-@f^U-07lI!(r~ z+TMRcuj3Z`Be4?j6NOo3RYtJ8Od>dqQcvB^$u^8x=ATcsffkK{dis*q^R#8If4Ls04C3*m(UkToi$^476mF z@H8Q2-Q8gWL#cHeyIQ=tyIS9R=qvs>9m|bareMzS@*E~{_TWWPEHapnBB-(fk;58- z(BQOcaTXsOwB{Z>sa&me-Mw}Lz) z2N&*xzMGHjd}jxcpF#>3SbQME1z;oPe?x#_#zfH^e-Sl?kf$THPM$pYtMU&R&lWyP zbMZyb^w*p#|3z`hK0si?MN}vwbkL~YLApB8uH)oQ#SiBnE@65%8jJD+Ni1-W$opQe znYv6kf3Q~4?$~dl`W$Tuv08LTDwobkG`efS2-t3XMz!vgXL^kh?^ZCh;#{!EWWUft zO^_0iRElQE{?OUFInuW>$6)II5wIGy$FUxsWZOZ%a$hIjI2A+egl<%I!H5AvpnXyX z105at1v?gA61*#V`h*qaVJ_BTGu0ZQzX*i_9VL&PnoSj{$}`vm-OQywr$&J&f2vk8 zL^wv7cB4W1xe+>wV}>4*SzQq91~gt)*oZdx?VXsYonRIcYO$v?N8$yBV@I4$1_rk> z^iWsZwt1oa8WG1`PT9&Np^6I&|4iiZE(|T8X_+kT-4sS-%4HB{5^ff*2Xk#2wLBVd z2&+2+`mc`RT5glM3_#W28`sf7o>rGLTslCtAIB3bI21c=?!z#GXt zQHtJ?=^=x~?DEtqUs;A?UqizPj-D}*>b#gik3r+5!3aJ~Nf4c8Qhvd-ZwW)Qcsev& zajCiwy5}c2uD0#w`^Ttk>2l%G->1NK|KO73X3V)+#{BfjK zro|bQyl4Z8(#>Wc3?FWGk{ra=G~Al+3-Oq7PLv{+3bdMvuSLFJ*x8?L6g%PVYJWKZP3VnJhDw9%jqi@uAC>4JfxvPwBwU!f1 z7_F2x1j)NRwdBWcs`N4GGb>=LJck<2FjzF15Dujgkl1+ku$cwr((;>Xjkp{Q>Z;T7+wI5^!KUV5 zkU9gRcF*gNP5lHN@+2+H*L1yy7k~)F0iNb1o~d?hX3GnNw#B3!%R}9HXU~|3a9RvQ z(jw+UHZ9IR-Fl9M48kt-HM)L_qYvprkQ6~ECZTi-AVQO!5two6H{8F4qA^#`pjyFu7|9` zZ8lLsuv#TE7TAZIY`h3|_!;~M4@~#%^HEN{PeB#-25m>OrSyx`zF#>Hesa|tjhY+6 zhskYOj})#5m>e8;sjd))R$myOVYawt#>cgvL%<1ec}3`i`Vi5cH1RRLNewS#T@zVq zHLf$+p7b)47=2k{FeM}i#|~NO=~%?w{0FdrRe1(_i!19GEe53%)tiZnw*<}bXsw>&0gMqSJ zuI};c1}Y)z>R^ZU5`4ebI6L#~vEA2zRkk-~*{s%n_03?0u%Nc`QH78Q;%8gJ`?V%s z#q||CNcjNKTvP}cr*|aD9~B(rxDfo2NgXY9G%~(*Au_a&5k<*{k!ARd z^t(0AcKID$ms>TVNhtPyFD8S&cB5~M^E;jrKPzzsVRp0Ma=4i-r;a=g*o-DzTOz?c zZ9u@Y`t5Vl*!>>U@9B`>#*A6z5HbqA@3@a~hn{(iztR_sE77&RWB}hAsvHv~WU6-v!ZT#bDe?q5G@Lzd2qhh_cLEJJS zmcPe+F&WI1KUf+4Q>ksL~U`9MQot3o$}hY<8iT>AV512_u- z^Z{m)2@!x{I%Plv0LI}eHM|f8P@Dl91UN|aM3&E!ZUBHx0yx#xi=Yf52yr5Cijc5@ zicqqH6@!(BL?=kfL;wmhgECPJXUHIuQ5?W25YH?SjiLsCynuxhzadbol;Fum!2s6Q zz#swvPzaO=2nfUo1Rw>cg494D2{jsr-&kP98l64hzc(Nh|vKcC|Upj_a?K5I$)vN8ihQElu-R9mq0 z5&=eZoi6wv^IzWED3uvr@hV-YBsNp09!_}%dH52+SHx9*-WHuxz&zS|y`M>Lkz1iN z88F_t;h4ZFECCNG>y#*++QLY_uc@e#@M+?em250dbb2x>s?o?-g}uagJWdl9 zy;8pzd)2jtBzX;@fK6O|Q(X&A(L0a1vs6KipkfyC`|YEG8&pArB$uPA9{nq(V< z0bUb}zOyQ-4Qd=1>KW%@+GN@iOpse{=SB(5`l1AYMJI}M)>Q062?hg&IO5@BLJAN8;DQoMl!y()Wq_S=4)9zHP$+J3 zU>FO4(kKqYG6e%d=^_W&v9JkNGAN)2LJoN0l&AzYnn;oW0Luh`%IFCvhbaq0DgYVi zumE_dR9tdkGl{4UiWtl=tQv*QA+l3B8ITfb7z4&l4zL6c6&^~kpe-|)(O?LkVuG<~ zr^6tSzyY98TChY4kw{Vyd8`va!4fF!ge-x8K|)LQYVrjDBM?yml2oSv1{7EbI4}~# z638o301W1?(5npN69zB>5dbk6PpttDRu}-?g;4F31%R+v>YW`4Bmj@#i9`fD5UKli z4UnLtPUS+#-~j*(B}aoF$`1sxg+M0}2-md4;Xwqa>Fk21Q~+Fu*|&r67o+Ly}t(m)=Bau*5`wK*}P4BnF-|PzHh$ij*Y~ z(ErJZrPpg}QJW}L;^Eb7pU9#I&4V0B@=2eOqkcjc=s!t5p#Kxd2Eb5JNJ&^(L_gWH zq@bdv|I%jvN$)|S4(oMeXrV2w7({F*Ix;8BfH5R4=lX`1En-~G?xcjCO-$%RZv%Ik zjq(jW(np_GO6`y>E-JMoI=tH(+YL)mJNPEI`plga!FJro8>cw6*f+Cr7Sjh0@!b0` zQvP!6PY281Ey9Amy?vKX=;+Yw5X~$fGj)+^rx`VN9v0I(;kb@Lt0;o5mrlvgeMqh! zsb_NqrW73`K9ha+KYw-IN_V<>FpJmR$Vu=5u-}9-gL^p0ac@ zz#>9BiID7rI4dWHZl&47B83q{4>^SD9E#QHtrBby!+Y|-WQIoxJ!HFUVug*Fx|8?^ zxQveo6r)FtM;{6ZgNBE&#Z#GtT_t|XK-_YqA)|PSkB}t(g=eP`yJ|WXM6kR8Dg6(5H=%UMo2X3pmvl5!l8 zV~Cs&nPF(5oaL}Fot%cG9Orzjp)wI0swpyu5Yf-~cl!T+-`D%P?)!S&uj}=EK3}oc z_`mPRe_m(DBB5qQ&VCp;00|9*2&T2@G?=t1*WPRN*XQPRamwXxhPrXIPYXy;v+ICb zDGG-WOPtVRX5f^ugm>v=E(2SaDM1iF@eiHu%IN#NX7WRf&mWjAD?GooImri{NS4 ze#)6h3EMf@X7@jOIiH-Z<>|=Lbi-(D+=Gy3tF8hJ{s`fht1cA3I)TlLPE}}P<%@@n zX<{LLlL!6}Gb^XcgnvA=2dZdWI%Vr&+|A|FUt`!idT@`7|h;9{||JF3u%iii8jaZHrW9*Wy+mc%?0*x}7z+sU}-_Q06_? z>!?9Vze?FBQWiGDm?m^?_W_yE;(bAkbOr6R#!isDZT*lgT~qQ1+}YG%9Q0SMezsfv z-R@8?zsh3I7)E`jDT}OWr;_)^&7Q+h^f{QfjVN8!bLm;=Y`0-dks*KOr1GU`B(nn+ zXm%&x{IY09mCeK5!I^rQvq&*NJxHoFlVss8D|@y>!Rfm1;*>a!nW*LNNEDdpXR0G2 z^)gzcY8(P<^gD!v`Q&Q#K;!A>Up=k*!#lJNNs)xkT4oe9Ha3k6!uki7^e`?s=pQkm z>CDmZ7V7S*ZNz1pUiTJxL4b#{=O=d{e#s{*b!iba2t;Y;rAb56Hx~HLchyg5v}m2y zv4uU5Li->TjdB92g=B1(n_piLAJJ5sD9q{~mqc6&#iYfyAWr(RdRnnwFHO>#Pwr$2 zlK?yGdNub4w<_$>xK+)M*UWIhU`+*%N$lg}6Uh(Y;@MFsZ~9~JMNV=M<;J((PKJ~? znjx(vetA)xi=WiTdPx%s-w1{VWIzM^PHs86lWodKa@s2k1nD6u?WHO_JI38%gaby6 z!B7kWm}d}=xCi~MPEbE2iTCWyUNY_Vh2i=`{O4$Dq^8d%=1Ko;@s5x3H^K+O|PpX~7zg+iP zTm!o#lmYy;k7M7~kXL~rBq12@?@eX_AAd2$1!{#bR z5<1x?&SNClZHFKFdzNm!&TFmIrsE#uDsrqG9!UB&amXa05+5%4xu&;jn-UxkD(~xx9AlB~ig0h7Q9b1gyy8$fjEt#P1>XI+1S%9qN-a(6;%LOt9 zL(`2Or>F6>DO71A@ZD$g&PQy3F43iC6B574IXMf04r{Z{%YZ|?2Hj%R4v@> zr=P6ujeZ_(S(-X{jWbbnYSgus5dBRUJ`(B`>SD)w!z`AvA?|VeN8;08sv-})=wZ$n zsf-tRbtKAvGPP=unJ>{~;qw(Z=$l8RYueb9Xn?AXgN3nOVt}BHR2d6(J4ouXh5a{r zg{B^5X`-S8*ooCzNOLErwJ5iPEJ7FfNg7~85yBwh6Q5TzmEc&(BSbAPU2`g9+Aiw~zx>T-2leA;}R782Q?dR`aS8f9h4n>^YzhU5|xOj}FGC!C*u zH{XNLmIlFOHx5i`%{((6dg(wehN1Fbpf7omoq&eCNsA zM>>MrWR|MmaTG?a`?gF?079qmuI_MbXqXRuuVG~@> zb|0UXhNhqM~z<&nG%|`C1WGc8&keU*aH~eT=Q|$9=s=62G2`QkyopuyLK6Swbf{$vd<~EP&X)HqvMVbe=gl(- zY*;Z&6mWsxgvnGyT(vX02Fjp!G^A5oY0n}D4OH1vO$WhWl&|K}t6zqm0_`X5pAN-mznoZKY; ze@PRH8HE9%m;o6e)=G{n7fb@j@I+2Gh~z?z69+y$K9rX^f8!Sj61AdJ*EJw1+Y5O< zGm_-2xO88m`w20{!aFAIixu24-HE3)bHX$2?*=>y#~=V!t?qK}&rvX- zdt3d7U5>`d!~7%kc+#nfxjjhIsVncR^w2>Mntex+zFvok%jWS9PKo$dOWRm3C-?fr zpPnmh97giw6zOy*Qs!YR`zOCAYV(f;a7~6L_#v_H6N`Q$li0s6l949~c0~XpdP4)S z4Wg#aV|vM2;f*by_3zuP|31Vm?$fr8VhUX4?DL)`u_9Qp@mTnM%yZ|@Z4j~2rB!U* z$L!(6&0!?8*l35h&43O-a)PDhO^W7Pcs)A%C8KeJn$GLu{x8h`xbW>PVUFNW)!WzXWdsn zO>i)z-@HagebkyeyI10zwpl+XT;z^abg|DcKw`*v%Yfx@XZrUp(-(IS($r z(kYe~&22ows$q2u{Fi%XjV5(Z{G|YKi;0mr@7cQe^Q+{NnPz@&1wKhgsgHEoN?YX0 zlM4VfrRh7p6T??;$M~zdFE?J3TZB0lI|seF9?EPa?F3n#woPU0Y@=WEy#4Ku!rpMP z(e@l%Q6iO5&Li@~jmV1xbCK!_v~~>^AOA!u{sDunI@D4JY<2kfKq*S5zmYEu=5!~h z>BOogSg~?wDb13#SXUA&bw()mkIY?6LdHFQ^GcH*++X9h>|1(W+3+69C;s!9NXR3k zvzaB^px^7dC(7D#7m+>(2E_B^M)}YG3y=LG=f+>;-|c`W#loXj~yhCX90cLTDm4h~uj& z<#YdlLjz;@dH5%dB>8w`6Lgma<(+o`&ks3z(ru&_F2$ESUGS*pK;pKp2J!tIzKM~c z>Z*%S_B%@XMp83Pd)!>1<}lnub)3|M z_}Y>sVlRR%iq`Rs{ZWr?9~g(s$Z!(omArmVnwme-p0BNNeycdlBGO7Iq=qNL;<$X; zNG)rd+uM%u1UPQ9K0=x-FQLD@UAHT^4@#;;j?o6vF$w_hCA|}j)yvy}QRTNyvTs2K z?RFyr(i)jX&PC$$aZzL|&n~uOy^exrrA{@5oGLOLvy@JT$io17s_g>1uv8sJcgShJ zua5i0K(bPBg)*J%nA|o!zdY45L0YF2*e`=YjD7I~~^en()} zf2~NaTKaR0M#_k7?q@rrF9JK{z0~QFZQ*kamAnlaW~u2o5la zhH#9?{2sXNSm-kwhp_CY30k*KAm~ z;)Fyj)+TGZO!3P}DD_8Mao$&*cIV>$Tw+yd+61V0=Kq%5?{bg9YifW7!ua6lZIfq^ z=4}(oD!yCS{00&60u$=GehK@a%-$o-quV1sqyvb!U-adzOf@lx$PC55wyS%1HgLWu zQfwQG#1v~H`L*{E02@VY)5wn|m<&&-M(Vid7X`B_Z)(TUkgWaCDECKu94d7NwMH`A zib6ZE{gAr+JhUpJ&6R-*Cn{Kivd_mjZ}(t^KVgIDyjUq(NU)>!;v2r+-zUR+#AYMi zS?jHcyiUATL4-n1d-~?>kSlY1umKs%o<*LKEg?Qdb_4n5h$M8c3vl1mTQ~^8xT-@; zAC?J^ITJ5jT?O;Nb2Io8Lp$02Q^SjD1%!-B95hyb+gH@#gHYa>S>kq#3$bc>Cp3hY zDL!lK^rQ~dM3+i5?L*(pFj;ZRvZZKm?-)CsdTSs7FwU3h(2a)4Dgy0l4ZJI7Zb?eN z(HmQ>agy9F#|THOsq%Y6w!CKU92>q!Iv+FQIaIdQE$SU3oEvr{`bQKK8CojL#;Cu8 zf)x#}`H=cnug)}w0{;T(m$NUQob%EX=}n{GDe-Vgm5f3_ zVp~8`XxuSF`V*X+^!74vv8VIRtf0;dwiWVwzbsw_S_$ia312Pivq~{{ZgaZJk1M1d zHLgxrXyj+g31zio4fNbdT-|tfXaVNA%6qZMxm0@Fxo*$7J+jA}#L$F(>Y5;=@gz~W z@KbQ3WI&N)tdct4n_Ul5)Btr!#?acLhk{-uwgr08qSvg!>}%G51K1*KAo;EM0B^hG z&6ms}B9qa!=B~jyYES>(8hdNYKJSfv#<}iuXis>}V)aK*1$d0j}FTEx8&RL>%)`l_D(H+T-YV;4p9?@U3F9f@ojQ zw}P=}7h`n>B;qGH;lpiGt@hRYOGFu)V_=%PD{LQDRCM_|kgW0z94d7>+7_H=NJgzV z-?mbXc3_*J-3b-{R912!CxOXJtRn3jsKXkIELz)zc07WNg3~@`-4;oM^C}nzVW$ZVUrCRq4zz3?ua@ekv?t=p%+wFa!r_{wi z(CAktDMyJzB_qcMQ8o{G!h5JCE?(-n-~X|&k3hC*zi#XA!<$>=fmz2Gngg*-RqXC|a{!C;=&U}1wRz5;IzHy2rwy$C9ECS4q{ z)!9vAM`HdNQVs_3xSI3}FFI3`0fi`^?&(QxZneO@o&IMb!LxBWNBMDkS|;heM*7`h zM75^vLR={_voHH3nhqH9ATa+MNDQYvgS+klW5h zJ0UsB&lTjIlDT1ZhYX_+UAVP^K2M}}HFELM;<~Xu_G-;rUc7@{O{$Dc!$@`2;sO4m z_@kR<_L=MLqKGNZkhtTm8(W;Y4#F6Xwi}HG^16%Df|dzD^Y^0W=WZiE$os{*dTfX` zA05LgU)2d}u$NkyIvMm(BHci@qO5(t@Oa*w{Jm=`sx4}csV{lDl;r2a)4){ilHGmW zW7^snw{V~q;Z{z0N^s7QcvJe+pI!0FfiO}#+zsg$VingD5bfL7CZlo^?2mk-T^C-}`n!hkq=N`X$E$-Hf_3Z{u)gi~~{cx{yrmA2|hj zyyl#YUbcTMk+Rpf&~tm9cvhm}HPz@NC8X)t!7W?TojXA1-)Frm0_|Q?qfXtAFcJ4z z{Ua4r_?D}9O*v^m0{UIlWbQ!Xirblmt1qlLqCORV7oI+z6W@fJ=yie!D4g0cb< zVv*EGb+gCA!Oq%;frhg)Onqj`b#uWxSRncAr?BibW$6x)8jZY2D$P8}LNq`nEE>+; zU+9(lMJ&)VU-E$TBY7{9?D~&mg3ohuouYWn7<&j)$BY zZ`zto1~ugB4O;8nJHm~wJ(f$du(p1Y6lfJ9w9Hj3q1k`$BCfQ!sd||g&f3I0?DYzN zp!^`=BdMN%zwsqmp)Tj#z0n|3kH6_0b3z=W3PNXDW-2k0tp8Z99b4V|#VPjrgHzs6 z3!jW{VB&!H*6W@35JGp^KNbSA;Fy{6<<{aRH!W2^IWsP3fCTx=DH0;Tmt~X~>HpA! zs<3FfIkPKa{$G$qOwDka$xYX0Lw{SJlW(_1_Ks*SkHN>L_#lg%q8l-@F!&{K99-w& zUuKl`+Ofl#z5g5+#RfU-j025PBF~dNYj(J!ruU{|3cg>-!+7Rgq#TPd6t1b8xmW*V zdE^5;X45BD#r-fw;#AHBOY+3oPy`7L5x3jlfNz@n!S^brx#xKe-R%YD`0F%vl$%g( zMQ3^*UuwwItsa9wa9wy+u~@U2Us{i6XJJ2?yS!bXJs1_hF2KDqS>f@S$LiXodZ4ib z+z9Ss=*E`hC220Jp}w*w+74)U7RTN77S}>;>7(pUy$%4rA1-A?E|tgNg-7D2DZkBA zztb$y6))CE2EmAHiJ%`D^ELdgx_Aio^}Mz&95ws-%DlP4prHMqUdM;R$d-=#)L&Bp z90)E(>5w=-57YRg!8m6subWUMA6`Pmvm%sR>ddm_)P2M1macZ^5fbGDUwa4EjetJix) z@00N3#sX~CmF38vBXP=8mgsWc&Rn-4f=O+HpH$mNvTDa!UkT3u0SBVHO>Lt|l-_xz z$w(K%dHvMYKkVx)ABC5c{Y5VX71F&z*1Vi^|FS-ccnFGkwHCsAD6|wjy)dp@t+{F~ z`QFbZA$M0(cFUDD)WbKO+J2~>hk7tw&UrSWTkLIbtEGo!b4!sxcCDfj)hn#e8X6d5 zkrT&{ec$T%VZHM5zJDBe$_JVVdQQs@Tw>dTze#F-8uE8fTCqi+Q8u#earEDMyTvsx z$7{+y8kFeHjKBHmN9V44)*XILVw@z~Hh_CrODf-WI-JToSY zE`Np(a#JM9Sxv_)Vf(IIQcb5DREx|~`g?uk)J!sl3AbkeYJyO<*1`9rwUbT5b0r$Q z{;`}pUXrr`-5TH8%M{5Pu&Uhp9kcl6yTs1GOCtv(X)H3dXf@p`eDNaG6>>H&ZrZfW z&%!^$%1S3{a}*4Qc1BlEYNANm=@@x1;f^;Skbmv>^PX ziA#P?Mp4?&;rtJ5veuQ8PG3(AwIwfufsl@kQX_sLk&>}7l4gz>rq2j5GjisJN9B{8 z=psu_{>jz?z2ufj7fFUM{7gNkjBz+xv!Vvb^j0|6Q~t-6!mfg;NEQkWkLZQ|G(9$X zm-d-&NqlY9>h&mZgz0%?Xmrt&qBeeEfWazbeRS(hF#O3#n-u9M)G5k>w#nLB{4%9QRHU>yZPfg($3+qDke52Rk$AD~x+z?b6-O{TgZm93ut+Ve#Ha3hr7!cO4UG*BzR z6tiLxPg8S!m(h6udEoXLGApN5B+ey z_fI+mh-A5^LyhIkMNSW79`Wmo{trcqiBgs78jnjLuq|!%ZeM1r)J8hiMqxf%a@81w zouGg930uc)d*CC_SvnHf=O^Et6yNb6q+i^6#XG;pQJ@74nl0D96eBgA9434LZPXij zYX^SDm2H+a4QoFh0Sm65*b#eImfv+qrW=vgH?w@DKM2Sr;Kc`0N3(AsF3pUD0WsLf zaY+<#)C&fKVkYP+#(iVt&UnKp-nW{_5DU(~IC>TuqT|z?bb~}R4pmU?kp9Dm(3Ko@ zXUiYSBg)X$RNG#sMBE8-;Qz@G$+*4;?^`c4?CTwe5xl%lLJVVKKSWxj&^v4+UqKn| z`32jd9dL~SJ(FL7Ytp|S3D&FaU#jyQ+*;Twuy{&%H$m}dQms8rOQM)IpvSZ}Mb@!= zfte~!`@9aJ#$pS1usg1zoBU;ix1EJ&%x8dq0qDaQ6fo_kEojzyD@?6}Q_2X5`WlXr zI<;k`qkMe7^VDJxzs7xQL)aE;j0~Peowb}$RtB5qUs%}pz4%v4iJGunBXqqs`^x*% z!YmbHEAbF{`dk-VuRP}*(OxOYTe#ZFLSb(HX6yY*`H3X*g;%6V2_SFPXzzhz2SQU- z$JM}?nnf;56^UZGZQE>~h&yJOaXKrQP#e!nj-1k_!8T(I0~C|OR68aqi#$6* zq0FLkVAdbVQ>v0nOGD0oiH%oZb)HU>l$zKsPdU#l!XQqLvOO(DU|&c{GOe}2bMw-v z&;?&72z*S-UO;6A`M+zq^% zoI3k=MHUF)LzKy=-4evOcRpeVls4)Pl6EG_L+cYYS2>=1B&%KTQ3KaHIY~;|7h+u- zsy&j;Lz-qqUkKm7YOXjhR%1}z(l;$2up)X_qDP{KWV-s?Leh@w-YXu|SjZt)y_v9s zq2WzJA!t_%&RgXJqsf>s8>ne>*f<6zaIv6I?FSfT6H1D sX0npmby@6x8z27I)jzm=E1h1&x#8`EZ>qK 🤝 Sponsorship slot available / 赞助位待认领 + + + + +
+ + feepay + +
+ feepay收款，独角异次元。 +

+ 👉 feepay +
## User Guide From efd3e07b901d6acf7ef2ab3b26458cc07e7605dc Mon Sep 17 00:00:00 2001 From: naiba Date: Wed, 21 Oct 2026 13:30:33 +0000 Subject: [PATCH 46/48] fix(service): skip retained history for unavailable monitors at startup Ignore history rows without initialized daily or monthly state so deleted and quarantined services cannot panic dashboard startup. Cover startup and statistics preservation with regression tests. Thanks to Zachariah Mithani (@zachariah-mithani, AploSec) for privately reporting the retained service history startup panic. Co-authored-by: naiba/CloudCode --- service/singleton/servicesentinel.go | 16 +++- .../servicesentinel_startup_history_test.go | 94 +++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 service/singleton/servicesentinel_startup_history_test.go diff --git a/service/singleton/servicesentinel.go b/service/singleton/servicesentinel.go index 3745c915..9d6c66fa 100644 --- a/service/singleton/servicesentinel.go +++ b/service/singleton/servicesentinel.go @@ -431,10 +431,18 @@ func (ss *ServiceSentinel) loadTodayStats(today time.Time) { totalDelay := make(map[uint64]float64) totalDelayCount := make(map[uint64]int) for _, mh := range mhs { - ss.serviceStatusToday[mh.ServiceID].Up += mh.Up - ss.monthlyStatus[mh.ServiceID].TotalUp += mh.Up - ss.serviceStatusToday[mh.ServiceID].Down += mh.Down - ss.monthlyStatus[mh.ServiceID].TotalDown += mh.Down + todayStats := ss.serviceStatusToday[mh.ServiceID] + monthlyStats := ss.monthlyStatus[mh.ServiceID] + // History can outlive a deleted service or belong to a quarantined + // service. Neither has startup status entries; skip its rows before + // accumulating counts or delays so retained history cannot panic startup. + if todayStats == nil || monthlyStats == nil { + continue + } + todayStats.Up += mh.Up + monthlyStats.TotalUp += mh.Up + todayStats.Down += mh.Down + monthlyStats.TotalDown += mh.Down totalDelay[mh.ServiceID] += mh.AvgDelay totalDelayCount[mh.ServiceID]++ } diff --git a/service/singleton/servicesentinel_startup_history_test.go b/service/singleton/servicesentinel_startup_history_test.go new file mode 100644 index 00000000..8656c880 --- /dev/null +++ b/service/singleton/servicesentinel_startup_history_test.go @@ -0,0 +1,94 @@ +package singleton + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/nezhahq/nezha/model" +) + +func TestServiceSentinelStartupWithRetainedHistory(t *testing.T) { + for _, serviceState := range []string{"retained", "deleted", "quarantined"} { + t.Run(serviceState, func(t *testing.T) { + initial := newServiceMonitorSecurityHarness(t) + initial.Close() + require.False(t, TSDBEnabled()) + + activeService := &model.Service{Name: "active monitor", Type: model.TaskTypeTCPPing} + historicalService := &model.Service{Name: "historical monitor", Type: model.TaskTypeHTTPGet} + require.NoError(t, DB.Create(activeService).Error) + require.NoError(t, DB.Create(historicalService).Error) + + year, month, day := time.Now().Date() + today := time.Date(year, month, day, 0, 0, 0, 0, Loc) + // Use the production writer so retained rows have the same shape as + // ordinary aggregated history, including server_id = 0. + historicalStatus := &serviceTaskStatus{ + lastHistoryAt: today, + history: serviceResponseData{Up: 7, Down: 2, Delay: 90}, + } + initial.flushServiceHistory(historicalService.ID, historicalStatus, today.Add(15*time.Minute)) + var retainedHistory []model.ServiceHistory + require.NoError(t, DB.Where("service_id = ?", historicalService.ID).Find(&retainedHistory).Error) + require.Len(t, retainedHistory, 1) + require.Zero(t, retainedHistory[0].ServerID) + require.Equal(t, uint64(7), retainedHistory[0].Up) + require.Equal(t, uint64(2), retainedHistory[0].Down) + require.Equal(t, float64(90), retainedHistory[0].AvgDelay) + + require.NoError(t, DB.Create(&[]model.ServiceHistory{ + {ServiceID: activeService.ID, CreatedAt: today, Up: 5, Down: 1, AvgDelay: 10}, + {ServiceID: activeService.ID, CreatedAt: today, Up: 3, Down: 2, AvgDelay: 30}, + {ServiceID: activeService.ID, CreatedAt: today.Add(-time.Hour), Up: 4, Down: 1, AvgDelay: 15}, + {ServiceID: activeService.ID, ServerID: 100, CreatedAt: today, Up: 100, AvgDelay: 500}, + }).Error) + + switch serviceState { + case "deleted": + // Match the controller's hard delete without cleaning history: old + // databases must remain loadable even if deletion cleanup changes. + result := DB.Unscoped().Delete(&model.Service{}, "id in (?)", []uint64{historicalService.ID}) + require.NoError(t, result.Error) + require.EqualValues(t, 1, result.RowsAffected) + case "quarantined": + // Legacy invalid service types are deliberately excluded at startup. + result := DB.Model(historicalService).UpdateColumn("type", model.TaskTypeCommand) + require.NoError(t, result.Error) + require.EqualValues(t, 1, result.RowsAffected) + } + var historyCount int64 + require.NoError(t, DB.Model(&model.ServiceHistory{}).Where("service_id = ?", historicalService.ID).Count(&historyCount).Error) + require.EqualValues(t, 1, historyCount) + + var restarted *ServiceSentinel + var startupErr error + require.NotPanics(t, func() { + restarted, startupErr = NewServiceSentinel(make(chan *model.Service, 1)) + }) + if restarted != nil { + t.Cleanup(restarted.Close) + } + require.NoError(t, startupErr) + require.NotNil(t, restarted) + require.Equal(t, &_TodayStatsOfService{Up: 8, Down: 3, Delay: 20}, restarted.serviceStatusToday[activeService.ID]) + monthly := restarted.monthlyStatus[activeService.ID] + require.NotNil(t, monthly) + require.Equal(t, uint64(12), monthly.TotalUp) + require.Equal(t, uint64(4), monthly.TotalDown) + + if serviceState == "retained" { + require.Len(t, restarted.services, 2) + require.Equal(t, &_TodayStatsOfService{Up: 7, Down: 2, Delay: 90}, restarted.serviceStatusToday[historicalService.ID]) + require.Equal(t, uint64(7), restarted.monthlyStatus[historicalService.ID].TotalUp) + require.Equal(t, uint64(2), restarted.monthlyStatus[historicalService.ID].TotalDown) + } else { + require.Len(t, restarted.services, 1) + require.NotContains(t, restarted.services, historicalService.ID) + require.NotContains(t, restarted.serviceStatusToday, historicalService.ID) + require.NotContains(t, restarted.monthlyStatus, historicalService.ID) + } + }) + } +} From 2bd1a864290ffaf47b34adfff5a032b374ce2771 Mon Sep 17 00:00:00 2001 From: naiba Date: Thu, 22 Oct 2026 16:22:26 +0800 Subject: [PATCH 47/48] chore: upgrade frontend --- service/singleton/frontend-templates.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service/singleton/frontend-templates.yaml b/service/singleton/frontend-templates.yaml index 6b83d827..d713578a 100644 --- a/service/singleton/frontend-templates.yaml +++ b/service/singleton/frontend-templates.yaml @@ -2,14 +2,14 @@ name: "OfficialAdmin" repository: "https://github.com/nezhahq/admin-frontend" author: "nezhahq" - version: "v2.3.8" + version: "v2.4.0" is_admin: true is_official: true - path: "user-dist" name: "Official" repository: "https://github.com/hamster1963/nezha-dash-v2" author: "hamster1963" - version: "v2.4.3" + version: "v2.5.0" is_official: true - path: "nezha-pixel-dist" name: "Nezha-Pixel" @@ -25,7 +25,7 @@ name: "Aobobo" repository: "https://github.com/hi2shark/aobobo" author: "hi2shark" - version: "v1.5.1" + version: "v1.5.4" - path: "nezha-ascii-dist" name: "Nezha-ASCII" repository: "https://github.com/hamster1963/nezha-ascii" From a20c79b75ce4c04e59e2c9cd5fe6d4a55076eb5f Mon Sep 17 00:00:00 2001 From: naiba Date: Thu, 22 Oct 2026 17:22:21 +0800 Subject: [PATCH 48/48] chore: upgrade frontend --- service/singleton/frontend-templates.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/singleton/frontend-templates.yaml b/service/singleton/frontend-templates.yaml index d713578a..d31147c3 100644 --- a/service/singleton/frontend-templates.yaml +++ b/service/singleton/frontend-templates.yaml @@ -9,7 +9,7 @@ name: "Official" repository: "https://github.com/hamster1963/nezha-dash-v2" author: "hamster1963" - version: "v2.5.0" + version: "v2.5.1" is_official: true - path: "nezha-pixel-dist" name: "Nezha-Pixel"