From 7603f8561ea13c0876bfd117ce4291366c2a6370 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 29 Jul 2023 11:41:34 +0530 Subject: [PATCH 01/32] Entropy Methods --- all_rules_entropy.pickle | Bin 0 -> 19375 bytes capa/render/default.py | 17 +++++++++++- entropy.xlsx | Bin 0 -> 23367 bytes rules_entropy.txt | 21 +++++++++++++++ try.py | 54 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 all_rules_entropy.pickle create mode 100644 entropy.xlsx create mode 100644 rules_entropy.txt create mode 100644 try.py diff --git a/all_rules_entropy.pickle b/all_rules_entropy.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a29e925d6a795bf802fc4ed5091237d601a186e GIT binary patch literal 19375 zcmb7MNpBoUc6JpPsi}pMxJe=`_!s!(t6{+3dl8XaB{kzl1I^A@GBV;V-}~N+ z`pfaZ|LvPUkbnOCHxKA~;B=h88MGYB9z^cShg~a-0(a02r+)KhfvlE4h-`OY`OQuo zw(Q7hqup*DB%O9wCh1Psj(SdDbzIEmA6vt~@3@|0xr2@$^zF#?2bgV)raixF9XKZ# z>>oJ%hFMz7KeG;9+Zy_whb5rBMl&sY7{!5Og{{DG24T;SWMxyKGiY0FJF@Y=|4J)? z({kKH{))DB6u6NiZB{4n`&Q@#hge?ZTP@FZ20XzMoe!MQ_b_7PZ37SfhHj->@p^y1 zZoS8_m9;MV>n%aApV1w8uXG5UU&l@uSx0WvvtIl7tLNBlCqTbTbRNsac3DGr=-|P1 zTGS7sBOid7TI<&_&btNyagKRQU(+?r!%b@I)kGWhwfZ(%muRV^ zb+7i!s_(s8Lx&bE^z5*w#ac9J+x&;OEw%9k_PW;VMn`U$R__AZsZ)^_>+;h6XoqN!{f!*i6 zKhkY}62ihb{p{3M_QT%%Db7Xq`(49=VF;Cd`eS;bPYV_(@bM*~QtiIqb~`RC-}cw< ztR95Kb3$cj!dBdy!tU?zdHCVr8p1t{smZ5coQg2YN;k4y_u@gZ$1mxMc8H->-&og*{^kXJo%)V~8?_=B6DQ(@ zeQ<*)w!L-#7{_ywE*O`ko!H;f6&U&;vc$P4pM{Q9nqw0MVSM~{b3aitw2aYpY?C{T zJ(dqF4F1fqnsEmn9nIg+jOW9p$S^w$oqkhH#yCyK0}Qmg5NRlBOH5@!(xq+k_G- zbFVzb8G_~}PL8cOcPq?w4xqysD7zRyCE?|v2u+-{>8U1Lpm~;Vm`s2iHY}+feo9j< ze+VfPEBvNIb3*_h^l_ST*Xm=(u|wUD+&gSH*8-!dRz zi7v4!IIIW>Ao(x46~gAVV@OD3!|w8aZtvr-hJMTuu8$=NKLf0{UUS(F$C++HdVc8)%OIgs^<8p8{m|oxXCBRl2H?%I0y) z@vKifJC@yUuegjOtF)jSKl=FakJ&b8dx?l)Uf{r{0)t)bR14Fft8xO6Z;LGp4&JjB zzsq!e5V8_nK7$|@17&@v*ET=xqw6HyNVSC3=)>+u8|ZcW9Z+q+ zlYK!~d6PuR*_g0cdd{HBkO2oUrOxjC-VXkEmKM|+l}Qj72t{uzIa$E)0bm7dY~E&E zrCBXk@>~N|dHDjrT;s~T?_JJxa)+CV5N5N_8eWY7d8+>N zG_AdB`s^lMRGuMyw4xwXgg8p{hAm&u9m*?j@KLeQg$xzl=)`i{kTS+F0e|CJn$@eR zZ?0QjJNQ9+pkHw$VBiEcALGUSV$9#sU8Bcg&+$Bo7x-`ifyAa7PqIq4^xIAh4)g3m zHx|>#4p^dr1-jew{R1oTW4K1ZDW0nDwgNx&J8Xg@U`bx<7(F@7&MPgntzCST14Xd* zkKpCS_1&OZ!L$hXhE1Q%qyEBqe7eU62z#{m@l$GX?$gus(-REFurYv9LhrT{9z^~S zd!n!nqfV@EoUYj_5Ym1lcfhO4W+y-bgN+9caz=NNOq%2*HY^uA^{ZB z@jVrEj%(EI_$ly~Z&mpA59y||Eiesj7iz-$k~aZ&m0yE5<(MDV4PGXIRTL?Jc=3sh z7h_N{F{%5RrpvPT@;LE7U$A#|L@Em==_1V=Af#KuA)pMKxS4E7n1pw$&tIVJ4$XQD z>hxM!Gn8~=kJ#^?60JR=`5xqRBs9T{KSwNRWTVV$qs#LwNdN@pod8(E{Q4eLIH66J z=}UuvN^s8c%NM|>F#8OVc|&LD%MzbuV9`kTwbDSk>G?+;H)J1jrnEy15fWGxW=qb| zLdgp^HXCTWNjFQ{YB6?q_4)HBzd)Dsbg!gK!}daZ^&9r+wVsWCWhILxJwbrzm5$TZ zqCe=z0seg|E6MNy$>BeJ`3!C2rGoMN2|KG9oN&S;sBw|o6|+zU0TRP0!xoAKc)hz; z!-%ix;b}I8&C*`vy>bS70y5G2EQjxm5Sy{v1u$EI z!)gInz(yOhjL|X{i)WFwT{r$qvt%N`=rfKeM~!~J0fZ?W6dr#`I8Eanc;#Wvh9L@C zM8KDX&Kv%4=y=_K|Lvfwz6_u)+YW2s*)50Z6ig18WZ%k|3Q7py4;u_C)@KDdLl3cL zgLFRbZc9uVVrPK1aX=rNPJf4Oy3ZK)A}@1HD9kli?D|LGP_dRO-Oa$iG_KY9124wmuk3&bSjzZ7*3vc!tC!pa9ejEI~#0@IFRpnHpAg}^eA2B=?AHz z67-*@tC?VX8$fqG0Q=_zY)Z#D$kw4s+izi+OnpM*3jGj~m8cv%80NT>r487rVRN8Q zN|zRCiC<7xx^8WWQAv~vqb~p>`NnC6*FIuBCh4BmUS`>Kq{W*S z*hkjd)`q2`56n=ZZ^|*XiMyGcX(HK@NF@fHPmD9qml)>>*!4ug8ABg&0MNwd9SnU$ zKm0A1fktx)7&%LL9>}mjJW#3r0Xj~>edw&qk<)D1pi=w5?`S~?#K8rJ^EPPE;OVNe zXCU!poV2D~p694{of`%4u*w!wH1-8;%ER+uEtU!iYtB07NJ8rBQX+ z+5;yTI3D)MSQS2nZKYJ=`ZZDQH-F?XK99xE(q$b8W){c(o0)x^=YMWg|64^BqKPWiWV&5ukDNZHHN@km*%MOkG9KTvX25 zcp-p%L?N(0{CP`sgViG6=EL|UJuF(x_7Kq$-PTO;1)5VphfOMbP2KKHn?#TB# zs%?y}xP7L2tf6uaid!B>Xx$$o6lYEd@yBJAidVmT*ld=&;)Y*VuO@oMy>6N2}mY zT|ijQpq+@bkr-85dZm0`+yi6cK@yR5Wur3*W+IhW6}f5~rODu(jkP*Djwz|t0b^d- zs&pUJ0a)BOj#Fp$hbbWV?slljK0M#Dl?zAK!X1dhCgL_j)65{^82iu!#V22;`^J71 zqQG7ExW-N&c@4ajIUD3)B$x#Z1e$NPn0MqYsnSj5V-&Tlw2^fSI~TbE3(nFFt@SKf z=5&DinO5`}5?r8#EUCM_-2mWZ+$b;35x(0EnCdh-l@6D2>To8^$Emamo{o>GSKasQ z02=T$b9dJMPD92bM9(qSpS~B0YLc!?5U!(a$oKOkw&)@bfUn#6V> zZZiUwk3r7Tg5>7`e(RRg>iN($7!#pt&d^JsyRSy(bbHeBH`vbk>pQ+ z#Q9KQCnTjHX9g?0;|^j@?0|nr<&kAU!lblyMgWZ%@XV+SG@FS^G7a-dY7q$>q*Hyk z^-MF;u%LJjpjjo2=jeJoIK3HD^M_;*kRSLE&oIM106RVZwQ0Ur;kp3`QD#Lu%r>FS@E~eq(FIrW4J{R-*tOws8}P~E14CKpuqV%}bi43eGvg*dswmJ3-9WU69ISVu=o`sCW+RfZEW;;qjJSsty#+cu zVfX-pOV0Pxr~OT47^*a_N&nQP&m+rj!;~qslZKuZdaR45T0N910TaM^wgKEF2xAM5 zzIk6Yl_i0h4aw0HzoJ{GTDkPKXX&Qkzl>*6;3(}fNtY#k{AqV5BZy;kIkj#Q$tXXC ze4XT|c|9^|IuU4TM#UUmb772@wSIWml6;udsQ`+KMO2Eiwa@-~9I3;^v8MvZ%KJ9@ zj!SYHqWUZ!X_eU*bkk&tbecy@K3ldf=R(x^Xr3vJ#U>f_M~VMTAh)8%-n?&Y?)`w$ zqP@+HA2!zZ`IM&>&&;Vfl(T%#1mhf z262Uyqa2g9TI&DB?LV)*jp< zmmBO#DL{a=3KWw*$oENwk|HNy{%kJaLAqRNHs~UxXxJbi8_EYFOa_>WwuGH51PtMg z8TdNi8Tk}c3##z^)G^ttUTAk5v3zWI3@ZrtB;Utju)Lnd4vU|pC4M*5Rfi8ap^0VOF%ptS ztX>*><3u>=3jV-U2aVDB(D6DzdMtL>M+mhEsU<7a^xLeuTu9|Pb{d8xK$p$X6q_tj ztt@7oplcd!rudjkAeuHu!V`2gLm<{RR`$Txise9Km1c(i5tb=gUDWrviesLFH9vxS zE8-R_ zF&U6MdzaBlfcG_);46xJtJU8ZeAZ-=kkBzqVh|_jMr!3z>V_<}$2+8qxB7rt@m1Od zFc+bagk&OBU@4x;#a?8&)(N^}Xe^YQ=;C29SIiC~oZ%ozc4#4oS(Ot~z8pbgiKqWu z+g^>+lhrI+8$uA8sEfvbkgL(r7k}DWl-HVEg84io6j2nw)eTmR0>OYi3v0w=Vz^NQ z9YiRpkx$uf^pM#B+-2gfO4lX)Vdhc1Fx%#jTXmgv=Q>lac+!X;30|9X6c|{cyQ<0$ z?=!9XAc|Am0(w#jbcuY-vUAyVsA&fq4p)!)Ob-P(jXf#Gjtg+xB4%+FR)teJCf zubAvY*+;n`eix%~XXkr#8KcEf>K0E}_j&UX$CMnc#&8hJ{9D)k0Cfy)P5favL|UZo zZEeq5+uzO5N)xnNGBIbM#67o=RY$oRWKg~tA-m#7tF)+%tRWv56R|EBBfLENJ*tE& zVU8;FWk#EuKqgGp0=POLaRq%rMKnnuyU6v<#hS8j8}Hs`J_j&fgS%`&*#e@#+_AKd zf7rsAOU-VDo|&qM)M11QOX;^;wbYYn1cC)k&_iQDCFBC+P-WS?^+uWD4Z%R8_gPvl z*=WtvSsRk>tA4f*JQU`E+N3tfK<^$BdW!CZG#7S>=|00hSBmiKt;@6&XZmvcv+E9C zeaup5=rl&(7_cgVjvM`evj)w_;TP99f%rtc?=ZqmQl-U;R*sO&z(T3+I6cx-RT98z z_$|)WfYyv=Uf0pc)AmXy8fFz#zUi87w!N~}CEbcB@&HEbcw~6e!LxsI%4RS(Nt!Xx zK?KX;cS_d5w3xba((=I5xvjFP7EKp0CZp|8Hq#CbDQmxB+D}kh)s6ux@?luI2o)@p ztP^E%dXglbSn-Fr^^ue!3II+>a>wA_jH#I00abc;H(%yElzeT(-RiHbx^W0 zh_va|`P|2IQ1%m;60Y;gMp-BnHREvtz0Pq_RK}17(3gUsJKd5cAU|w+AUy1iuA-9Z z!0dwaw6F-&Oqm&?;fl5tFqiqILJ>2)u)`h%qSy&mBGc*6KM`bGoGAIEXb1*ylzl&U9@$f*J z>D>U<9*(xBrw^=9F0N*)UqvpmeB4m25 zWC{dR`a3yedR*G$e$oR0lGKf^ASYV((YQWf`bNN2rL!6P7WpW1IaOrSnPC1)KR#8S z*kcqoqJK#%z!G+?Qyh+YXu-#;Jj4?bDPsU zsMLiGMGq+`6O@%L64RI zB~<+75hy?}&nu{1lqz34jEN?|P$~8(i!@{wzth11a&l8&LrU|k75W-9bpAHY?-~PW zq-dOEJ_DFP^P(|&%!m5_8G?|VhvchOC+UHT%eJ|0T=*T8R7LSNa<~!_xdTq9V^1dO zE5mP^q$tOz>j6XWJ@_rpTDRNGl;}z#IN!1Xx)qV5#1$K^(?n&j_0~OZ`$4A=V{qQ5 zfZVwr!hiHg1PNFA)OFZGv8}wC&j+iKCQUM_>jrWa4DzH0C3U!!q#zA-!CvUe6Y>SCEa@VBfh-*80a=B1maZe5<30~A~&mB|Qj=N&| zavEPNQ5wQGX0y5VB=Ky@&MKMmD(s9@Jp=lyD_IWkcmN-JyCWZ<@y6rqWUg#W+37MA|vgkm1{#Y_qV^EeTe)5W|{F zJU0_t0gGsgQ1jW#XJ-5J;hq*>WCi5@wv?de2ZCl5<1E+$c3Bq^WIO3hQ&I|;1+W#G z5~`4g%ZE#K;11_-nN3y~X-(2-nY|Sr%J)7tHv!o83xlrYBil!P3@p5?j%|L+lm&o0 z&h|uux(X$eBzG(nnOax_ed+aEtvL2C(58300);16;wv&P{Xgd}Q9$wuSm$*L7 z%T$Vi>-Vmbwkei2EdE#}OYWsP2_yc7&=TrjN`z$j4}V8bE_lH0M3>X#75eJ)y^$M@ z*`G*uO9njp`~Y#NDBaM+Ceo{)?{B<>%$@nj^co604c8xm`czjSq-SPUvkmMp`K?_1 zp1F-YMz>!)TR||a)2_^`^Nyij3e023VTqQ#qJ5H!S6ua5rIoC-VE>U3#5Au_eug6l zS&(yf7{Rb5XMTs_Ds)Wdy3V(a>P%=idLRwCo&$DAUw^K`=LK9iUs^}PHQXknBHM7q z7#7X-OJPR^_to+*cLGnp9%Xf>*mhgx{*W)@8hHHF1Q*xHOUf?uy{@gmuFFP#OscLb zbX%NJ3i)a~TMbU1ia`ZIu8|uYqtxvq5i2Q?NZ;ZI;w?c);ek8kWSuNVN>$DEpCq4% zyL-5bFCCt?ADMg7E>zEubC4Rfk$Nt|L{TFc2zCmWtAM18digu zk=Pa-msFBr9u(^oj{L+9aE()5VeS#6m9X-d>w5Us8!mThgw4Am=YhCebMlXJoN4=1Y?+^oz8Qt~fFkVWR3%2H#epi6}W- zH16IS0QexebI4RY)`)NeM%h>_Ua(|7oOC-I&Q+3cg+_&u>WaW0$QMk_2#~5`pSENa>OO^gzf) z9UidBbz#cNWa{lo^6|IX2_SZa!0?X_i8OJ|fIC1E!Q2Z4LV~oRrQ%vyiRJ!KrZ7(>Ih8X-Gs58f;e3!W$jy$7 zY4}`hIEdLR&W@mslLT!@eu(jfN=2#{?tK4FEhZCFh(2eVahbu07p$fv%?`sA^T?p` zd*u$oVAPJwccSHG`VxIz{ep^^|GK)2Uo|)TRVmRMDcF||h}XAsls9tiy2Q-9fd!lI ruCT9AiyG%#B_hE=3`Izia#gxGv?0>DN4K)L?iGIGr3A}W;>Z67V2c~S literal 0 HcmV?d00001 diff --git a/capa/render/default.py b/capa/render/default.py index 79567e4b2..795677d27 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -85,6 +85,12 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): subrule_matches = find_subrule_matches(doc) rows = [] + + entropy_dict: dict = {} + import pickle + with open("./all_rules_entropy.pickle", 'rb') as pickle_file: + entropy_dict = pickle.load(pickle_file) + for rule in rutils.capability_rules(doc): if rule.meta.name in subrule_matches: # rules that are also matched by other rules should not get rendered by default. @@ -97,7 +103,16 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): capability = rutils.bold(rule.meta.name) else: capability = f"{rutils.bold(rule.meta.name)} ({count} matches)" - rows.append((capability, rule.meta.namespace)) + + # Get the entropy value or use zero as the default value if not available + # entropy = float(getattr(rule.meta, 'entropy', 0.0)) + entropy = entropy_dict.get(rule.meta.name+"\n", 0) + + rows.append((capability, rule.meta.namespace, entropy)) + + # Sort the rows based on entropy values in descending order (highest to lowest) + rows.sort(key=lambda x: x[2]) + rows = [(capability, namespace) for capability, namespace, _ in rows] if rows: ostream.write( diff --git a/entropy.xlsx b/entropy.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..b598d7c4b2528c003232622062c84880b2df2c5f GIT binary patch literal 23367 zcmeFX^LJ%I*EJe-Y}=Z7Qyx( zES$+)t|4+9tyl(}Z~MR!*yASE3d|RoW7oN)qaZ(=beVHTF5h}nI?!n)%$rEn8J9QT zjV49av=Atd)%tbCpx!6{?m zVZ!Op`FRH>zuCB58H=rUDcJ8^S)Jd6@);6nRze&mqvnn^EZIGJ@{n^0MCq9W zPQYPCE(OwkfKK)G1rDP4|8ch_HD=5s9jmeDvXy&vt~yelF{!yF`Y>(`zLCbprj3DnQBtPy@>^MbWF&5T z(mRR1JXJ_CLAqFOZaa5A10Navc{08r0o} z>A&diX76ZiY;SMxZBf7P<#u=F7XX4yyY5k+%|@O+zNcSYTH_BuQJw~BC}L1R-&3w^y1{N{9A!cw z53K3%L&U;TlCLZ-YXGqD*^XrQU!_Pp+?BV}_6n0s@MAI}kY|q+@aD?6#}`J+CWYOo zIZb~v&Ch|KqW!3Aft}mj46rCkOZ<95v<2XK!5(^xO`DUF->~gNg-MhmYW2o4ekU8b zyUAqVGShIiDS^ZHEyTsB2;i-g>XXX>kSlA)^ z7DkSyCMwR3mUiY&|NOL34-kVVnc!_;_?4emiUY4*Z!&rfZ-UT1Y8UR>B9_R|w8SJ) zmqtFWS5;9iYw*s0NA>p8Q^AC?Qc5XMN*NCOj~v}qItrD7?&*Ud(UO+4c?A!{ZEg;q z@9Re5$i6hokGq2(KLqp*vfiJ`D>AuqhqdA}f!!Ieqk>!SPrh9>XOtp<&d-z43d=EH zK2LQjx`vpuQgKVi_grzM>OIIWEr#4hQ`6nwUEfQz%%}Ceo;HxCmmD$#!xIW7firc! zzKv5UWtM(fP3FBFOrG(0-?jf{wPd|})w(-;VEt_QxbW`CNvEa7oK;&l-1azP`epZ3 zHqOX4F_3685U zw-sutyiR(V0`1RR4>h*2kfzb2Sw1`CgRNRQ<2l}(MZWghxgn`N3nh#*Da6Kz3{si; zh5a>}ZAx^UcKq(#3aXvjiH@@~hkF$8+$=_xf)1WBJCln!QPW@JH}047W*A*`*QFSn zA>~N~M)M9=1>YIuH6stZdlWEV_S!A1Eo-}2ygEq084);2rR-j6Debf`xGZxxEVGJz zGma@IS4ESG8)$0?lj*8E{BM-8PU@?7CBCwDi^6w#+6klj8Iz&q+_(X_NHnkSMg>{* zyi@DzcNF8+_^Pp{P8si&!Cu%x+i6f27eitzO9E)Ti965Ee9BX%Kki1iP1$lf`PCl( zM4TlPWE+1ArM8B#qyMpI^-SBEuXlI9adJnNF-8J0#jk)G82uySA!8194&As?yV)8V z_TnK_vy3p``)aOYWR13Mh1!^L#jgf((H(!b`K*(4Lo;fAA$xZtd`aTP>YfzQPMR=cs-(G3l|!rpyLBJj+#M)9ihUDtL(>(@Tn zIikO{`^D@r9;%cxIM0v#R{dE1tTYAGeDei;0R60Z)H^8^O-EXX!;Flhfv=$whiNN; zI-_W1i8403_^!S8xJPSQSC+UOP?wY%a@sp~N=+9=X01tzjcTTtCd1c`#`$|Jlx>!F z4-uu+b*;K!tk)?r)ew{92x^bT5{AxJEwtfU}tuNEVZ7Cup!~E9OlR*Yoi^p*YROg=LttU_&>dbbE^v z_cAs9@X)Rb)l|`bPfu0vog|b6(cn6*(BV0MHnWdVvJh^b2jpXZffcn57vD^+CbB;0}ack&~duz>XwTH(i-Ddw{bBfHqRT0LauhS(3% zl5-p{){7g3a3^)V2GKkS#MPZ|haf8w59siNh?&Bw^-=8hk0LJsIMBj(jZhx^3es`> z7XK3LCdGXcO}}bBPxp3A2siTQ9L0n-&pBRL5Nhz#oz@B|_x^ehiP*(RB5rMIbAz9H z@lIu{;;)waI@6r?8N?=QA?Uhp7{Z*y`T2>FyMS8Ge9llH=F|OPrN;D^k2}#kSLGho z)@&A0mGQfiy9RVV%*xcpa=RVf`(2mg^Tv|<+1A(d?e$P?jy1xcjrdIGatQesK0#SLxYl0Y@5W}%Yv+d(d=yws<_ zc_Sp`*>F7zC&e$r7ZzOu_#y7!xFD!#b%W~lLdn~{T(}eieI56hi7-tq>kqy@c@0jH z{%nEHh=fR~4&759P47|hLw-P=f>>Z{+Ey>pul)Xjy`dXDFiso)Cj~S;{ej}6MQI4xK2B~uUfP{=X_faqklkQO;-y< zZrFv0|0L@}x-`<>wOahKwYNjWpZO)6ReS63G}1SvbB>NA$Lf)t%&%nP(kmUOPZIqk zE~nf1<$TL0@hbth$5rFNR*AkhCkCLELK*cKa#}K4s2iD@*9N)|ZasR!La4CX>FI>J z;HqH~CORw7T2n{VrSI=F&v9jfK-0_$Zp2yyo~i^ANxzgZvZ3ztv8Xj1>G;E=k0-Wa=crid&;Dnx`-ZBT1S-Qxaia9%<* zhjQ$+5c0W67?pqjngw{ml>!Vp=)H0O2*)^B-xt)FGPe%SiU6?{A2KB~rLd5dz2g&d z)lIGKQ?Vc~V*0J{^T3H&31aI6)aEp8cdPTX|@(J^FV?3t{b(&~Fi{)p8_%Uj2HS8g6Mds+u0u-N_Bu@iT0N(4_W zqVt#YKUI-r8QKQ@@3HCXIaiMGDR{HZpSU`PrNUpkGbYdyv5ZjmO>ZwVVSneXJ!Y9& zO2*_vI(FN9FM>;t(fYRry|Eeuk9(2KN3&I!Rh4XpN>`_T>BLm6!yM)Pblv3-Lhw`R z-TS(J0Jot01L)$<3?S$q+AXD$7x8Py* z_(8dm^vTqS3#~VpIzj7O{+_$GZJKwRy^onu&Jnn8fQxF8(O9(^t{ueYfJi0lBAp;; zm`4yzUX!u5ykz_IQw=y##mjG)f9@W^d>uN;h?i8dtVZzWYSv1<&R5azrMpjKX4E7g z580M#7ZD{zk4z_LG9ZI!(l4FFX6|Ea;`-Y{96ZvUd2&)Fg?n-68A(ITv^XaM)s}V= z&x0&XOv9)pe&p6j@&^qjpJo*X=}%{whQ1eUT69Jl$;xVC_Shrwi%bP1*7P>F0fb78 zii4B20dIhb3UbH9t0y|R^m{$xv(8{z6Wu_u339F!|I%(w4tn#ZHYv_FHCBfgHy;6A z>R7eo=kPD_u#s?h@xvT8= zfm3J!&>?wDZJB%`Vm@4X0;KXzKAq^&o{j8&wN-FJ?g=c3-PLzI(*}Y;X2$x~05kWY zKi$W#jvktvxxOmHy5Vo@ZGVgJ{g0HA$(FadNp6IUSY1ZG%~Aty4|~FoOto9_#K#Ez zSP3O5wJ;Q&dRR}(2qXeXo%5dDbva-dFh;Dq9!xQ8e~e0u)Nn@g#D~0rkdsPd-GYIZ zGtGA{9@4moIPb;4T5fWqf)8K()(6w!=>|EG7&&Dye1K|}6q{v8TJx_4yth8`F%9P^ z|CMDs10!rj4!3N2H5zT+w1Qr5tVTxDzdcbUP;4!>0t1XaqLG zYINeQBWXvH6H5$x3uv8Si)*MfW|TxK;B5eHcIPGW$#QuGX@W*7Oi2d($&M{6g=o%C zMJrgBhTs-nK%QE;7M7s}R6_7_8K{r0vm^CZXu&o^(Ez|tQ(G7e{T-Qg>m;^$YC+Ub zYw+7K?MbUD&=<2uTJuVjGmdXCc0op`0u8>E;KqiGy@cgiAFi~5+r$zqUs`I#2%(9W zrtbMWK4%kBhH}{0aJ5e-E=m^kd4~t>oTI=rg+0VWOe?m?9W9{XVti7ZzCeI(3h(%P z67C&MFCVR8!;}yEu>``u@<0|eG%ByKx?qeXV8~edJNvEjKvgpla)Ws|W7_4ryp7}~ zZX-w626X*NhS^yWjYgdZiil+1F?2}!C`Wd8asbCdKK1_m4)O>l%sD=)UR~GJ-TB;S&d0W#<%jC8@qi{UG8d zFhjEnorWYU<0Fj0@z}7?VQ;w@bf{EIrf@RJzs+ z9~CL!S4^9`^2?$0HPIU0oUx+RT>D|{aU&ME>M$JHb@T`&`Jtjh_@|u~$+vXJ(6-O$ z&7X4Ozg7hVbm52t&^%q0o@de=t#+@W>D_(|%mnSgC=k?2wSjMV{v4QkJG51w*gXe} zsfhCL79lAR>2;v8K#Bv(84tgwz|pH$HA%1t;Rt>!-W+^l(ABC(H?XS+QO~r z0iaVpQ>Qv%d(gZjZ-5k)WXU=SOHN&xMccEhn>+&f2l95r)n=VsAsyicoY?1LmCk|d z%=SzDRPV6A`Y#nm_#q;sKMVNZ9c%K4O;-JV#9)Z8c=y!m74igeh z1_fjts6_T0|5oWT`E^PIq<1e~@pDFQkl`0I`Ps@Mg`V1Z* zK8l#n$-97L*ZrU&Ekvb2(kuWIUoWpTI=66%c4O-qoykLw#UHVd;lLpO!$GfDr3q;e z@l8I@+ld8nWGJ;yQ(zspJ@z-@Sf!ygwee(kn{mP)_X5Nw=Bs^^u*F;OUM)ieqTXkT?r5q*&vP()P2-c-v?zk% zeY1v-x@s9Zi<^J=7)Ds0r(l{`>9U4@ysGEnR1t$x7tGv+QUIdyj~L9N(9OitiEl)H zy@t>gAp8ZNx2iTxb;P?3r4i=af3cU~nwN-4HQkvlAn7AC6+31Dc_BO2fR#TOll;u&}*-zDeO-1^#fjgv+zsB$8}b94iYuXc|n8K zm6A(iGj_2UX3=(^YR1Rr*wbC2xXL4YrsH^43;9%x6=LolgW+ud4qBS&P3~bUcOx+p z?skB&!@52>4(avWe%WZxJ)a$W)w=sSJfYI3)n)+gRd4nmStI^$!?uS*fM3cy?h5G*bc(`L3_MHuc+F4^(L=BW6uw2+|j>3rj_?> zlLYXWz%}z%1me?Pq`~C*EuEGjimeiE_ABdRQWXhXHsCj8%NF4f-pus~rAoShuzpIR zz!GdNq4S1bnd`3)4u``nyfMU_30v`H45GLfo5=y<6MJmkxtig){EPffVCp4e`E;HW zup~uFON7)??MWm+7i0PNILKCAqRhYvYZJ*f>DIo|2^YILhBsp0URxiLLc{T%t>Z;i zMmQ}-mS~KQVlHr$G26)GU8##lmPyp6guOoYL4#Z|#?(;J&Qr0qO4oQ2k~dgnhG^qN zo!U_$X#1;Wmq}FA$FX%P@z*o^sdYo{DbX44CQwvvkJG~-6&a*<;MRl7#{7#=GR*{OFg%>B{SRpRWM4=Wx4DIX1!ZdOgw}mX2^t2nPf5I-UgkzAK^p==}JdWj|XU?eemua`5)@b*$3)C2Xox z{nKJ#Z3h+N=UH67J4e0lAW_1h9I-}D6_V{ePP#K@j@rHl71s?tPvjB&0CWYCNg%}= zKO`F#k55_#j@5<4)qLo0B?iR2LpQNj#n8d6&yr1yw^1?;K}-#q>>s8~FwJsn?S@1j zlJH_{9Y%p;OYKw<+JCjO%l~51YfIeA81~!dpwmB&3odxR9efxnMaJq`2^+ zold+SmZ2r702tC(aH6McL#7Sh-3>ic5PfcxS64j2n!AN5m^^-Q$4`^c@+M$8@7+p5 zJ*pI^cKh)NNDcX^m}*=q-}JYA$o;~i!N3%gXK!?=!wH3^n5rLE?Ax|WN;H-{%2}O( z;y4c>L8W}&TKIO*sWSAIUl+ufq>M<%aA7|-xXtwTJ;8%y)@j}Diu&N|c-{6W*d2js zTW-Nqk0T<7U0#=FZl+4iH=bc*r;MJI-^U#PVKxqN=Zv#A)Ey^o=Y+T{?L#P+@wcF} zPp4q!u4leYanQ5_*x22HVZGSUIma!(u6LaK*3R96#vxT$iiPz7{A>0K8K`Os1G&3` z$oGpkz5L>jphQR9PxY1K?=&gq4H9*9wln6GJZO;=P?iE=y6GY@6M?j5Mv^F)F$svz znC2GdHaRi^mHl)?5NjeaZTyD1Z zH3d15@*1hdE2|(Xrl3047;*TK1Cn7~sxq-)WGWQi4F(Lm8`}^dtJ*oj4_O2E$L`g= z2ZKqL@d2 z(Z)#_#W?!RpdL@{C$xN_a_xi}Xz28izj-g*Ufze$rExxTfX|>PGkL&`H4M;+ z#WWXFrg#PA^(SYZ^&FR0f-38Y!p?gpF)HIGv*!ivy5?Bzk=in5CgCLO5*#=OPp+x1 zDRZp--idVxW-_z_Xsg#$hE~`d^}*fj$Vr~u`bZNZzjvNq+aMjE%0r_xO$i-3GZ-LC zHha6Gm9Fh=E(&OfR-dfXyda^{Q#GZP3IUK8N-FO0+J&`+a1p-48r2v^DvqPKv$L2A zh1haZbn5G%+=K~exAJQB!rwWhL>8r67!+kkN&cX=CZc@c?fH1fieiC9B0HCs@3ek( zHlWp5=FJ;eQwhSs6~EFvc4Q(RvKu7^{R@$F?I`o4%5S+0`~sL%+tQ(xX|r%`+^a`b z0~TBBMiGHe(|WC{S?W(sXvoDP1rZ$I=Om}%1aTL62`$zCX zcJ0Ra{&P4CMHTe@K4j~IlvA<|w^9$$E;}OM|LUji4@M>NeIDGLP;yomLyDxAUu!E= z;c$U}Xz0m3-_>*L^U>*bQ7uWx2}%mxUv)1;OtCVb z)OdS1e@fFbz@CsF3Wu)-y-Owd1@#vHd5BHl$vK4fz*4Q!A3%k`3zDmCsM*yOgZWpi$fUZO}4O}=YjS0mv;!;~8KHZrD7pjwbNv@2M`Y$+}^EV0lk z^Pvvz^!`4ukJ~4DdM=4dSG4K$&8bY2MY_qtEOyZ>TJb=_PO$06n7XG#HfB{MmOwx}$f}rlVpg_cqDiy73yeR0D`_0)GWW=7s;VI0yE^fEBQXsnGrm zd!6C%YIqi*VQ6ZxB>bY%ztqy>wEz6^ucG09rd+45nuAG!Ic~^rARq|;opN>lZDabM zTx*glAOgUQ?9Fsf6tvk91FpAb>}d(-FQNmR+1E|mYy;=aGRHOW_K+QqKkVO_&$Kl0 zJTZ|TsmhlfWhQwN=-9mf(jUTVX}wGhNPd6vdAbBQ2Mw|fa#c_9O7Is{v)@7orDVwU zr13X`#*^6uhtI92U+7{ZB;BkWbr3#|s6oJ?e$jXb&&-D!j!73K7gj4$8K?na9jxRbbI5>0T8LwwLR9`~!B%|0^CdNAoE>v+eC>coZiVZkr9riIt^h~Ji}0tc;`b{pD$c3IN zx4F(I4-tn(s<=sdGTn{nKRHUr<2aJg20*tkakmn_nJPbV1c6V)pfzO?4~ zlotkK7wlub@NPuDvn*k_@e3F(68$g9bW%dm8UP6bqJsCYhVwr~6V4W z?YLmJp_>8gaL%RF39SZy#dwsGsmC6dorA<%S_~B<_4WGN+9Pv$SPKlx7bPV>Z~#>5 z)p&#r2mn9BTX?#Ke8}PvCcqB=oN$+0+su->T3dR%(i_aAB}WndBao8WdJ!x`k(nHE zZ-hNQ&PT#BA>SJICqM&5bqI_IbR$e{sbbNLqL1xSF^Y`X)AM`2fUv0i9AGT$3A=Mh z<`gtH+@DfD3j>ljXNgT@$}|o175q47!i>1lf z)<|#bea_`1;Kq6K8oig=T>ETUN5j?e6|!gHd(HBhi)T|-4yfT4lzcV4c*eu0M0MgU zz2EuX_Y1owrRt7nu#X3dbe(L4(peePIx&P1fu+=IyhSr_&&giAxIxa8>xU`V7J&0L zq{+=Bh1Zu%rIyH#UAE`5I`5!(q{GoDO0ES?D5U;|FJ}m065WaS{NSN5g3eCuC zZ^l=U8KDr2==cmOlK}6qUB(!R0OLszdsN-DI?--j+_}!U`)~59fH&seI21UK+8p1n z(-)Qd)0f1((CHYWPv`gJ*_(bM1_tvT#+bm}zSJ$B$D?JULctuLyW1PA19z=~uh)L6 zzwhtAxsktHkjzFh<-R|jk4DxCe0+2aQ(W8nONu|oNr)Bb>_Zb=Lp!sEI8acLIdo(6 zBdeHyL-corImv+~*skU_Gi^n5h<4Kux}gTw&)sk{AqX-qv4A-ltK6< zdPl42&J`KxcxbFXKw#*lIn)vPbGWHX3rvj_PG2$><_BB9OKK5 zgqf0z%G5jzg}>39^u;c(Ejb62ltTz+xaf@q<5ZxPq8{651Fsx7Meli^L5Jx!yi-w! zH%hSrc-el5X+wnNq{CduQZFccL|eSO4W=oT4RJtjV(c}yAn2%Q$(-j<_g4P;>h0?Y zG`|t>*EHC4-(1{qb6NI0JG-pV6|ip7L)R?pYXAMmyY3{RFj@XK=sSm^lZ2hjJeBMU z{g5~I!r269{C;Q&dC+iUD{rfiVFM&gjX(h{eh`;GVaK5*slS{w6e@*o-|7^6>fDn1 z@q98r4#$0Z?-F?UA|VlA$Av$O0KxIQ**MI^3+vCu8knr))z{=C>aq`>2`lX~R;Nmw zS-Enj)&tr=q*S%Q-t~-`VB%xFo-ui@0_&w;9KBIdO&b@GS#5*vJEVH#qI{I)v5lMK z0u?w^rgg@Q;KZ&nUPaTtOEs%|s*6XvL0cNQ%13It1N08NIB%3Od*PT)I{D*c=ou-g zB4?EV*6d5TgtrcJkCeZvf`_R=Yh^c{jWByQ;%b8s-c%A-O$eG?_jDnLizZOp&!lZ}MhS8zXJ%-_19(P89EXc%s!i8}rC~G&sAoLZ z()4k+XzSFau<2Wy^!a0@vWN6`lE`e&L7fQx`BK_peRB}8FBA0vX~O7jfs}{ugDiMW zu;q(mjC1`35uHfT*jps6HD6Ml*sR^GW$>#s^N!!hhGA9f-CN=3(PaAg)aYO&o2!-4 z@RF-dad~e8l?r+Y<*M)y*O1pDt&jU|;oJ5xLF;ZV3xzy$uQ`Q({sVp*pWl89{^VTs z-~VOEs$=D%m?s4Rksb#D!Tzt1^{>v)$->ms*@@{tKmUnaQQ11N1l*26TL{mBY_A)I zHKePl{t}UTOqd*o;32is{ByHXn$ir;Pg7N2U4f|;Cd@dxf8Dc31sv)VqKwtD?PctJ zpO$M4YJ9r*{(AYmKF&Wp_D?=69su4-k#hfj+|EDvejMd~?tH$U9z5ItLWh}tY%vPz zeO(UKHk3{rT$1AKf^%p?s|9rIC*$@i?!eE1m+}PUY~FK=by$oqrAL4 zK3|_Bu3u6)nYMD7Vu{8=yM2B)Y^|?%1C3q)E^5y=FJFZ>_cwb-PpN_eU0>tN<1Ndy zw_Y!|&k?T%0^XmWjvtp7q0`voP|gotFVFK2S1m+eafE#jnQr#uZeLGFN*!nuSy?G#FGU(p_v6cVr#lqLeZ#PR z@$1p+VdwVdhRHzx>*J)wx83XWV4lu-4~fP1_+uW+i~I9xae2CJZ&|~D=u7+asb!i- zU+}}(8_x&Y=r0T)r>54YyAxP`=k)pfu=w`3vs1wPBllH9XTE~gQ>FKdBN;)wR`9(< zT{&haHnukQYEwY)!!=mCN2ANjcS5E8qqQ*m%IWrrTW3g7&hEDM%5Jm!1w-C_>vI?B zld1WRX;bL!9ZZ$`1jtB zJDZEI12>0GAqK)DIUY~uBx+aP$F@)EoEXo{7-Em;Uf2*sR{oFtMJ+pQoaM}=KL#f= z46Fw%^n-O87e@>FxTG@67+NXYF=q2%vpr4EChN8MAPWbV)CHfM>@usLCOL4S3QZH}r!$<+Y`WaqyPJO6Un&g?v`jC(nd2b@ z&FUTn9<)uh1>b6I9F8o>RW`IsrCVfnP?yR(-J?i2WuH#eO-CoqmmUh>!W5duTwY!P zj*p+(5*^o6C8>h3A=1VMds|f}zme#B1b&uAiJ+F%F=B+XorIAiojuh5B zsV)i=nKRDdm(XgQE{nYUjfU-FEM3&8DC$oCGa1`OgM0&z^wjKshHl``vsEUxvw{-6 z{nu1JnPx5tx(`lN7hQ?2nOh}V*^LxBCaM-`Iy9xzAtAFe*pp~PoV*4N$LrGhZptlI zR>jzp1%PPOY#o|=by(LV`9>}2>Nn;mWd*>Fj<9^A-pI!4?m>TUCe6JWY+*Wc2GZA( z$7F^RYw)IA@2O>QGh92b10{?T=RqUE$1!D8%5RT9`cB18yuq6aMvZ6^4P{v+ZP_CB zrI*Gf1l~<5QXIaz$Ll}xpH#3vfS_8aGDJHG9^9|aSvcv^4O(DcOvnZgwPic5v0ad_ zvjK)o&Z5|l>>|1{+hwOpRvB;F(ZC|7Hqbpe#WSNRwKvGwkIW*v(hCSo=BvzfF$vt>ERTOfAU7@tZSu)@*rfRFWIp=NdO8~QS*bJ;&dLZ@3>+&sK znRXXzg&5i!=tNbE^gM3!wpS%Oyryikr<8AuDOsK;%};X||JE9Kg$vwJx{XKmc7p1l z+4_w04&X|Q0GrY+KrU)iN?T54I|F#jJEAQwY+B@I5&ht@bxxXF=Bg*{(QM!5UWj8T zjuhH2vYmN(Uo2K3S2qJBp5u?0PRTnjXC6Y(JA<*l1^; zG$UmNv;J>m0RYf=A%B|RdD-)LN&rBf847GMB>BqLYY{O7&348uP#kZpi*nm$6oxWq zuy@eWSRH+=i*b8fx#DT@yGEH!UJRu$L{8~BZ*`QtofL;RZ%FZ@cICcNmsTs&m4&Ml zxgbrn&a}l|dJ=8K3iMfUd#uj2C0=?`ZNw_{SzxM7+g%+gj9K!!^JcaL)+(G54qo?8 zXFG54mRFWqs#BGB#J;s9JFcDL199~_S9Y>1e%q9JDoke9W(&-QecnFOZ@=nV;Lovx z`nw)Wc&I*ha<@V>^Z{MiHgyS^JLZ+fOH(R8uOQqc7iA{zqV&ehsjQHBXh9BanJT<@ z3cLw30QH9rRbm`}@@X=(E!uI7ToTX7nS$GGR`jYMbXFXDGGWOiC$BA>&;yNs1=j{` zq2^@3f13Y(#zok zc1gKjkXS=-nGb+Bt-*FaN?Eh>_(9_6T~?FOz|pekI{yY6pJ@A*_~1?Lc3FK+*CKSaXd_hLRI*p&0brGY&)lCvCwS%RP0AhrFJ-O}bxNb86?|EVNfmqsDIr z6=W5?+l*l{T%!1MUABASv}l;{$#jIhwoSck4-E3&q=CcrB`%l;z$m3qcmk(}Gp~GR z%aBYN6?>)-g-rLni=B0&7-#;vfxoQHfGjcYG*5KepKhdjV{h_|v3%8TU12#+0n1{D1|msnWDRjcNrDYNvVGzGl$ zNfJz*pY-ia_-n-L5=n!H%y1?$ZU?kzW@J;wXG@>6Ay&uGwcEx z7TSb+F!5$F!?J8T;#6%%Oa^V1!dj1MwhGr>DQM0xJn@9FG7A~w%1iYDq9Kp4wJ7d& zE`Q+(A8NWNXCk9l;KXXWCaqoPu*WdW$leRKe#JUwuAaj|=^1g5GIN9)6058MK12kM zfiE0f=OTzr@Sxt{a`Y;!(HbokHJOh+lH}1LE4PF}KLjigFL2nzjhX$h;69G1j`oYH z_B`Hror!~YdCORhOb3LG9_NpUHn&Zl^C!`*OK1))O}^#>*3CX}oPz;j6D#d+38F1} z(iL!Ts#!quys=F)cE?2J{vL1 z?Z4v=?Pu8x;5hS#q+9IEQ|ppgpCQW-6?G_;{ucBtv$$3aCiFkmfu-x2d+c%LZyin9 z%XXBkCG8_Rc=+{oJ5DdVTkQfPu$3qcb=(%<>7;a(7z$)|Kb^KAzl%3_wBnBeo3204 zrUGqQ*gOgWc|d&FcR8Cc{b%xZswp`mCamPlqiaYB27q(@UNZ3vUJTn^!5{LMv%P97 z31x(?L>aB)P6AF`slWwxxDs|K1c^1(koMFc%S)Z#1xP)X{!#URq_QP0mptb)hGP*?FO1LWe_GvVv|OvNH>5HW;HMqK|U zBewUMiACBVVhNX|&{4@hN5#cYvXG4|cnsq9M~am+sNWvZV9ox)zIERu8K}BQM@**= zNw_kREIhcOw^qo%yu?9FAA97qe5TW|W!HYX_MFMKw{TCCYPlhHY+tn8_#qN_=_Lw) z!4=+j27sfy^#OIiDZ30sOrgc2`c2+qCOgxX$_gTWzS``JFbkpGspUc}8aQ;)whzN| zNVMb(^u;cHRb8hg{JrhqdnVA>1|pu$?L*20Yei~Qr)`76mafH!tXZzLBLH_b99Nhv z1yF{PIVr537F`zK#fwI`$jC(K1{$5@wG5&28fw!a&+3G90OS)}y<3ZXsNF-xZ__vo zbutT+E1xZH6M$_-^@%W$Px_^GZc)8JnlCXV{HI-B$7=zRZVY4nMogC_F)?zPhGG&? z#RxP_i|_Kx^KH!6mao?Hk2>eJkDMSO``x+zyn+)5AdZXYKjPIhZwiSjyM#41p)!V; zJ>Hgx()1})x3LzBI^Q7hwPqZW#X4t(1bP>#gsWr}v5L!dKlVQ#z*;3>0&ZJ!#~+Uk z$Q#m*J@Gi2_6@Yce0?K!pCbv4rP)m@_O_v*|J#zLJ5PDc^Sj5hP9K=jn>4qeG1j~N2titPg(+X7bC1t;$uR>$3c9_#jb}ia$_``o-*pca*Trs92Z8X=d`op&y|0FXsxq6NUYM5w+N;`o5VNflxXcA+ z@*rfqpk!fL=dU61*QC5Rr2mB7i1%(C5EVQzg%A{ZzEP`W8_ljOrp2W~qN25L8_un5 zGJJH4npodwDHeGig9+rb43s&oi8kY{UR!fQ3}d6L)S_-(H41$_uO>vgc1;3?0GYap zTNU&_ZR;ak3b@9pBklMp9#`Ieh`xkTHKI@jU2ykH#|@9pSOLdmuvwyPOIIDaJu>QC*>3IR=k zT=~+JXkgVZ+<4LsGT?(I)fQn(QE2LUl_An;t&xcT4GlwI{b#wT_EZIiv3inC7&g_< zPg$*XG2zq}fW8`zE)2||kJd=X>&1~$*q2I5M(p2sacC_61VdjvStJb8ZJM;Of|?Q= zIk!b+4gOyO*$fNN+tv5H5ff-w$Mx_ab4f033%tC+S4hhK~!h-Qx8Fd`jEvP!P% z5p(SmL(b;6S>%u#hzY2~NE%T{f+nTXu`QWm2SZ(jK^5LNsho6u>(19TVl5DRyv+GJ z?prg%flO!k79i#d6;9lc!)nCpfiQAOjAF7>-)A-ub{>QBPm#XFw`O#ASdGP8hr@_f z#+z30or7ngQCA7b1LUfobjbs&m~mNvyCxP_UT+l*BV|G*1u?6ouJaQC^(5!cp(_A& zH5~YO=gP%}DKhH#Y-*s=Gy}0j`UgBkJ+BNz`kbDNF>>5U&YkO=OPmxT(=@1RQ=cV7 z_&Fk~uo(l;x8p`G^)Uh_wAHbriQ_kfNk>$*+R%e$ff`Cc5irrNYs6L{<|+xq%H)DJ zKqD7V!um`m#3mJyo!NHy z)b&JpBfqixi(m>-sOotYAkrWH{iUqum4ir6o(;$ollim(E4VW}(;nG7=uK;Ng$F~U zc~Nk4)+RK_8V~Fdlj%+*h)x-9s5-Em5-Aa;l+EAg)#7a-)YHUeFu^vV1=w3W(=D$b z6voAA#g*ImhKBrgtY0*DMHfXo6o3l~1BRhVb90}#YydqnNx<>@^DN*Y`6qIa)To=2 z8QlGh-Yfi72gam0@G-eT`_sw6)(Sfu&?4mfohPn;OIjT9Nai+%klOD`WBid(#D03> zks4p%m>eMy+*~m!wV9O_u3NF`{AY{1G{~LZNJ4GcGql^&A=0@6pyRipfy^!qmJ5(G z9ktj{>s7L4&v@~$2w=tqCG7r zJuof?*849ucZl-diP{6lAi^Lb3YXk3rEC(@c}dtv{jvdS&=Mo5rg|QU{|vI&QE#)# zPf=0TCNKn;pb#L+XX6sWA=?J%=hLvjgSlkWqL~!{#ZZcv86v*2d2Ir7b4F+9`>sg9 zo=99do$x8}^nitmj|G&~rhedw7Jy?!(rZLALXTCGCjzEK5~l04%rSmFe6B01DD_Jb z`g`%(l&>gO&@?nG3?i&>$!Sh#Z#@_+3`8(7kKm@dU7*MWbzoJi6*)CI7c~&EEEsBX zWc2^5oojz&djJ2!WRuGb#Yr|c4w3u4vQ3K>iaKbaj@T55lbX_Htfke#MkLV%QMr`J zC7N@Ta=%rSIc}kwbU}r~d|&TPr*h8sKlt#2*Zci?KcBDH>v?%RpRc#q`^l>F@II=W z2e^~8s!3XP`FX1LSza8t%n9JlW`5=@2i9>1o_UpWfMe+KhfXeSUc*3-zqkw*D(^`K*WuC`I@p$7_feMKF!MsX)2{s%gYIlOV8995i z;3zZ#0dVK((>O^4(zfqF^;Euu7F}h#EV!I_fg;}!yfH7iz8H&d`+Y6gnO0ota+m7> zDIp52fnvE(Wf>~R^9$Aa*7))-G1%hI#|zaZw5lbv>Qt!ST0R9Z2T2Jn&cj|SEV<$I zXsmdG8%kcB$=G)5!aR=z52MDvYTq_dR#U=OLk6~_*R23(Q4?>Y2~RzsT;qZJh^_vJ z1uTb)nvS}p5i3&%N%yLcryB(9Vuh&>V5^q%1m13I8N#hOI3@kOX*>zJ1dmz_fr6H|Z%zv`b?V$olbs}% zKZ!@RCY8r-<@IkxfrQB}F7So_wv}V0>+F``EtZfkD4UI&ChMfwimp5jd>joaVTK5^ zkzUypC0VqFEUPIp8u_)k+4C|s+6_e>tO(KLbg@UVZaktM2tAdIR4hp{0SkGZtV*TL^cpa4L8k6_8yw!;xqXoCJ0uEtYDnS4~Q?7NhYfB!KHqXUE{_HaLh% z{**8*@R4o$274T_qINI=k*0(|Yam!n8?0neSB6^6ahVxqhmPDDAy-Ey9r<~cz>phA z^=`GGxrbqO`sxKwo=BgqP3dT{E~bPoCJmi;^VNN~KC_>=p(?M_{@5?xzVv4G^Co>w z+I$1x(u_m(YUb}vyP7~ZV?R}M$7ja7B8mn=+iP?z(NtFM%1~v{Xwtq3#_J+I6COyv zN3|XSDbp8oue!Kl6ZBAIf90Q~Gd&$**3h>&qA2@O2s2tkK*{*BVhRdy3F&TXnQ4O$ zn6ZeN$B8_u*Q&loZ4#E#k}F8EMAUSSyA_0~^L6ozKR#fJf>UVE{OxG7I{pwF=M*FY zkA}1AEd54{5%uY3KGDzT%EAk z1Qg^l!>ax&2oPO5>{1588^JyXA#ti_Q(Ln_FdR>R0%lnUby>n|I?cODVj}<0rw9r} z2kv?D&CKVw+m^r!aSJoV-h6!yFQ)n&z zbeZRb%s~Q=`q}dX#`NWAdjBMM^~F|X5%M05%_N^>g2biclzTrW;e#UiMp*~#Gcl?a zDLKcn?X#@*HMN?_s#MnmXZARAocK8+y+<1OgcfE0q>R}}Hg=EWq!ZM1rbaW#_Xw!C z@Xim4mDl@wWoK|`VKkKXbkRTt{PnqPWP>70$RUH-b$~9GX0ECgZW_m!Tkc+2>irnw z-iFZ$Jr*|Mq`AXa4>65H&qmi-X6&VRre}cZl9^;DFx}i_Xwxcc7jOS|JD|Y!FF1)w z&IHrohfJJIQdLwxCFH~liS#?@nh~OQI|gLO!6`^F+xi=!O=iQU-I3yFax6#}s=;+~ zb}c4e$O)dt$xdetHQ$XF)(skr(;Fa=x{!attNS|Lm09C*$c2{hJ|N_!eL!AgKq%Nz ziw4TEsP;ruy0l>uUIArD22Y#}aRm9kdON1Q9aCp`b-w3@;FC_)WdbxIu*FF<-J)5X zOq_7qKLKQ_uWXcM9SyCs_+3G{>DTsIU?I~uyJ;yg42$?dW-&i*-HLHp(Xw{={5n7N z(O(8%aaUK_nTk6=xc1de!)h{7GfKlFpRd6t^aHgEJ`E~LFv%;J^)Fa;9rZQZM@3fH1PIMqiK zm8y2iui+ZgUVfqJuhqJczw+<3j5{ehAJGt8)$$FwDM1CRZr@A1OaN~>mxlmw^gS0! zKLw<=x8(_17o|Xcz8e;zOW(^*ghi%tz9JL{-_V%`+ zN?n^2%jx@7@k`Ggl%|NE8#N;_ucW`vJzxYB`02SOoNMJFow*J^NF|+(9NVeLp8r zG^{ZU+HN&IN&06HU)_JrW>0wP@`r6cztvMFo%R6=I$z~QD*?@iL2pxloFsYVNoe{Y z>0M2?!p-1pXQV*5G<#v1ysnqA|0MHD4D%W5g}(BnllF`hNR};y32>qNm}kL#!r0`S zL;xqJAUFY98?SF;PmOa${JcZxJFM$YJ3HPZB;E&*B+&AQOdrevV92`8cOKBoDa0H_ z0RW9}!Lz3M=1PzTh@XKlZ=SXRK#Y($a~G63LJ9;Ld^8)l7u3wD z%XR&Ei{|5?-NxY)PTL-1)dPX_*#!8kJHWR}V*tX?>Oj&m{M+9ifwS+pN&4K^71{izcFa_GUN1ls zV7j|}ka0q{#fAX6f=-FW1%)f|_`j@r=2wV;i{)EUSo%%A^ykAOyVwYJEn{nyYvR-g zF!cjKA}?L`(AiAu#ms$AgqOhwF4~^8!mc(0<*SCid9xd@mTxKTKW|o@GGpni@9hlirmVS5>5sd1I;@sDZU1FB zLa*vSYpsPIi$F(#9w9J2)c2<67Z&zE%Y$rA{F9zY3LZ2;op^s@R?+c`_9`5{RKja};Kq54U#bpY z=Df7@#6c>-XKsmlA~O4k4J3foWGXpe5y2tUyW>xPF|72 z*XqqFi1ipYG|W8Oy%uZRIKBg|0MB5LRZPjLtbKkr`+dyWcE*u%DrVWD&NrpI?-s{Q zo}I4mQZe;R>cJQu)TzzfS)ux@Z0OasaGhH3lA?$Am&=Sa9E9t%%6~%SEE7B`XKyR9 z2(8}iqFxc9+3gZ=b5X{9eR6Q3d9+$yv3y$9XZg*KnEnDkdEzzvvKab~c>bVZ_Q~Vk zKObb@TC7IgT2A&^H5S#{@d!W7I%uc+PJL0YL}159`M~(8X@l|ga{+U|`JFUwwlk8e z-pxpHEy;CdCDtB&1ctBkcdlA|C(ren!=bSmA-s1ywLT}jr3gg;ia{GQMo9KZkn z>sv15v#`qHH%?qom3cwU!-d2PtMz>&hRceG|5dwhA%}$@pM2vG0iqNbKK!3gQ5KRe z{M6tZX^p;!bm4~w3mGgt`1y@Ngvmck|JO0lg0$6!#D|GdL-xstL literal 0 HcmV?d00001 diff --git a/rules_entropy.txt b/rules_entropy.txt new file mode 100644 index 000000000..602e39fbf --- /dev/null +++ b/rules_entropy.txt @@ -0,0 +1,21 @@ +contain obfuscated stackstrings +resolve DNS +initialize Winsock library +act as TCP client +encode data using XOR +create process on Windows +terminate process +terminate process +link many functions at runtime +copy file +enumerate files recursively +read file via mapping +read file via mapping +enumerate PE sections +resolve function by parsing PE exports +copy file +enumerate files recursively +read file via mapping +read file via mapping +enumerate PE sections +resolve function by parsing PE exports diff --git a/try.py b/try.py new file mode 100644 index 000000000..c4e6aadc3 --- /dev/null +++ b/try.py @@ -0,0 +1,54 @@ +import os , subprocess, pickle, tqdm +import pandas as pd + +def get_files_with_extensions(directory, extensions): + files = [] + for filename in os.listdir(directory): + if os.path.isfile(os.path.join(directory, filename)): + _, ext = os.path.splitext(filename) + if ext.lower() in extensions: + files.append(os.path.join(directory, filename)) + return files + +extensions = ['.exe_', '.dll_', '.sys_', '.elf_', '.raw32', '.raw64', '.cs_', '.aspx_', '.py_'] +directory = r"C:\Users\HP\Documents\GitHub\capa\tests\data" + +all_paths = get_files_with_extensions(directory, extensions) +print("Total number of files to be processed ", len(all_paths)) + +pickle_path = "./all_rules_entropy.pickle" +write_path = "./rules_entropy.txt" +pbar = tqdm.tqdm +entropy = {} + +for file_path in pbar(all_paths): + cmd = ["capa", file_path] + + with open(write_path, 'w') as file: + file.write('') + + try: + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + except subprocess.CalledProcessError as e: + print("Error running capa on " + file_path + " : " + str(e)) + + with open(write_path, "r") as f: + for line in f.readlines(): + line.strip() + entropy[line] = entropy.get(line, 0) + 1 + + with open(pickle_path, 'wb') as pickle_file: + pickle.dump(entropy, pickle_file) + +with open(pickle_path, 'wb') as pickle_file: + pickle.dump(entropy, pickle_file) + +def save_dict_to_excel(data_dict, excel_file_path): + # Convert the dictionary to a pandas DataFrame + df = pd.DataFrame(data_dict.items()) + + # Save the DataFrame to an Excel file + df.to_excel(excel_file_path, index=False, header=["Rule", "Number of Matches"]) + +excel_file_path = "./entropy.xlsx" +save_dict_to_excel(entropy, excel_file_path) From bf1f59b3dd0b0bc30d61f59d83a4dd3890aacbb8 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sun, 6 Aug 2023 02:59:00 +0530 Subject: [PATCH 02/32] Sort rules in render based on match probability --- capa/render/default.py | 48 ++++++++++++++++++++++++------------------ capa/render/utils.py | 12 +++++++++++ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/capa/render/default.py b/capa/render/default.py index 795677d27..423aa108b 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -6,6 +6,7 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import pickle import collections import tabulate @@ -82,46 +83,51 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): | ... | ... | +-------------------------------------------------------+-------------------------------------------------+ """ - subrule_matches = find_subrule_matches(doc) + subrule_matches = find_subrule_matches(doc) rows = [] - entropy_dict: dict = {} - import pickle + # Step 1: Load entropy_dict from the pickle file with open("./all_rules_entropy.pickle", 'rb') as pickle_file: entropy_dict = pickle.load(pickle_file) - + + # Step 2: Combine the loop and categorization + common = [] + uncommon = [] + rare = [] + very_rare = [] for rule in rutils.capability_rules(doc): if rule.meta.name in subrule_matches: - # rules that are also matched by other rules should not get rendered by default. - # this cuts down on the amount of output while giving approx the same detail. - # see #224 continue count = len(rule.matches) - if count == 1: - capability = rutils.bold(rule.meta.name) + entropy = float(entropy_dict.get(rule.meta.name+"\n", 0) / 593) + capability = rutils.bold1(rule.meta.name, entropy) + matches = f"({count} matches)" if count > 1 else "" + + if entropy > 0.3: + common.append((capability, rule.meta.namespace, matches)) + elif 0.1 <= entropy <= 0.3: + uncommon.append((capability, rule.meta.namespace, matches)) + elif 0.034 < entropy <= 0.1: + rare.append((capability, rule.meta.namespace, matches)) else: - capability = f"{rutils.bold(rule.meta.name)} ({count} matches)" + very_rare.append((capability + "*", rule.meta.namespace, matches)) - # Get the entropy value or use zero as the default value if not available - # entropy = float(getattr(rule.meta, 'entropy', 0.0)) - entropy = entropy_dict.get(rule.meta.name+"\n", 0) - - rows.append((capability, rule.meta.namespace, entropy)) - - # Sort the rows based on entropy values in descending order (highest to lowest) - rows.sort(key=lambda x: x[2]) - rows = [(capability, namespace) for capability, namespace, _ in rows] + # Step 3: Combine the four separate loops into a single list comprehension + rows = [(f"{c} {i}", ns, rutils.bold1("very rare", 0.0334)) for c, ns, i in sorted(very_rare, key=lambda x: (x[0], x[1]))] + \ + [(f"{c} {i}", ns, rutils.bold1("rare", 0.1)) for c, ns, i in sorted(rare, key=lambda x: (x[0], x[1]))] + \ + [(f"{c} {i}", ns, rutils.bold1("uncommon", 0.3)) for c, ns, i in sorted(uncommon, key=lambda x: (x[0], x[1]))] + \ + [(f"{c} {i}", ns, rutils.bold1("common", 1)) for c, ns, i in sorted(common, key=lambda x: (x[0], x[1]))] if rows: ostream.write( - tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50)], tablefmt="mixed_outline") + tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)], tablefmt="mixed_outline") ) ostream.write("\n") else: ostream.writeln(rutils.bold("no capabilities found")) - + def render_attack(doc: rd.ResultDocument, ostream: StringIO): """ diff --git a/capa/render/utils.py b/capa/render/utils.py index fb3932340..077bc6d9a 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -19,6 +19,18 @@ def bold(s: str) -> str: return termcolor.colored(s, "cyan") +def bold1(s: str, entropy: float) -> str: + """draw attention to the given string""" + if entropy > 0.3: + return termcolor.colored(s, "cyan") + elif entropy <= 0.3 and entropy > 0.1: + return termcolor.colored(s, "yellow") + elif entropy <= 0.1 and entropy > 0.034: + return termcolor.colored(s, "green") + elif entropy <= 0.0334: + return termcolor.colored(s, "blue") + + def bold2(s: str) -> str: """draw attention to the given string, within a `bold` section""" return termcolor.colored(s, "green") From 31bd6b3086c16e76094090ea9dec9dde6912954a Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sun, 6 Aug 2023 18:28:44 +0530 Subject: [PATCH 03/32] Rendering rules into two sections. * for interesting rules. --- capa/render/default.py | 79 ++++++++++++++++++++++++++++++++++-------- capa/render/utils.py | 10 ++---- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/capa/render/default.py b/capa/render/default.py index 423aa108b..5a437fdd9 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -71,6 +71,64 @@ def rec(match: rd.Match): return matches +# def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): +# """ +# example:: + +# +-------------------------------------------------------+-------------------------------------------------+ +# | CAPABILITY | NAMESPACE | +# |-------------------------------------------------------+-------------------------------------------------| +# | check for OutputDebugString error (2 matches) | anti-analysis/anti-debugging/debugger-detection | +# | read and send data from client to server | c2/file-transfer | +# | ... | ... | +# +-------------------------------------------------------+-------------------------------------------------+ +# """ + +# subrule_matches = find_subrule_matches(doc) +# rows = [] + +# # Step 1: Load entropy_dict from the pickle file +# with open("./all_rules_entropy.pickle", 'rb') as pickle_file: +# entropy_dict = pickle.load(pickle_file) + +# # Step 2: Combine the loop and categorization +# common = [] +# uncommon = [] +# rare = [] +# very_rare = [] +# for rule in rutils.capability_rules(doc): +# if rule.meta.name in subrule_matches: +# continue + +# count = len(rule.matches) +# entropy = float(entropy_dict.get(rule.meta.name+"\n", 0) / 593) +# capability = rutils.bold1(rule.meta.name, entropy) +# matches = f"({count} matches)" if count > 1 else "" + +# if entropy > 0.3: +# common.append((capability, rule.meta.namespace, matches)) +# elif 0.1 <= entropy <= 0.3: +# uncommon.append((capability, rule.meta.namespace, matches)) +# elif 0.034 < entropy <= 0.1: +# rare.append((capability, rule.meta.namespace, matches)) +# else: +# very_rare.append((capability + "*", rule.meta.namespace, matches)) + +# # Step 3: Combine the four separate loops into a single list comprehension +# rows = [(f"{c} {i}", ns, rutils.bold1("very rare", 0.0334)) for c, ns, i in sorted(very_rare, key=lambda x: (x[0], x[1]))] + \ +# [(f"{c} {i}", ns, rutils.bold1("rare", 0.1)) for c, ns, i in sorted(rare, key=lambda x: (x[0], x[1]))] + \ +# [(f"{c} {i}", ns, rutils.bold1("uncommon", 0.3)) for c, ns, i in sorted(uncommon, key=lambda x: (x[0], x[1]))] + \ +# [(f"{c} {i}", ns, rutils.bold1("common", 1)) for c, ns, i in sorted(common, key=lambda x: (x[0], x[1]))] + +# if rows: +# ostream.write( +# tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)], tablefmt="mixed_outline") +# ) +# ostream.write("\n") +# else: +# ostream.writeln(rutils.bold("no capabilities found")) + + def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): """ example:: @@ -93,32 +151,23 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): # Step 2: Combine the loop and categorization common = [] - uncommon = [] rare = [] - very_rare = [] for rule in rutils.capability_rules(doc): if rule.meta.name in subrule_matches: continue count = len(rule.matches) entropy = float(entropy_dict.get(rule.meta.name+"\n", 0) / 593) - capability = rutils.bold1(rule.meta.name, entropy) matches = f"({count} matches)" if count > 1 else "" - if entropy > 0.3: - common.append((capability, rule.meta.namespace, matches)) - elif 0.1 <= entropy <= 0.3: - uncommon.append((capability, rule.meta.namespace, matches)) - elif 0.034 < entropy <= 0.1: - rare.append((capability, rule.meta.namespace, matches)) + if entropy > 0.05: + common.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold1("common", 1))) else: - very_rare.append((capability + "*", rule.meta.namespace, matches)) + rare.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold1("rare", 0))) # Step 3: Combine the four separate loops into a single list comprehension - rows = [(f"{c} {i}", ns, rutils.bold1("very rare", 0.0334)) for c, ns, i in sorted(very_rare, key=lambda x: (x[0], x[1]))] + \ - [(f"{c} {i}", ns, rutils.bold1("rare", 0.1)) for c, ns, i in sorted(rare, key=lambda x: (x[0], x[1]))] + \ - [(f"{c} {i}", ns, rutils.bold1("uncommon", 0.3)) for c, ns, i in sorted(uncommon, key=lambda x: (x[0], x[1]))] + \ - [(f"{c} {i}", ns, rutils.bold1("common", 1)) for c, ns, i in sorted(common, key=lambda x: (x[0], x[1]))] + rows = [(f"{rutils.bold1(c, 0)} {i}*", ns, p) for ns, c, i, p in sorted(rare, key=lambda x: (x[0], x[1]))] + \ + [(f"{rutils.bold1(c, 1)} {i}", ns, p) for ns, c, i, p in sorted(common, key=lambda x: (x[0], x[1]))] if rows: ostream.write( @@ -127,7 +176,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): ostream.write("\n") else: ostream.writeln(rutils.bold("no capabilities found")) - + def render_attack(doc: rd.ResultDocument, ostream: StringIO): """ diff --git a/capa/render/utils.py b/capa/render/utils.py index 077bc6d9a..a858ba7d9 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -20,14 +20,10 @@ def bold(s: str) -> str: def bold1(s: str, entropy: float) -> str: - """draw attention to the given string""" - if entropy > 0.3: + """draw attention to the given string, set color based on entropy""" + if entropy > 0.05: return termcolor.colored(s, "cyan") - elif entropy <= 0.3 and entropy > 0.1: - return termcolor.colored(s, "yellow") - elif entropy <= 0.1 and entropy > 0.034: - return termcolor.colored(s, "green") - elif entropy <= 0.0334: + else: return termcolor.colored(s, "blue") From f5f3e8708db90979a17fe77f0a73ba1ad077c130 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Mon, 14 Aug 2023 00:29:09 +0530 Subject: [PATCH 04/32] update Update default.py Update default.py update color format Update default.py --- .../rules_prevalence.pickle | Bin capa/render/default.py | 99 ++++++------------ capa/render/utils.py | 9 +- entropy.xlsx | Bin 23367 -> 0 bytes rules_entropy.txt | 21 ---- 5 files changed, 34 insertions(+), 95 deletions(-) rename all_rules_entropy.pickle => assets/rules_prevalence.pickle (100%) delete mode 100644 entropy.xlsx delete mode 100644 rules_entropy.txt diff --git a/all_rules_entropy.pickle b/assets/rules_prevalence.pickle similarity index 100% rename from all_rules_entropy.pickle rename to assets/rules_prevalence.pickle diff --git a/capa/render/default.py b/capa/render/default.py index 5a437fdd9..102522280 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -71,64 +71,6 @@ def rec(match: rd.Match): return matches -# def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): -# """ -# example:: - -# +-------------------------------------------------------+-------------------------------------------------+ -# | CAPABILITY | NAMESPACE | -# |-------------------------------------------------------+-------------------------------------------------| -# | check for OutputDebugString error (2 matches) | anti-analysis/anti-debugging/debugger-detection | -# | read and send data from client to server | c2/file-transfer | -# | ... | ... | -# +-------------------------------------------------------+-------------------------------------------------+ -# """ - -# subrule_matches = find_subrule_matches(doc) -# rows = [] - -# # Step 1: Load entropy_dict from the pickle file -# with open("./all_rules_entropy.pickle", 'rb') as pickle_file: -# entropy_dict = pickle.load(pickle_file) - -# # Step 2: Combine the loop and categorization -# common = [] -# uncommon = [] -# rare = [] -# very_rare = [] -# for rule in rutils.capability_rules(doc): -# if rule.meta.name in subrule_matches: -# continue - -# count = len(rule.matches) -# entropy = float(entropy_dict.get(rule.meta.name+"\n", 0) / 593) -# capability = rutils.bold1(rule.meta.name, entropy) -# matches = f"({count} matches)" if count > 1 else "" - -# if entropy > 0.3: -# common.append((capability, rule.meta.namespace, matches)) -# elif 0.1 <= entropy <= 0.3: -# uncommon.append((capability, rule.meta.namespace, matches)) -# elif 0.034 < entropy <= 0.1: -# rare.append((capability, rule.meta.namespace, matches)) -# else: -# very_rare.append((capability + "*", rule.meta.namespace, matches)) - -# # Step 3: Combine the four separate loops into a single list comprehension -# rows = [(f"{c} {i}", ns, rutils.bold1("very rare", 0.0334)) for c, ns, i in sorted(very_rare, key=lambda x: (x[0], x[1]))] + \ -# [(f"{c} {i}", ns, rutils.bold1("rare", 0.1)) for c, ns, i in sorted(rare, key=lambda x: (x[0], x[1]))] + \ -# [(f"{c} {i}", ns, rutils.bold1("uncommon", 0.3)) for c, ns, i in sorted(uncommon, key=lambda x: (x[0], x[1]))] + \ -# [(f"{c} {i}", ns, rutils.bold1("common", 1)) for c, ns, i in sorted(common, key=lambda x: (x[0], x[1]))] - -# if rows: -# ostream.write( -# tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)], tablefmt="mixed_outline") -# ) -# ostream.write("\n") -# else: -# ostream.writeln(rutils.bold("no capabilities found")) - - def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): """ example:: @@ -146,7 +88,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): rows = [] # Step 1: Load entropy_dict from the pickle file - with open("./all_rules_entropy.pickle", 'rb') as pickle_file: + with open("./rules_prevalence.pickle", 'rb') as pickle_file: entropy_dict = pickle.load(pickle_file) # Step 2: Combine the loop and categorization @@ -157,21 +99,42 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): continue count = len(rule.matches) - entropy = float(entropy_dict.get(rule.meta.name+"\n", 0) / 593) matches = f"({count} matches)" if count > 1 else "" - - if entropy > 0.05: - common.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold1("common", 1))) + + entropy = float(entropy_dict.get(rule.meta.name+"\n", 0) / 593) + if entropy < 0: + raise ValueError("match probability cannot be negative") + + prevalences = [rutils.bold("rare"), rutils.bold("common"), "unknown"] + if entropy >= 0.05 or entropy == 0: + prevalence = prevalences[1] if entropy > 0.1 else prevalences[2] + common.append((rule.meta.namespace, rule.meta.name, matches, prevalence)) else: - rare.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold1("rare", 0))) + rare.append((rule.meta.namespace, rule.meta.name, matches, prevalences[0])) - # Step 3: Combine the four separate loops into a single list comprehension - rows = [(f"{rutils.bold1(c, 0)} {i}*", ns, p) for ns, c, i, p in sorted(rare, key=lambda x: (x[0], x[1]))] + \ - [(f"{rutils.bold1(c, 1)} {i}", ns, p) for ns, c, i, p in sorted(common, key=lambda x: (x[0], x[1]))] + rows = [] + def process_data(data): + if len(data)==0: + return + capabilities = "" + namespaces = "" + prevalences = "" + + for ns, c, i, p in sorted(data, key=lambda x: (x[0], x[1])): + capabilities += f"{rutils.bold(c)} {i}\n" + namespaces += ns + "\n" + prevalences += p + "\n" + + rows.append((capabilities.strip(), namespaces.strip(), prevalences.strip())) + + process_data(rare) + process_data(common) + # rows = [(f"{rutils.bold1(c, 0)} {i}*", ns, p) for ns, c, i, p in sorted(rare, key=lambda x: (x[0], x[1]))] + \ + # [(f"{rutils.bold1(c, 1)} {i}", ns, p) for ns, c, i, p in sorted(common, key=lambda x: (x[0], x[1]))] if rows: ostream.write( - tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)], tablefmt="mixed_outline") + tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)], tablefmt="mixed_grid") ) ostream.write("\n") else: diff --git a/capa/render/utils.py b/capa/render/utils.py index a858ba7d9..eab59adbe 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -19,12 +19,9 @@ def bold(s: str) -> str: return termcolor.colored(s, "cyan") -def bold1(s: str, entropy: float) -> str: - """draw attention to the given string, set color based on entropy""" - if entropy > 0.05: - return termcolor.colored(s, "cyan") - else: - return termcolor.colored(s, "blue") +def bold1(s: str) -> str: + """draw attention to the given string, set color to blue""" + return termcolor.colored(s, "blue") def bold2(s: str) -> str: diff --git a/entropy.xlsx b/entropy.xlsx deleted file mode 100644 index b598d7c4b2528c003232622062c84880b2df2c5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23367 zcmeFX^LJ%I*EJe-Y}=Z7Qyx( zES$+)t|4+9tyl(}Z~MR!*yASE3d|RoW7oN)qaZ(=beVHTF5h}nI?!n)%$rEn8J9QT zjV49av=Atd)%tbCpx!6{?m zVZ!Op`FRH>zuCB58H=rUDcJ8^S)Jd6@);6nRze&mqvnn^EZIGJ@{n^0MCq9W zPQYPCE(OwkfKK)G1rDP4|8ch_HD=5s9jmeDvXy&vt~yelF{!yF`Y>(`zLCbprj3DnQBtPy@>^MbWF&5T z(mRR1JXJ_CLAqFOZaa5A10Navc{08r0o} z>A&diX76ZiY;SMxZBf7P<#u=F7XX4yyY5k+%|@O+zNcSYTH_BuQJw~BC}L1R-&3w^y1{N{9A!cw z53K3%L&U;TlCLZ-YXGqD*^XrQU!_Pp+?BV}_6n0s@MAI}kY|q+@aD?6#}`J+CWYOo zIZb~v&Ch|KqW!3Aft}mj46rCkOZ<95v<2XK!5(^xO`DUF->~gNg-MhmYW2o4ekU8b zyUAqVGShIiDS^ZHEyTsB2;i-g>XXX>kSlA)^ z7DkSyCMwR3mUiY&|NOL34-kVVnc!_;_?4emiUY4*Z!&rfZ-UT1Y8UR>B9_R|w8SJ) zmqtFWS5;9iYw*s0NA>p8Q^AC?Qc5XMN*NCOj~v}qItrD7?&*Ud(UO+4c?A!{ZEg;q z@9Re5$i6hokGq2(KLqp*vfiJ`D>AuqhqdA}f!!Ieqk>!SPrh9>XOtp<&d-z43d=EH zK2LQjx`vpuQgKVi_grzM>OIIWEr#4hQ`6nwUEfQz%%}Ceo;HxCmmD$#!xIW7firc! zzKv5UWtM(fP3FBFOrG(0-?jf{wPd|})w(-;VEt_QxbW`CNvEa7oK;&l-1azP`epZ3 zHqOX4F_3685U zw-sutyiR(V0`1RR4>h*2kfzb2Sw1`CgRNRQ<2l}(MZWghxgn`N3nh#*Da6Kz3{si; zh5a>}ZAx^UcKq(#3aXvjiH@@~hkF$8+$=_xf)1WBJCln!QPW@JH}047W*A*`*QFSn zA>~N~M)M9=1>YIuH6stZdlWEV_S!A1Eo-}2ygEq084);2rR-j6Debf`xGZxxEVGJz zGma@IS4ESG8)$0?lj*8E{BM-8PU@?7CBCwDi^6w#+6klj8Iz&q+_(X_NHnkSMg>{* zyi@DzcNF8+_^Pp{P8si&!Cu%x+i6f27eitzO9E)Ti965Ee9BX%Kki1iP1$lf`PCl( zM4TlPWE+1ArM8B#qyMpI^-SBEuXlI9adJnNF-8J0#jk)G82uySA!8194&As?yV)8V z_TnK_vy3p``)aOYWR13Mh1!^L#jgf((H(!b`K*(4Lo;fAA$xZtd`aTP>YfzQPMR=cs-(G3l|!rpyLBJj+#M)9ihUDtL(>(@Tn zIikO{`^D@r9;%cxIM0v#R{dE1tTYAGeDei;0R60Z)H^8^O-EXX!;Flhfv=$whiNN; zI-_W1i8403_^!S8xJPSQSC+UOP?wY%a@sp~N=+9=X01tzjcTTtCd1c`#`$|Jlx>!F z4-uu+b*;K!tk)?r)ew{92x^bT5{AxJEwtfU}tuNEVZ7Cup!~E9OlR*Yoi^p*YROg=LttU_&>dbbE^v z_cAs9@X)Rb)l|`bPfu0vog|b6(cn6*(BV0MHnWdVvJh^b2jpXZffcn57vD^+CbB;0}ack&~duz>XwTH(i-Ddw{bBfHqRT0LauhS(3% zl5-p{){7g3a3^)V2GKkS#MPZ|haf8w59siNh?&Bw^-=8hk0LJsIMBj(jZhx^3es`> z7XK3LCdGXcO}}bBPxp3A2siTQ9L0n-&pBRL5Nhz#oz@B|_x^ehiP*(RB5rMIbAz9H z@lIu{;;)waI@6r?8N?=QA?Uhp7{Z*y`T2>FyMS8Ge9llH=F|OPrN;D^k2}#kSLGho z)@&A0mGQfiy9RVV%*xcpa=RVf`(2mg^Tv|<+1A(d?e$P?jy1xcjrdIGatQesK0#SLxYl0Y@5W}%Yv+d(d=yws<_ zc_Sp`*>F7zC&e$r7ZzOu_#y7!xFD!#b%W~lLdn~{T(}eieI56hi7-tq>kqy@c@0jH z{%nEHh=fR~4&759P47|hLw-P=f>>Z{+Ey>pul)Xjy`dXDFiso)Cj~S;{ej}6MQI4xK2B~uUfP{=X_faqklkQO;-y< zZrFv0|0L@}x-`<>wOahKwYNjWpZO)6ReS63G}1SvbB>NA$Lf)t%&%nP(kmUOPZIqk zE~nf1<$TL0@hbth$5rFNR*AkhCkCLELK*cKa#}K4s2iD@*9N)|ZasR!La4CX>FI>J z;HqH~CORw7T2n{VrSI=F&v9jfK-0_$Zp2yyo~i^ANxzgZvZ3ztv8Xj1>G;E=k0-Wa=crid&;Dnx`-ZBT1S-Qxaia9%<* zhjQ$+5c0W67?pqjngw{ml>!Vp=)H0O2*)^B-xt)FGPe%SiU6?{A2KB~rLd5dz2g&d z)lIGKQ?Vc~V*0J{^T3H&31aI6)aEp8cdPTX|@(J^FV?3t{b(&~Fi{)p8_%Uj2HS8g6Mds+u0u-N_Bu@iT0N(4_W zqVt#YKUI-r8QKQ@@3HCXIaiMGDR{HZpSU`PrNUpkGbYdyv5ZjmO>ZwVVSneXJ!Y9& zO2*_vI(FN9FM>;t(fYRry|Eeuk9(2KN3&I!Rh4XpN>`_T>BLm6!yM)Pblv3-Lhw`R z-TS(J0Jot01L)$<3?S$q+AXD$7x8Py* z_(8dm^vTqS3#~VpIzj7O{+_$GZJKwRy^onu&Jnn8fQxF8(O9(^t{ueYfJi0lBAp;; zm`4yzUX!u5ykz_IQw=y##mjG)f9@W^d>uN;h?i8dtVZzWYSv1<&R5azrMpjKX4E7g z580M#7ZD{zk4z_LG9ZI!(l4FFX6|Ea;`-Y{96ZvUd2&)Fg?n-68A(ITv^XaM)s}V= z&x0&XOv9)pe&p6j@&^qjpJo*X=}%{whQ1eUT69Jl$;xVC_Shrwi%bP1*7P>F0fb78 zii4B20dIhb3UbH9t0y|R^m{$xv(8{z6Wu_u339F!|I%(w4tn#ZHYv_FHCBfgHy;6A z>R7eo=kPD_u#s?h@xvT8= zfm3J!&>?wDZJB%`Vm@4X0;KXzKAq^&o{j8&wN-FJ?g=c3-PLzI(*}Y;X2$x~05kWY zKi$W#jvktvxxOmHy5Vo@ZGVgJ{g0HA$(FadNp6IUSY1ZG%~Aty4|~FoOto9_#K#Ez zSP3O5wJ;Q&dRR}(2qXeXo%5dDbva-dFh;Dq9!xQ8e~e0u)Nn@g#D~0rkdsPd-GYIZ zGtGA{9@4moIPb;4T5fWqf)8K()(6w!=>|EG7&&Dye1K|}6q{v8TJx_4yth8`F%9P^ z|CMDs10!rj4!3N2H5zT+w1Qr5tVTxDzdcbUP;4!>0t1XaqLG zYINeQBWXvH6H5$x3uv8Si)*MfW|TxK;B5eHcIPGW$#QuGX@W*7Oi2d($&M{6g=o%C zMJrgBhTs-nK%QE;7M7s}R6_7_8K{r0vm^CZXu&o^(Ez|tQ(G7e{T-Qg>m;^$YC+Ub zYw+7K?MbUD&=<2uTJuVjGmdXCc0op`0u8>E;KqiGy@cgiAFi~5+r$zqUs`I#2%(9W zrtbMWK4%kBhH}{0aJ5e-E=m^kd4~t>oTI=rg+0VWOe?m?9W9{XVti7ZzCeI(3h(%P z67C&MFCVR8!;}yEu>``u@<0|eG%ByKx?qeXV8~edJNvEjKvgpla)Ws|W7_4ryp7}~ zZX-w626X*NhS^yWjYgdZiil+1F?2}!C`Wd8asbCdKK1_m4)O>l%sD=)UR~GJ-TB;S&d0W#<%jC8@qi{UG8d zFhjEnorWYU<0Fj0@z}7?VQ;w@bf{EIrf@RJzs+ z9~CL!S4^9`^2?$0HPIU0oUx+RT>D|{aU&ME>M$JHb@T`&`Jtjh_@|u~$+vXJ(6-O$ z&7X4Ozg7hVbm52t&^%q0o@de=t#+@W>D_(|%mnSgC=k?2wSjMV{v4QkJG51w*gXe} zsfhCL79lAR>2;v8K#Bv(84tgwz|pH$HA%1t;Rt>!-W+^l(ABC(H?XS+QO~r z0iaVpQ>Qv%d(gZjZ-5k)WXU=SOHN&xMccEhn>+&f2l95r)n=VsAsyicoY?1LmCk|d z%=SzDRPV6A`Y#nm_#q;sKMVNZ9c%K4O;-JV#9)Z8c=y!m74igeh z1_fjts6_T0|5oWT`E^PIq<1e~@pDFQkl`0I`Ps@Mg`V1Z* zK8l#n$-97L*ZrU&Ekvb2(kuWIUoWpTI=66%c4O-qoykLw#UHVd;lLpO!$GfDr3q;e z@l8I@+ld8nWGJ;yQ(zspJ@z-@Sf!ygwee(kn{mP)_X5Nw=Bs^^u*F;OUM)ieqTXkT?r5q*&vP()P2-c-v?zk% zeY1v-x@s9Zi<^J=7)Ds0r(l{`>9U4@ysGEnR1t$x7tGv+QUIdyj~L9N(9OitiEl)H zy@t>gAp8ZNx2iTxb;P?3r4i=af3cU~nwN-4HQkvlAn7AC6+31Dc_BO2fR#TOll;u&}*-zDeO-1^#fjgv+zsB$8}b94iYuXc|n8K zm6A(iGj_2UX3=(^YR1Rr*wbC2xXL4YrsH^43;9%x6=LolgW+ud4qBS&P3~bUcOx+p z?skB&!@52>4(avWe%WZxJ)a$W)w=sSJfYI3)n)+gRd4nmStI^$!?uS*fM3cy?h5G*bc(`L3_MHuc+F4^(L=BW6uw2+|j>3rj_?> zlLYXWz%}z%1me?Pq`~C*EuEGjimeiE_ABdRQWXhXHsCj8%NF4f-pus~rAoShuzpIR zz!GdNq4S1bnd`3)4u``nyfMU_30v`H45GLfo5=y<6MJmkxtig){EPffVCp4e`E;HW zup~uFON7)??MWm+7i0PNILKCAqRhYvYZJ*f>DIo|2^YILhBsp0URxiLLc{T%t>Z;i zMmQ}-mS~KQVlHr$G26)GU8##lmPyp6guOoYL4#Z|#?(;J&Qr0qO4oQ2k~dgnhG^qN zo!U_$X#1;Wmq}FA$FX%P@z*o^sdYo{DbX44CQwvvkJG~-6&a*<;MRl7#{7#=GR*{OFg%>B{SRpRWM4=Wx4DIX1!ZdOgw}mX2^t2nPf5I-UgkzAK^p==}JdWj|XU?eemua`5)@b*$3)C2Xox z{nKJ#Z3h+N=UH67J4e0lAW_1h9I-}D6_V{ePP#K@j@rHl71s?tPvjB&0CWYCNg%}= zKO`F#k55_#j@5<4)qLo0B?iR2LpQNj#n8d6&yr1yw^1?;K}-#q>>s8~FwJsn?S@1j zlJH_{9Y%p;OYKw<+JCjO%l~51YfIeA81~!dpwmB&3odxR9efxnMaJq`2^+ zold+SmZ2r702tC(aH6McL#7Sh-3>ic5PfcxS64j2n!AN5m^^-Q$4`^c@+M$8@7+p5 zJ*pI^cKh)NNDcX^m}*=q-}JYA$o;~i!N3%gXK!?=!wH3^n5rLE?Ax|WN;H-{%2}O( z;y4c>L8W}&TKIO*sWSAIUl+ufq>M<%aA7|-xXtwTJ;8%y)@j}Diu&N|c-{6W*d2js zTW-Nqk0T<7U0#=FZl+4iH=bc*r;MJI-^U#PVKxqN=Zv#A)Ey^o=Y+T{?L#P+@wcF} zPp4q!u4leYanQ5_*x22HVZGSUIma!(u6LaK*3R96#vxT$iiPz7{A>0K8K`Os1G&3` z$oGpkz5L>jphQR9PxY1K?=&gq4H9*9wln6GJZO;=P?iE=y6GY@6M?j5Mv^F)F$svz znC2GdHaRi^mHl)?5NjeaZTyD1Z zH3d15@*1hdE2|(Xrl3047;*TK1Cn7~sxq-)WGWQi4F(Lm8`}^dtJ*oj4_O2E$L`g= z2ZKqL@d2 z(Z)#_#W?!RpdL@{C$xN_a_xi}Xz28izj-g*Ufze$rExxTfX|>PGkL&`H4M;+ z#WWXFrg#PA^(SYZ^&FR0f-38Y!p?gpF)HIGv*!ivy5?Bzk=in5CgCLO5*#=OPp+x1 zDRZp--idVxW-_z_Xsg#$hE~`d^}*fj$Vr~u`bZNZzjvNq+aMjE%0r_xO$i-3GZ-LC zHha6Gm9Fh=E(&OfR-dfXyda^{Q#GZP3IUK8N-FO0+J&`+a1p-48r2v^DvqPKv$L2A zh1haZbn5G%+=K~exAJQB!rwWhL>8r67!+kkN&cX=CZc@c?fH1fieiC9B0HCs@3ek( zHlWp5=FJ;eQwhSs6~EFvc4Q(RvKu7^{R@$F?I`o4%5S+0`~sL%+tQ(xX|r%`+^a`b z0~TBBMiGHe(|WC{S?W(sXvoDP1rZ$I=Om}%1aTL62`$zCX zcJ0Ra{&P4CMHTe@K4j~IlvA<|w^9$$E;}OM|LUji4@M>NeIDGLP;yomLyDxAUu!E= z;c$U}Xz0m3-_>*L^U>*bQ7uWx2}%mxUv)1;OtCVb z)OdS1e@fFbz@CsF3Wu)-y-Owd1@#vHd5BHl$vK4fz*4Q!A3%k`3zDmCsM*yOgZWpi$fUZO}4O}=YjS0mv;!;~8KHZrD7pjwbNv@2M`Y$+}^EV0lk z^Pvvz^!`4ukJ~4DdM=4dSG4K$&8bY2MY_qtEOyZ>TJb=_PO$06n7XG#HfB{MmOwx}$f}rlVpg_cqDiy73yeR0D`_0)GWW=7s;VI0yE^fEBQXsnGrm zd!6C%YIqi*VQ6ZxB>bY%ztqy>wEz6^ucG09rd+45nuAG!Ic~^rARq|;opN>lZDabM zTx*glAOgUQ?9Fsf6tvk91FpAb>}d(-FQNmR+1E|mYy;=aGRHOW_K+QqKkVO_&$Kl0 zJTZ|TsmhlfWhQwN=-9mf(jUTVX}wGhNPd6vdAbBQ2Mw|fa#c_9O7Is{v)@7orDVwU zr13X`#*^6uhtI92U+7{ZB;BkWbr3#|s6oJ?e$jXb&&-D!j!73K7gj4$8K?na9jxRbbI5>0T8LwwLR9`~!B%|0^CdNAoE>v+eC>coZiVZkr9riIt^h~Ji}0tc;`b{pD$c3IN zx4F(I4-tn(s<=sdGTn{nKRHUr<2aJg20*tkakmn_nJPbV1c6V)pfzO?4~ zlotkK7wlub@NPuDvn*k_@e3F(68$g9bW%dm8UP6bqJsCYhVwr~6V4W z?YLmJp_>8gaL%RF39SZy#dwsGsmC6dorA<%S_~B<_4WGN+9Pv$SPKlx7bPV>Z~#>5 z)p&#r2mn9BTX?#Ke8}PvCcqB=oN$+0+su->T3dR%(i_aAB}WndBao8WdJ!x`k(nHE zZ-hNQ&PT#BA>SJICqM&5bqI_IbR$e{sbbNLqL1xSF^Y`X)AM`2fUv0i9AGT$3A=Mh z<`gtH+@DfD3j>ljXNgT@$}|o175q47!i>1lf z)<|#bea_`1;Kq6K8oig=T>ETUN5j?e6|!gHd(HBhi)T|-4yfT4lzcV4c*eu0M0MgU zz2EuX_Y1owrRt7nu#X3dbe(L4(peePIx&P1fu+=IyhSr_&&giAxIxa8>xU`V7J&0L zq{+=Bh1Zu%rIyH#UAE`5I`5!(q{GoDO0ES?D5U;|FJ}m065WaS{NSN5g3eCuC zZ^l=U8KDr2==cmOlK}6qUB(!R0OLszdsN-DI?--j+_}!U`)~59fH&seI21UK+8p1n z(-)Qd)0f1((CHYWPv`gJ*_(bM1_tvT#+bm}zSJ$B$D?JULctuLyW1PA19z=~uh)L6 zzwhtAxsktHkjzFh<-R|jk4DxCe0+2aQ(W8nONu|oNr)Bb>_Zb=Lp!sEI8acLIdo(6 zBdeHyL-corImv+~*skU_Gi^n5h<4Kux}gTw&)sk{AqX-qv4A-ltK6< zdPl42&J`KxcxbFXKw#*lIn)vPbGWHX3rvj_PG2$><_BB9OKK5 zgqf0z%G5jzg}>39^u;c(Ejb62ltTz+xaf@q<5ZxPq8{651Fsx7Meli^L5Jx!yi-w! zH%hSrc-el5X+wnNq{CduQZFccL|eSO4W=oT4RJtjV(c}yAn2%Q$(-j<_g4P;>h0?Y zG`|t>*EHC4-(1{qb6NI0JG-pV6|ip7L)R?pYXAMmyY3{RFj@XK=sSm^lZ2hjJeBMU z{g5~I!r269{C;Q&dC+iUD{rfiVFM&gjX(h{eh`;GVaK5*slS{w6e@*o-|7^6>fDn1 z@q98r4#$0Z?-F?UA|VlA$Av$O0KxIQ**MI^3+vCu8knr))z{=C>aq`>2`lX~R;Nmw zS-Enj)&tr=q*S%Q-t~-`VB%xFo-ui@0_&w;9KBIdO&b@GS#5*vJEVH#qI{I)v5lMK z0u?w^rgg@Q;KZ&nUPaTtOEs%|s*6XvL0cNQ%13It1N08NIB%3Od*PT)I{D*c=ou-g zB4?EV*6d5TgtrcJkCeZvf`_R=Yh^c{jWByQ;%b8s-c%A-O$eG?_jDnLizZOp&!lZ}MhS8zXJ%-_19(P89EXc%s!i8}rC~G&sAoLZ z()4k+XzSFau<2Wy^!a0@vWN6`lE`e&L7fQx`BK_peRB}8FBA0vX~O7jfs}{ugDiMW zu;q(mjC1`35uHfT*jps6HD6Ml*sR^GW$>#s^N!!hhGA9f-CN=3(PaAg)aYO&o2!-4 z@RF-dad~e8l?r+Y<*M)y*O1pDt&jU|;oJ5xLF;ZV3xzy$uQ`Q({sVp*pWl89{^VTs z-~VOEs$=D%m?s4Rksb#D!Tzt1^{>v)$->ms*@@{tKmUnaQQ11N1l*26TL{mBY_A)I zHKePl{t}UTOqd*o;32is{ByHXn$ir;Pg7N2U4f|;Cd@dxf8Dc31sv)VqKwtD?PctJ zpO$M4YJ9r*{(AYmKF&Wp_D?=69su4-k#hfj+|EDvejMd~?tH$U9z5ItLWh}tY%vPz zeO(UKHk3{rT$1AKf^%p?s|9rIC*$@i?!eE1m+}PUY~FK=by$oqrAL4 zK3|_Bu3u6)nYMD7Vu{8=yM2B)Y^|?%1C3q)E^5y=FJFZ>_cwb-PpN_eU0>tN<1Ndy zw_Y!|&k?T%0^XmWjvtp7q0`voP|gotFVFK2S1m+eafE#jnQr#uZeLGFN*!nuSy?G#FGU(p_v6cVr#lqLeZ#PR z@$1p+VdwVdhRHzx>*J)wx83XWV4lu-4~fP1_+uW+i~I9xae2CJZ&|~D=u7+asb!i- zU+}}(8_x&Y=r0T)r>54YyAxP`=k)pfu=w`3vs1wPBllH9XTE~gQ>FKdBN;)wR`9(< zT{&haHnukQYEwY)!!=mCN2ANjcS5E8qqQ*m%IWrrTW3g7&hEDM%5Jm!1w-C_>vI?B zld1WRX;bL!9ZZ$`1jtB zJDZEI12>0GAqK)DIUY~uBx+aP$F@)EoEXo{7-Em;Uf2*sR{oFtMJ+pQoaM}=KL#f= z46Fw%^n-O87e@>FxTG@67+NXYF=q2%vpr4EChN8MAPWbV)CHfM>@usLCOL4S3QZH}r!$<+Y`WaqyPJO6Un&g?v`jC(nd2b@ z&FUTn9<)uh1>b6I9F8o>RW`IsrCVfnP?yR(-J?i2WuH#eO-CoqmmUh>!W5duTwY!P zj*p+(5*^o6C8>h3A=1VMds|f}zme#B1b&uAiJ+F%F=B+XorIAiojuh5B zsV)i=nKRDdm(XgQE{nYUjfU-FEM3&8DC$oCGa1`OgM0&z^wjKshHl``vsEUxvw{-6 z{nu1JnPx5tx(`lN7hQ?2nOh}V*^LxBCaM-`Iy9xzAtAFe*pp~PoV*4N$LrGhZptlI zR>jzp1%PPOY#o|=by(LV`9>}2>Nn;mWd*>Fj<9^A-pI!4?m>TUCe6JWY+*Wc2GZA( z$7F^RYw)IA@2O>QGh92b10{?T=RqUE$1!D8%5RT9`cB18yuq6aMvZ6^4P{v+ZP_CB zrI*Gf1l~<5QXIaz$Ll}xpH#3vfS_8aGDJHG9^9|aSvcv^4O(DcOvnZgwPic5v0ad_ zvjK)o&Z5|l>>|1{+hwOpRvB;F(ZC|7Hqbpe#WSNRwKvGwkIW*v(hCSo=BvzfF$vt>ERTOfAU7@tZSu)@*rfRFWIp=NdO8~QS*bJ;&dLZ@3>+&sK znRXXzg&5i!=tNbE^gM3!wpS%Oyryikr<8AuDOsK;%};X||JE9Kg$vwJx{XKmc7p1l z+4_w04&X|Q0GrY+KrU)iN?T54I|F#jJEAQwY+B@I5&ht@bxxXF=Bg*{(QM!5UWj8T zjuhH2vYmN(Uo2K3S2qJBp5u?0PRTnjXC6Y(JA<*l1^; zG$UmNv;J>m0RYf=A%B|RdD-)LN&rBf847GMB>BqLYY{O7&348uP#kZpi*nm$6oxWq zuy@eWSRH+=i*b8fx#DT@yGEH!UJRu$L{8~BZ*`QtofL;RZ%FZ@cICcNmsTs&m4&Ml zxgbrn&a}l|dJ=8K3iMfUd#uj2C0=?`ZNw_{SzxM7+g%+gj9K!!^JcaL)+(G54qo?8 zXFG54mRFWqs#BGB#J;s9JFcDL199~_S9Y>1e%q9JDoke9W(&-QecnFOZ@=nV;Lovx z`nw)Wc&I*ha<@V>^Z{MiHgyS^JLZ+fOH(R8uOQqc7iA{zqV&ehsjQHBXh9BanJT<@ z3cLw30QH9rRbm`}@@X=(E!uI7ToTX7nS$GGR`jYMbXFXDGGWOiC$BA>&;yNs1=j{` zq2^@3f13Y(#zok zc1gKjkXS=-nGb+Bt-*FaN?Eh>_(9_6T~?FOz|pekI{yY6pJ@A*_~1?Lc3FK+*CKSaXd_hLRI*p&0brGY&)lCvCwS%RP0AhrFJ-O}bxNb86?|EVNfmqsDIr z6=W5?+l*l{T%!1MUABASv}l;{$#jIhwoSck4-E3&q=CcrB`%l;z$m3qcmk(}Gp~GR z%aBYN6?>)-g-rLni=B0&7-#;vfxoQHfGjcYG*5KepKhdjV{h_|v3%8TU12#+0n1{D1|msnWDRjcNrDYNvVGzGl$ zNfJz*pY-ia_-n-L5=n!H%y1?$ZU?kzW@J;wXG@>6Ay&uGwcEx z7TSb+F!5$F!?J8T;#6%%Oa^V1!dj1MwhGr>DQM0xJn@9FG7A~w%1iYDq9Kp4wJ7d& zE`Q+(A8NWNXCk9l;KXXWCaqoPu*WdW$leRKe#JUwuAaj|=^1g5GIN9)6058MK12kM zfiE0f=OTzr@Sxt{a`Y;!(HbokHJOh+lH}1LE4PF}KLjigFL2nzjhX$h;69G1j`oYH z_B`Hror!~YdCORhOb3LG9_NpUHn&Zl^C!`*OK1))O}^#>*3CX}oPz;j6D#d+38F1} z(iL!Ts#!quys=F)cE?2J{vL1 z?Z4v=?Pu8x;5hS#q+9IEQ|ppgpCQW-6?G_;{ucBtv$$3aCiFkmfu-x2d+c%LZyin9 z%XXBkCG8_Rc=+{oJ5DdVTkQfPu$3qcb=(%<>7;a(7z$)|Kb^KAzl%3_wBnBeo3204 zrUGqQ*gOgWc|d&FcR8Cc{b%xZswp`mCamPlqiaYB27q(@UNZ3vUJTn^!5{LMv%P97 z31x(?L>aB)P6AF`slWwxxDs|K1c^1(koMFc%S)Z#1xP)X{!#URq_QP0mptb)hGP*?FO1LWe_GvVv|OvNH>5HW;HMqK|U zBewUMiACBVVhNX|&{4@hN5#cYvXG4|cnsq9M~am+sNWvZV9ox)zIERu8K}BQM@**= zNw_kREIhcOw^qo%yu?9FAA97qe5TW|W!HYX_MFMKw{TCCYPlhHY+tn8_#qN_=_Lw) z!4=+j27sfy^#OIiDZ30sOrgc2`c2+qCOgxX$_gTWzS``JFbkpGspUc}8aQ;)whzN| zNVMb(^u;cHRb8hg{JrhqdnVA>1|pu$?L*20Yei~Qr)`76mafH!tXZzLBLH_b99Nhv z1yF{PIVr537F`zK#fwI`$jC(K1{$5@wG5&28fw!a&+3G90OS)}y<3ZXsNF-xZ__vo zbutT+E1xZH6M$_-^@%W$Px_^GZc)8JnlCXV{HI-B$7=zRZVY4nMogC_F)?zPhGG&? z#RxP_i|_Kx^KH!6mao?Hk2>eJkDMSO``x+zyn+)5AdZXYKjPIhZwiSjyM#41p)!V; zJ>Hgx()1})x3LzBI^Q7hwPqZW#X4t(1bP>#gsWr}v5L!dKlVQ#z*;3>0&ZJ!#~+Uk z$Q#m*J@Gi2_6@Yce0?K!pCbv4rP)m@_O_v*|J#zLJ5PDc^Sj5hP9K=jn>4qeG1j~N2titPg(+X7bC1t;$uR>$3c9_#jb}ia$_``o-*pca*Trs92Z8X=d`op&y|0FXsxq6NUYM5w+N;`o5VNflxXcA+ z@*rfqpk!fL=dU61*QC5Rr2mB7i1%(C5EVQzg%A{ZzEP`W8_ljOrp2W~qN25L8_un5 zGJJH4npodwDHeGig9+rb43s&oi8kY{UR!fQ3}d6L)S_-(H41$_uO>vgc1;3?0GYap zTNU&_ZR;ak3b@9pBklMp9#`Ieh`xkTHKI@jU2ykH#|@9pSOLdmuvwyPOIIDaJu>QC*>3IR=k zT=~+JXkgVZ+<4LsGT?(I)fQn(QE2LUl_An;t&xcT4GlwI{b#wT_EZIiv3inC7&g_< zPg$*XG2zq}fW8`zE)2||kJd=X>&1~$*q2I5M(p2sacC_61VdjvStJb8ZJM;Of|?Q= zIk!b+4gOyO*$fNN+tv5H5ff-w$Mx_ab4f033%tC+S4hhK~!h-Qx8Fd`jEvP!P% z5p(SmL(b;6S>%u#hzY2~NE%T{f+nTXu`QWm2SZ(jK^5LNsho6u>(19TVl5DRyv+GJ z?prg%flO!k79i#d6;9lc!)nCpfiQAOjAF7>-)A-ub{>QBPm#XFw`O#ASdGP8hr@_f z#+z30or7ngQCA7b1LUfobjbs&m~mNvyCxP_UT+l*BV|G*1u?6ouJaQC^(5!cp(_A& zH5~YO=gP%}DKhH#Y-*s=Gy}0j`UgBkJ+BNz`kbDNF>>5U&YkO=OPmxT(=@1RQ=cV7 z_&Fk~uo(l;x8p`G^)Uh_wAHbriQ_kfNk>$*+R%e$ff`Cc5irrNYs6L{<|+xq%H)DJ zKqD7V!um`m#3mJyo!NHy z)b&JpBfqixi(m>-sOotYAkrWH{iUqum4ir6o(;$ollim(E4VW}(;nG7=uK;Ng$F~U zc~Nk4)+RK_8V~Fdlj%+*h)x-9s5-Em5-Aa;l+EAg)#7a-)YHUeFu^vV1=w3W(=D$b z6voAA#g*ImhKBrgtY0*DMHfXo6o3l~1BRhVb90}#YydqnNx<>@^DN*Y`6qIa)To=2 z8QlGh-Yfi72gam0@G-eT`_sw6)(Sfu&?4mfohPn;OIjT9Nai+%klOD`WBid(#D03> zks4p%m>eMy+*~m!wV9O_u3NF`{AY{1G{~LZNJ4GcGql^&A=0@6pyRipfy^!qmJ5(G z9ktj{>s7L4&v@~$2w=tqCG7r zJuof?*849ucZl-diP{6lAi^Lb3YXk3rEC(@c}dtv{jvdS&=Mo5rg|QU{|vI&QE#)# zPf=0TCNKn;pb#L+XX6sWA=?J%=hLvjgSlkWqL~!{#ZZcv86v*2d2Ir7b4F+9`>sg9 zo=99do$x8}^nitmj|G&~rhedw7Jy?!(rZLALXTCGCjzEK5~l04%rSmFe6B01DD_Jb z`g`%(l&>gO&@?nG3?i&>$!Sh#Z#@_+3`8(7kKm@dU7*MWbzoJi6*)CI7c~&EEEsBX zWc2^5oojz&djJ2!WRuGb#Yr|c4w3u4vQ3K>iaKbaj@T55lbX_Htfke#MkLV%QMr`J zC7N@Ta=%rSIc}kwbU}r~d|&TPr*h8sKlt#2*Zci?KcBDH>v?%RpRc#q`^l>F@II=W z2e^~8s!3XP`FX1LSza8t%n9JlW`5=@2i9>1o_UpWfMe+KhfXeSUc*3-zqkw*D(^`K*WuC`I@p$7_feMKF!MsX)2{s%gYIlOV8995i z;3zZ#0dVK((>O^4(zfqF^;Euu7F}h#EV!I_fg;}!yfH7iz8H&d`+Y6gnO0ota+m7> zDIp52fnvE(Wf>~R^9$Aa*7))-G1%hI#|zaZw5lbv>Qt!ST0R9Z2T2Jn&cj|SEV<$I zXsmdG8%kcB$=G)5!aR=z52MDvYTq_dR#U=OLk6~_*R23(Q4?>Y2~RzsT;qZJh^_vJ z1uTb)nvS}p5i3&%N%yLcryB(9Vuh&>V5^q%1m13I8N#hOI3@kOX*>zJ1dmz_fr6H|Z%zv`b?V$olbs}% zKZ!@RCY8r-<@IkxfrQB}F7So_wv}V0>+F``EtZfkD4UI&ChMfwimp5jd>joaVTK5^ zkzUypC0VqFEUPIp8u_)k+4C|s+6_e>tO(KLbg@UVZaktM2tAdIR4hp{0SkGZtV*TL^cpa4L8k6_8yw!;xqXoCJ0uEtYDnS4~Q?7NhYfB!KHqXUE{_HaLh% z{**8*@R4o$274T_qINI=k*0(|Yam!n8?0neSB6^6ahVxqhmPDDAy-Ey9r<~cz>phA z^=`GGxrbqO`sxKwo=BgqP3dT{E~bPoCJmi;^VNN~KC_>=p(?M_{@5?xzVv4G^Co>w z+I$1x(u_m(YUb}vyP7~ZV?R}M$7ja7B8mn=+iP?z(NtFM%1~v{Xwtq3#_J+I6COyv zN3|XSDbp8oue!Kl6ZBAIf90Q~Gd&$**3h>&qA2@O2s2tkK*{*BVhRdy3F&TXnQ4O$ zn6ZeN$B8_u*Q&loZ4#E#k}F8EMAUSSyA_0~^L6ozKR#fJf>UVE{OxG7I{pwF=M*FY zkA}1AEd54{5%uY3KGDzT%EAk z1Qg^l!>ax&2oPO5>{1588^JyXA#ti_Q(Ln_FdR>R0%lnUby>n|I?cODVj}<0rw9r} z2kv?D&CKVw+m^r!aSJoV-h6!yFQ)n&z zbeZRb%s~Q=`q}dX#`NWAdjBMM^~F|X5%M05%_N^>g2biclzTrW;e#UiMp*~#Gcl?a zDLKcn?X#@*HMN?_s#MnmXZARAocK8+y+<1OgcfE0q>R}}Hg=EWq!ZM1rbaW#_Xw!C z@Xim4mDl@wWoK|`VKkKXbkRTt{PnqPWP>70$RUH-b$~9GX0ECgZW_m!Tkc+2>irnw z-iFZ$Jr*|Mq`AXa4>65H&qmi-X6&VRre}cZl9^;DFx}i_Xwxcc7jOS|JD|Y!FF1)w z&IHrohfJJIQdLwxCFH~liS#?@nh~OQI|gLO!6`^F+xi=!O=iQU-I3yFax6#}s=;+~ zb}c4e$O)dt$xdetHQ$XF)(skr(;Fa=x{!attNS|Lm09C*$c2{hJ|N_!eL!AgKq%Nz ziw4TEsP;ruy0l>uUIArD22Y#}aRm9kdON1Q9aCp`b-w3@;FC_)WdbxIu*FF<-J)5X zOq_7qKLKQ_uWXcM9SyCs_+3G{>DTsIU?I~uyJ;yg42$?dW-&i*-HLHp(Xw{={5n7N z(O(8%aaUK_nTk6=xc1de!)h{7GfKlFpRd6t^aHgEJ`E~LFv%;J^)Fa;9rZQZM@3fH1PIMqiK zm8y2iui+ZgUVfqJuhqJczw+<3j5{ehAJGt8)$$FwDM1CRZr@A1OaN~>mxlmw^gS0! zKLw<=x8(_17o|Xcz8e;zOW(^*ghi%tz9JL{-_V%`+ zN?n^2%jx@7@k`Ggl%|NE8#N;_ucW`vJzxYB`02SOoNMJFow*J^NF|+(9NVeLp8r zG^{ZU+HN&IN&06HU)_JrW>0wP@`r6cztvMFo%R6=I$z~QD*?@iL2pxloFsYVNoe{Y z>0M2?!p-1pXQV*5G<#v1ysnqA|0MHD4D%W5g}(BnllF`hNR};y32>qNm}kL#!r0`S zL;xqJAUFY98?SF;PmOa${JcZxJFM$YJ3HPZB;E&*B+&AQOdrevV92`8cOKBoDa0H_ z0RW9}!Lz3M=1PzTh@XKlZ=SXRK#Y($a~G63LJ9;Ld^8)l7u3wD z%XR&Ei{|5?-NxY)PTL-1)dPX_*#!8kJHWR}V*tX?>Oj&m{M+9ifwS+pN&4K^71{izcFa_GUN1ls zV7j|}ka0q{#fAX6f=-FW1%)f|_`j@r=2wV;i{)EUSo%%A^ykAOyVwYJEn{nyYvR-g zF!cjKA}?L`(AiAu#ms$AgqOhwF4~^8!mc(0<*SCid9xd@mTxKTKW|o@GGpni@9hlirmVS5>5sd1I;@sDZU1FB zLa*vSYpsPIi$F(#9w9J2)c2<67Z&zE%Y$rA{F9zY3LZ2;op^s@R?+c`_9`5{RKja};Kq54U#bpY z=Df7@#6c>-XKsmlA~O4k4J3foWGXpe5y2tUyW>xPF|72 z*XqqFi1ipYG|W8Oy%uZRIKBg|0MB5LRZPjLtbKkr`+dyWcE*u%DrVWD&NrpI?-s{Q zo}I4mQZe;R>cJQu)TzzfS)ux@Z0OasaGhH3lA?$Am&=Sa9E9t%%6~%SEE7B`XKyR9 z2(8}iqFxc9+3gZ=b5X{9eR6Q3d9+$yv3y$9XZg*KnEnDkdEzzvvKab~c>bVZ_Q~Vk zKObb@TC7IgT2A&^H5S#{@d!W7I%uc+PJL0YL}159`M~(8X@l|ga{+U|`JFUwwlk8e z-pxpHEy;CdCDtB&1ctBkcdlA|C(ren!=bSmA-s1ywLT}jr3gg;ia{GQMo9KZkn z>sv15v#`qHH%?qom3cwU!-d2PtMz>&hRceG|5dwhA%}$@pM2vG0iqNbKK!3gQ5KRe z{M6tZX^p;!bm4~w3mGgt`1y@Ngvmck|JO0lg0$6!#D|GdL-xstL diff --git a/rules_entropy.txt b/rules_entropy.txt deleted file mode 100644 index 602e39fbf..000000000 --- a/rules_entropy.txt +++ /dev/null @@ -1,21 +0,0 @@ -contain obfuscated stackstrings -resolve DNS -initialize Winsock library -act as TCP client -encode data using XOR -create process on Windows -terminate process -terminate process -link many functions at runtime -copy file -enumerate files recursively -read file via mapping -read file via mapping -enumerate PE sections -resolve function by parsing PE exports -copy file -enumerate files recursively -read file via mapping -read file via mapping -enumerate PE sections -resolve function by parsing PE exports From 0b5a32622bffed6337ac50c7539dd71b30840105 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 19 Aug 2023 15:01:27 +0530 Subject: [PATCH 05/32] Update default.py --- capa/render/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/render/default.py b/capa/render/default.py index 102522280..01c6c70cf 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -88,7 +88,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): rows = [] # Step 1: Load entropy_dict from the pickle file - with open("./rules_prevalence.pickle", 'rb') as pickle_file: + with open("./assets/rules_prevalence.pickle", 'rb') as pickle_file: entropy_dict = pickle.load(pickle_file) # Step 2: Combine the loop and categorization From 039fdbde33012724621549b8f543d2c1e22f7ee4 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 19 Aug 2023 15:05:24 +0530 Subject: [PATCH 06/32] Update utils.py Update default.py Update CHANGELOG.md --- CHANGELOG.md | 1 + capa/render/default.py | 45 +++++++++++++++++++++++------------------- capa/render/utils.py | 5 ----- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4f40db00..f59872c79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - ELF: implement file import and export name extractor #1607 #1608 @Aayush-Goel-04 - bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04 - develop script to highlight the features that are not used during matching #331 @Aayush-Goel-04 +- Show prevalence of rules in the output #520 @Aayush-Goel-04 ### Breaking Changes diff --git a/capa/render/default.py b/capa/render/default.py index 01c6c70cf..259b7ff7a 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -84,14 +84,14 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): +-------------------------------------------------------+-------------------------------------------------+ """ - subrule_matches = find_subrule_matches(doc) - rows = [] + def load_rules_prevalence(filename: str) -> dict: + with open(filename, "rb") as pickle_file: + return pickle.load(pickle_file) - # Step 1: Load entropy_dict from the pickle file - with open("./assets/rules_prevalence.pickle", 'rb') as pickle_file: - entropy_dict = pickle.load(pickle_file) + subrule_matches = find_subrule_matches(doc) + rules_prevalence = load_rules_prevalence("./assets/rules_prevalence.pickle") - # Step 2: Combine the loop and categorization + # seperate rules based on their prevalence common = [] rare = [] for rule in rutils.capability_rules(doc): @@ -100,41 +100,46 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): count = len(rule.matches) matches = f"({count} matches)" if count > 1 else "" - - entropy = float(entropy_dict.get(rule.meta.name+"\n", 0) / 593) - if entropy < 0: - raise ValueError("match probability cannot be negative") - + + rule_prevalence = float(rules_prevalence.get(rule.meta.name + "\n", 0)) + if rule_prevalence < 0: + raise ValueError("Match probability cannot be negative") + prevalences = [rutils.bold("rare"), rutils.bold("common"), "unknown"] - if entropy >= 0.05 or entropy == 0: - prevalence = prevalences[1] if entropy > 0.1 else prevalences[2] + + if rule_prevalence >= 0.05 or rule_prevalence == 0: + prevalence = prevalences[2] if rule_prevalence == 0 else prevalences[1] common.append((rule.meta.namespace, rule.meta.name, matches, prevalence)) else: rare.append((rule.meta.namespace, rule.meta.name, matches, prevalences[0])) rows = [] + def process_data(data): - if len(data)==0: + "joins combine similar rules into a single section" + if len(data) == 0: return capabilities = "" namespaces = "" prevalences = "" - + for ns, c, i, p in sorted(data, key=lambda x: (x[0], x[1])): capabilities += f"{rutils.bold(c)} {i}\n" namespaces += ns + "\n" prevalences += p + "\n" - + rows.append((capabilities.strip(), namespaces.strip(), prevalences.strip())) - + process_data(rare) process_data(common) - # rows = [(f"{rutils.bold1(c, 0)} {i}*", ns, p) for ns, c, i, p in sorted(rare, key=lambda x: (x[0], x[1]))] + \ - # [(f"{rutils.bold1(c, 1)} {i}", ns, p) for ns, c, i, p in sorted(common, key=lambda x: (x[0], x[1]))] if rows: ostream.write( - tabulate.tabulate(rows, headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)], tablefmt="mixed_grid") + tabulate.tabulate( + rows, + headers=[width("Capability", 50), width("Namespace", 50), width("Prevalence", 10)], + tablefmt="mixed_grid", + ) ) ostream.write("\n") else: diff --git a/capa/render/utils.py b/capa/render/utils.py index eab59adbe..fb3932340 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -19,11 +19,6 @@ def bold(s: str) -> str: return termcolor.colored(s, "cyan") -def bold1(s: str) -> str: - """draw attention to the given string, set color to blue""" - return termcolor.colored(s, "blue") - - def bold2(s: str) -> str: """draw attention to the given string, within a `bold` section""" return termcolor.colored(s, "green") From f6058b13d3c1e778c0193ded67d35cb330504669 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 19 Aug 2023 15:40:01 +0530 Subject: [PATCH 07/32] Update default.py Delete try.py, rules_prevalence.pickle --- assets/rules_prevalence.pickle | Bin 19375 -> 0 bytes capa/render/default.py | 19 +++++++----- try.py | 54 --------------------------------- 3 files changed, 12 insertions(+), 61 deletions(-) delete mode 100644 assets/rules_prevalence.pickle delete mode 100644 try.py diff --git a/assets/rules_prevalence.pickle b/assets/rules_prevalence.pickle deleted file mode 100644 index 7a29e925d6a795bf802fc4ed5091237d601a186e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19375 zcmb7MNpBoUc6JpPsi}pMxJe=`_!s!(t6{+3dl8XaB{kzl1I^A@GBV;V-}~N+ z`pfaZ|LvPUkbnOCHxKA~;B=h88MGYB9z^cShg~a-0(a02r+)KhfvlE4h-`OY`OQuo zw(Q7hqup*DB%O9wCh1Psj(SdDbzIEmA6vt~@3@|0xr2@$^zF#?2bgV)raixF9XKZ# z>>oJ%hFMz7KeG;9+Zy_whb5rBMl&sY7{!5Og{{DG24T;SWMxyKGiY0FJF@Y=|4J)? z({kKH{))DB6u6NiZB{4n`&Q@#hge?ZTP@FZ20XzMoe!MQ_b_7PZ37SfhHj->@p^y1 zZoS8_m9;MV>n%aApV1w8uXG5UU&l@uSx0WvvtIl7tLNBlCqTbTbRNsac3DGr=-|P1 zTGS7sBOid7TI<&_&btNyagKRQU(+?r!%b@I)kGWhwfZ(%muRV^ zb+7i!s_(s8Lx&bE^z5*w#ac9J+x&;OEw%9k_PW;VMn`U$R__AZsZ)^_>+;h6XoqN!{f!*i6 zKhkY}62ihb{p{3M_QT%%Db7Xq`(49=VF;Cd`eS;bPYV_(@bM*~QtiIqb~`RC-}cw< ztR95Kb3$cj!dBdy!tU?zdHCVr8p1t{smZ5coQg2YN;k4y_u@gZ$1mxMc8H->-&og*{^kXJo%)V~8?_=B6DQ(@ zeQ<*)w!L-#7{_ywE*O`ko!H;f6&U&;vc$P4pM{Q9nqw0MVSM~{b3aitw2aYpY?C{T zJ(dqF4F1fqnsEmn9nIg+jOW9p$S^w$oqkhH#yCyK0}Qmg5NRlBOH5@!(xq+k_G- zbFVzb8G_~}PL8cOcPq?w4xqysD7zRyCE?|v2u+-{>8U1Lpm~;Vm`s2iHY}+feo9j< ze+VfPEBvNIb3*_h^l_ST*Xm=(u|wUD+&gSH*8-!dRz zi7v4!IIIW>Ao(x46~gAVV@OD3!|w8aZtvr-hJMTuu8$=NKLf0{UUS(F$C++HdVc8)%OIgs^<8p8{m|oxXCBRl2H?%I0y) z@vKifJC@yUuegjOtF)jSKl=FakJ&b8dx?l)Uf{r{0)t)bR14Fft8xO6Z;LGp4&JjB zzsq!e5V8_nK7$|@17&@v*ET=xqw6HyNVSC3=)>+u8|ZcW9Z+q+ zlYK!~d6PuR*_g0cdd{HBkO2oUrOxjC-VXkEmKM|+l}Qj72t{uzIa$E)0bm7dY~E&E zrCBXk@>~N|dHDjrT;s~T?_JJxa)+CV5N5N_8eWY7d8+>N zG_AdB`s^lMRGuMyw4xwXgg8p{hAm&u9m*?j@KLeQg$xzl=)`i{kTS+F0e|CJn$@eR zZ?0QjJNQ9+pkHw$VBiEcALGUSV$9#sU8Bcg&+$Bo7x-`ifyAa7PqIq4^xIAh4)g3m zHx|>#4p^dr1-jew{R1oTW4K1ZDW0nDwgNx&J8Xg@U`bx<7(F@7&MPgntzCST14Xd* zkKpCS_1&OZ!L$hXhE1Q%qyEBqe7eU62z#{m@l$GX?$gus(-REFurYv9LhrT{9z^~S zd!n!nqfV@EoUYj_5Ym1lcfhO4W+y-bgN+9caz=NNOq%2*HY^uA^{ZB z@jVrEj%(EI_$ly~Z&mpA59y||Eiesj7iz-$k~aZ&m0yE5<(MDV4PGXIRTL?Jc=3sh z7h_N{F{%5RrpvPT@;LE7U$A#|L@Em==_1V=Af#KuA)pMKxS4E7n1pw$&tIVJ4$XQD z>hxM!Gn8~=kJ#^?60JR=`5xqRBs9T{KSwNRWTVV$qs#LwNdN@pod8(E{Q4eLIH66J z=}UuvN^s8c%NM|>F#8OVc|&LD%MzbuV9`kTwbDSk>G?+;H)J1jrnEy15fWGxW=qb| zLdgp^HXCTWNjFQ{YB6?q_4)HBzd)Dsbg!gK!}daZ^&9r+wVsWCWhILxJwbrzm5$TZ zqCe=z0seg|E6MNy$>BeJ`3!C2rGoMN2|KG9oN&S;sBw|o6|+zU0TRP0!xoAKc)hz; z!-%ix;b}I8&C*`vy>bS70y5G2EQjxm5Sy{v1u$EI z!)gInz(yOhjL|X{i)WFwT{r$qvt%N`=rfKeM~!~J0fZ?W6dr#`I8Eanc;#Wvh9L@C zM8KDX&Kv%4=y=_K|Lvfwz6_u)+YW2s*)50Z6ig18WZ%k|3Q7py4;u_C)@KDdLl3cL zgLFRbZc9uVVrPK1aX=rNPJf4Oy3ZK)A}@1HD9kli?D|LGP_dRO-Oa$iG_KY9124wmuk3&bSjzZ7*3vc!tC!pa9ejEI~#0@IFRpnHpAg}^eA2B=?AHz z67-*@tC?VX8$fqG0Q=_zY)Z#D$kw4s+izi+OnpM*3jGj~m8cv%80NT>r487rVRN8Q zN|zRCiC<7xx^8WWQAv~vqb~p>`NnC6*FIuBCh4BmUS`>Kq{W*S z*hkjd)`q2`56n=ZZ^|*XiMyGcX(HK@NF@fHPmD9qml)>>*!4ug8ABg&0MNwd9SnU$ zKm0A1fktx)7&%LL9>}mjJW#3r0Xj~>edw&qk<)D1pi=w5?`S~?#K8rJ^EPPE;OVNe zXCU!poV2D~p694{of`%4u*w!wH1-8;%ER+uEtU!iYtB07NJ8rBQX+ z+5;yTI3D)MSQS2nZKYJ=`ZZDQH-F?XK99xE(q$b8W){c(o0)x^=YMWg|64^BqKPWiWV&5ukDNZHHN@km*%MOkG9KTvX25 zcp-p%L?N(0{CP`sgViG6=EL|UJuF(x_7Kq$-PTO;1)5VphfOMbP2KKHn?#TB# zs%?y}xP7L2tf6uaid!B>Xx$$o6lYEd@yBJAidVmT*ld=&;)Y*VuO@oMy>6N2}mY zT|ijQpq+@bkr-85dZm0`+yi6cK@yR5Wur3*W+IhW6}f5~rODu(jkP*Djwz|t0b^d- zs&pUJ0a)BOj#Fp$hbbWV?slljK0M#Dl?zAK!X1dhCgL_j)65{^82iu!#V22;`^J71 zqQG7ExW-N&c@4ajIUD3)B$x#Z1e$NPn0MqYsnSj5V-&Tlw2^fSI~TbE3(nFFt@SKf z=5&DinO5`}5?r8#EUCM_-2mWZ+$b;35x(0EnCdh-l@6D2>To8^$Emamo{o>GSKasQ z02=T$b9dJMPD92bM9(qSpS~B0YLc!?5U!(a$oKOkw&)@bfUn#6V> zZZiUwk3r7Tg5>7`e(RRg>iN($7!#pt&d^JsyRSy(bbHeBH`vbk>pQ+ z#Q9KQCnTjHX9g?0;|^j@?0|nr<&kAU!lblyMgWZ%@XV+SG@FS^G7a-dY7q$>q*Hyk z^-MF;u%LJjpjjo2=jeJoIK3HD^M_;*kRSLE&oIM106RVZwQ0Ur;kp3`QD#Lu%r>FS@E~eq(FIrW4J{R-*tOws8}P~E14CKpuqV%}bi43eGvg*dswmJ3-9WU69ISVu=o`sCW+RfZEW;;qjJSsty#+cu zVfX-pOV0Pxr~OT47^*a_N&nQP&m+rj!;~qslZKuZdaR45T0N910TaM^wgKEF2xAM5 zzIk6Yl_i0h4aw0HzoJ{GTDkPKXX&Qkzl>*6;3(}fNtY#k{AqV5BZy;kIkj#Q$tXXC ze4XT|c|9^|IuU4TM#UUmb772@wSIWml6;udsQ`+KMO2Eiwa@-~9I3;^v8MvZ%KJ9@ zj!SYHqWUZ!X_eU*bkk&tbecy@K3ldf=R(x^Xr3vJ#U>f_M~VMTAh)8%-n?&Y?)`w$ zqP@+HA2!zZ`IM&>&&;Vfl(T%#1mhf z262Uyqa2g9TI&DB?LV)*jp< zmmBO#DL{a=3KWw*$oENwk|HNy{%kJaLAqRNHs~UxXxJbi8_EYFOa_>WwuGH51PtMg z8TdNi8Tk}c3##z^)G^ttUTAk5v3zWI3@ZrtB;Utju)Lnd4vU|pC4M*5Rfi8ap^0VOF%ptS ztX>*><3u>=3jV-U2aVDB(D6DzdMtL>M+mhEsU<7a^xLeuTu9|Pb{d8xK$p$X6q_tj ztt@7oplcd!rudjkAeuHu!V`2gLm<{RR`$Txise9Km1c(i5tb=gUDWrviesLFH9vxS zE8-R_ zF&U6MdzaBlfcG_);46xJtJU8ZeAZ-=kkBzqVh|_jMr!3z>V_<}$2+8qxB7rt@m1Od zFc+bagk&OBU@4x;#a?8&)(N^}Xe^YQ=;C29SIiC~oZ%ozc4#4oS(Ot~z8pbgiKqWu z+g^>+lhrI+8$uA8sEfvbkgL(r7k}DWl-HVEg84io6j2nw)eTmR0>OYi3v0w=Vz^NQ z9YiRpkx$uf^pM#B+-2gfO4lX)Vdhc1Fx%#jTXmgv=Q>lac+!X;30|9X6c|{cyQ<0$ z?=!9XAc|Am0(w#jbcuY-vUAyVsA&fq4p)!)Ob-P(jXf#Gjtg+xB4%+FR)teJCf zubAvY*+;n`eix%~XXkr#8KcEf>K0E}_j&UX$CMnc#&8hJ{9D)k0Cfy)P5favL|UZo zZEeq5+uzO5N)xnNGBIbM#67o=RY$oRWKg~tA-m#7tF)+%tRWv56R|EBBfLENJ*tE& zVU8;FWk#EuKqgGp0=POLaRq%rMKnnuyU6v<#hS8j8}Hs`J_j&fgS%`&*#e@#+_AKd zf7rsAOU-VDo|&qM)M11QOX;^;wbYYn1cC)k&_iQDCFBC+P-WS?^+uWD4Z%R8_gPvl z*=WtvSsRk>tA4f*JQU`E+N3tfK<^$BdW!CZG#7S>=|00hSBmiKt;@6&XZmvcv+E9C zeaup5=rl&(7_cgVjvM`evj)w_;TP99f%rtc?=ZqmQl-U;R*sO&z(T3+I6cx-RT98z z_$|)WfYyv=Uf0pc)AmXy8fFz#zUi87w!N~}CEbcB@&HEbcw~6e!LxsI%4RS(Nt!Xx zK?KX;cS_d5w3xba((=I5xvjFP7EKp0CZp|8Hq#CbDQmxB+D}kh)s6ux@?luI2o)@p ztP^E%dXglbSn-Fr^^ue!3II+>a>wA_jH#I00abc;H(%yElzeT(-RiHbx^W0 zh_va|`P|2IQ1%m;60Y;gMp-BnHREvtz0Pq_RK}17(3gUsJKd5cAU|w+AUy1iuA-9Z z!0dwaw6F-&Oqm&?;fl5tFqiqILJ>2)u)`h%qSy&mBGc*6KM`bGoGAIEXb1*ylzl&U9@$f*J z>D>U<9*(xBrw^=9F0N*)UqvpmeB4m25 zWC{dR`a3yedR*G$e$oR0lGKf^ASYV((YQWf`bNN2rL!6P7WpW1IaOrSnPC1)KR#8S z*kcqoqJK#%z!G+?Qyh+YXu-#;Jj4?bDPsU zsMLiGMGq+`6O@%L64RI zB~<+75hy?}&nu{1lqz34jEN?|P$~8(i!@{wzth11a&l8&LrU|k75W-9bpAHY?-~PW zq-dOEJ_DFP^P(|&%!m5_8G?|VhvchOC+UHT%eJ|0T=*T8R7LSNa<~!_xdTq9V^1dO zE5mP^q$tOz>j6XWJ@_rpTDRNGl;}z#IN!1Xx)qV5#1$K^(?n&j_0~OZ`$4A=V{qQ5 zfZVwr!hiHg1PNFA)OFZGv8}wC&j+iKCQUM_>jrWa4DzH0C3U!!q#zA-!CvUe6Y>SCEa@VBfh-*80a=B1maZe5<30~A~&mB|Qj=N&| zavEPNQ5wQGX0y5VB=Ky@&MKMmD(s9@Jp=lyD_IWkcmN-JyCWZ<@y6rqWUg#W+37MA|vgkm1{#Y_qV^EeTe)5W|{F zJU0_t0gGsgQ1jW#XJ-5J;hq*>WCi5@wv?de2ZCl5<1E+$c3Bq^WIO3hQ&I|;1+W#G z5~`4g%ZE#K;11_-nN3y~X-(2-nY|Sr%J)7tHv!o83xlrYBil!P3@p5?j%|L+lm&o0 z&h|uux(X$eBzG(nnOax_ed+aEtvL2C(58300);16;wv&P{Xgd}Q9$wuSm$*L7 z%T$Vi>-Vmbwkei2EdE#}OYWsP2_yc7&=TrjN`z$j4}V8bE_lH0M3>X#75eJ)y^$M@ z*`G*uO9njp`~Y#NDBaM+Ceo{)?{B<>%$@nj^co604c8xm`czjSq-SPUvkmMp`K?_1 zp1F-YMz>!)TR||a)2_^`^Nyij3e023VTqQ#qJ5H!S6ua5rIoC-VE>U3#5Au_eug6l zS&(yf7{Rb5XMTs_Ds)Wdy3V(a>P%=idLRwCo&$DAUw^K`=LK9iUs^}PHQXknBHM7q z7#7X-OJPR^_to+*cLGnp9%Xf>*mhgx{*W)@8hHHF1Q*xHOUf?uy{@gmuFFP#OscLb zbX%NJ3i)a~TMbU1ia`ZIu8|uYqtxvq5i2Q?NZ;ZI;w?c);ek8kWSuNVN>$DEpCq4% zyL-5bFCCt?ADMg7E>zEubC4Rfk$Nt|L{TFc2zCmWtAM18digu zk=Pa-msFBr9u(^oj{L+9aE()5VeS#6m9X-d>w5Us8!mThgw4Am=YhCebMlXJoN4=1Y?+^oz8Qt~fFkVWR3%2H#epi6}W- zH16IS0QexebI4RY)`)NeM%h>_Ua(|7oOC-I&Q+3cg+_&u>WaW0$QMk_2#~5`pSENa>OO^gzf) z9UidBbz#cNWa{lo^6|IX2_SZa!0?X_i8OJ|fIC1E!Q2Z4LV~oRrQ%vyiRJ!KrZ7(>Ih8X-Gs58f;e3!W$jy$7 zY4}`hIEdLR&W@mslLT!@eu(jfN=2#{?tK4FEhZCFh(2eVahbu07p$fv%?`sA^T?p` zd*u$oVAPJwccSHG`VxIz{ep^^|GK)2Uo|)TRVmRMDcF||h}XAsls9tiy2Q-9fd!lI ruCT9AiyG%#B_hE=3`Izia#gxGv?0>DN4K)L?iGIGr3A}W;>Z67V2c~S diff --git a/capa/render/default.py b/capa/render/default.py index 259b7ff7a..ac2c2eef5 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -6,8 +6,9 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import pickle +import json import collections +from pathlib import Path import tabulate @@ -84,12 +85,16 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): +-------------------------------------------------------+-------------------------------------------------+ """ - def load_rules_prevalence(filename: str) -> dict: - with open(filename, "rb") as pickle_file: - return pickle.load(pickle_file) + def load_rules_prevalence(file: Path) -> dict: + try: + return json.load(file.open("r")) + except FileNotFoundError: + raise FileNotFoundError(f"File '{file}' not found.") + except Exception as e: + raise RuntimeError(f"An error occurred while loading '{file}': {e}") subrule_matches = find_subrule_matches(doc) - rules_prevalence = load_rules_prevalence("./assets/rules_prevalence.pickle") + rules_prevalence = load_rules_prevalence(Path("./assets/rules_prevalence.json")) # seperate rules based on their prevalence common = [] @@ -101,13 +106,13 @@ def load_rules_prevalence(filename: str) -> dict: count = len(rule.matches) matches = f"({count} matches)" if count > 1 else "" - rule_prevalence = float(rules_prevalence.get(rule.meta.name + "\n", 0)) + rule_prevalence = float(rules_prevalence.get(rule.meta.name, 0)) if rule_prevalence < 0: raise ValueError("Match probability cannot be negative") prevalences = [rutils.bold("rare"), rutils.bold("common"), "unknown"] - if rule_prevalence >= 0.05 or rule_prevalence == 0: + if rule_prevalence == 0 or rule_prevalence >= 0.05: prevalence = prevalences[2] if rule_prevalence == 0 else prevalences[1] common.append((rule.meta.namespace, rule.meta.name, matches, prevalence)) else: diff --git a/try.py b/try.py deleted file mode 100644 index c4e6aadc3..000000000 --- a/try.py +++ /dev/null @@ -1,54 +0,0 @@ -import os , subprocess, pickle, tqdm -import pandas as pd - -def get_files_with_extensions(directory, extensions): - files = [] - for filename in os.listdir(directory): - if os.path.isfile(os.path.join(directory, filename)): - _, ext = os.path.splitext(filename) - if ext.lower() in extensions: - files.append(os.path.join(directory, filename)) - return files - -extensions = ['.exe_', '.dll_', '.sys_', '.elf_', '.raw32', '.raw64', '.cs_', '.aspx_', '.py_'] -directory = r"C:\Users\HP\Documents\GitHub\capa\tests\data" - -all_paths = get_files_with_extensions(directory, extensions) -print("Total number of files to be processed ", len(all_paths)) - -pickle_path = "./all_rules_entropy.pickle" -write_path = "./rules_entropy.txt" -pbar = tqdm.tqdm -entropy = {} - -for file_path in pbar(all_paths): - cmd = ["capa", file_path] - - with open(write_path, 'w') as file: - file.write('') - - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=True) - except subprocess.CalledProcessError as e: - print("Error running capa on " + file_path + " : " + str(e)) - - with open(write_path, "r") as f: - for line in f.readlines(): - line.strip() - entropy[line] = entropy.get(line, 0) + 1 - - with open(pickle_path, 'wb') as pickle_file: - pickle.dump(entropy, pickle_file) - -with open(pickle_path, 'wb') as pickle_file: - pickle.dump(entropy, pickle_file) - -def save_dict_to_excel(data_dict, excel_file_path): - # Convert the dictionary to a pandas DataFrame - df = pd.DataFrame(data_dict.items()) - - # Save the DataFrame to an Excel file - df.to_excel(excel_file_path, index=False, header=["Rule", "Number of Matches"]) - -excel_file_path = "./entropy.xlsx" -save_dict_to_excel(entropy, excel_file_path) From c5302cd593eb339d0edc8c5bbf46d542211db628 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sun, 27 Aug 2023 16:36:10 +0530 Subject: [PATCH 08/32] prevalence db update --- assets/rules_prevalence.json.gz | Bin 0 -> 6056 bytes capa/render/default.py | 25 +++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 assets/rules_prevalence.json.gz diff --git a/assets/rules_prevalence.json.gz b/assets/rules_prevalence.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..96aba1db5e7e064a2a5932ed941cfdb736f223e8 GIT binary patch literal 6056 zcmV;Z7gy*XiwFokGwWmm|8jL~WpiI}a%Fa5Y-Mg^WiD!SZ*BmsU2SvYHj@6nzXFv{ zyS3+R;+aY1?wc&x9w*XRkvuP(+Nu;tLJ~$G!2qNzW$XU?bps$JO8^LGZ!25t*i8@w zx}Sdf1=xT8g}{HlQEbH&t8zwY)$prNGol;CtF`{-f5rfGo|oCl4s9SSP-w&4wwm;_wQ&C$*9*E*Rm}N~)4J90tC4g^+8qWd|5hZkZnX zJK1qcYAFP~-ZNcVH?3ks=ZdjPZ)9^Rqghpuf;RMg@`~k*?{HBH@~n8nj1*ZZSrX0E z4nC}rBo~}jo?#W!Qoz%b4@=)4yA^-`^fV_QVYsW&8pd%eADE>PR_ynd>4rS>W<%af z_-Vsv!IbCg6@wijHLqFlkIl1$50urRl&v@Qt9D$;3T4QS zDo!&I&?oyFVxXc11PN?e0o#)|(x*lR6AUD9go}qa3>1*B?C@;QwQtI-f{#(ujiJHl z@>Xd;199-r*^EXTu->d~*aZ;kVUbeqjx+YypLraX1-7M-&litdgc=Y!e%89>1R<(b zxzFP9lh%sX8=g}EDu$yGzagM#!Syzd>b;Z(9Ppr#*_9I3WpD5n_9gJD4Io}c%McWS zUts0wRL)kz_u9uL0WjCzc{^RB=(}L0v^BR<7JS9Q`91vOk!)bXs3XQtm^X!?*bNfi zp)*0<>Kc%r6)_cc!oGcMm@0X7B6R*fYnxJ$<8X?t!6@LjfQKPt_CK(rqGdkSzm)LW zYy8dULV>Z28o-z0-nZabiC-KJMg?XB322pschykX2jdsAaJG@Rcpl zr;>%aKqpPI0|1*ca67*RWv``bF1DsdodoxeSUiH~V1OlS+InD_P9Iye)qu1kbeha3 zH$B*f$jTW#Bn^N*Yw#;Ran-a`+{nG}dCQ|Zj& z!G7?%6}YMJQLx~QWbJC@8&F8VQ!^X|vnewlsJ9giNY`NFfPlicaAgf(TObDnl52;n z;q{0-u|>S0n*H!LNc1|k02Wp-`(xZ0d+5dJcG*4TWh5tz481=-5`PC!Q)lV`azKB< z^fOSi=u6POJj=3nO-j%J*o_cg8qeX!0&B!XNc3bHCIQ2Q zutUF6-n|_am-}?`h>=c2`@sa%h_2BL8#12Hkx5qtRYjO5BYH5zjwAPffw|z=iWV%O zq2O|HmmZ$rr=|O~ilc^eO|j29C)QS}Bh<+6_SqhI6|Byun`N#zDDh8pN45PC4B#jC zTL?HyR(jRPg#owwoC)$}IwiCyu6Q_J_$*;rKS$&|<^=DCpf9tL-x~HL4>1X_eS$6k zlvYq1Ox&~PiRas~Y<(fh@iBl^`!>`9)mlICX+FOF@)Sc8gq}~c&#?F8;gO&p@o$0w zSOj}CN5QJK59zH|eOydK6SXjH;yi*X;PodJm^z|j6w{sMoJO?K;4(@s&w`=xV*KXX z#~%>06}pm>&871x&ys_-7~q`;QmbkgPN2|=fa75WP&+hZKL7$@#wf`wHCWKP@_LLD z$CpVt7&rr_0wm=wflw%pi46QMoSKrtyv>7C*`Wq+NoG`SiCh`Q`^+-`rdjZOnD5@A zov8ph)tj~fps5^d_k0=B>;rNBAKbwXmTB$HPyo@P5yW!bG5B}OLV|xbenf7W!6Ec~ z8(#Y<0UDylV39Wm!V9oA3>1_OvYPqr4H1R<&CiaG4g{10#x-0sjH7zPgfQBI8wxOP z{BD53xY-@Lph?hb-5O_ygww0xQf`ToEl@s}c`JFYq?Rjm(+vm?3j@#~ zPtV|hd{|-paV~hBNpy<-m4Z){n2;ok&tHPx!V@!LET%vl6ADak8(H_!1BRNueKUz6 z$4g>LX~Kk_^B)6*NceX~24?;8PD0CjbOlC-Av{<~;Y|mcn4ZaFSZ@~0ixJil07e%a zwFee`IJlQ3;u^Rvjq#o^qG%_B8A0O%f=tJ3z7YwfIRVm**~NWH7lk+}UBm2HQa0@R z-4DLK)%`bHAE*6vGA-@30b{&KxI~oPM^fXF8x;p2cz|M3=x~0&3pqT&#Qw)0AW{YX zpaU3G{eM213HiL@dXpqPh6rB>PSdiqx68CaTiDg>ckjOc?dF; z2cH%qv}P?AMT9~5BelPN+X(%)&R52Asm6cNy7rPPgf#0#0RL*5gF6r9!kqVTB_ZVU z5X5j{n48~MIr6I*!iMwO-VcSpH$bC066I=7fy{v$_2XpjNb_)7JC+-5^)U@Ok#)_) z`d{sL-?6vV+WY!bc|c@Ab5?rGGHXNU!CMJTpTHx4d!>$s5}#)eCXJ_IC+x<3&R97` zER-sly#KuP6kJa7=J28+A9_U8GJc!|gVSTU z*jA<78DNmO@K~LY;V|pw;b}>}$}E8qVQ#w>kpST1XlyeI#P{~FbE05iP@uy znF7#G3i$VE&8-_#Z-Ce7n0jIKbi+VJCRWb?+67(z$sD#xqeR*214O(x3gD8d>qwrZ zka=qfAKFRu89!@^eE^awOmH{Ui#s@pxy4j+XFxuZd^OJSTlm000s;rKZW}{>hY(T) zUb|%n8=(n$fFLn2oZVaeP+;wV>5Z2nhWvXHV_Mk66#xn0V zyCbu&2-nd#-et)mfYFb7bV4f`%xz)c*E84wIFY6?XlnowTT(Lh`c0LyKV}sM$?KR{3F%sPhlWnXHBoIx13VF8314m6+~a2#Du| z73RRww)(Q`hvl>g=_VrfAp4Q3<)25(2tWuy-p` ziU?&^97l(6bR!EJ{z2s7++#4)Qu00KZM<{t!^49{wsG3m;mNHTEk&u-nN2pK*B|0c$IrtI zdn}zD^jyKInON>b-UcE;anqCeWi|~CMS3() zOn#s@i9LlqKllh3N9pw5D}MAGE`_sLj{Iqylla|r7>eMb49&!f*zF9bLQRP!lZCID zTrjFGgI^s8FfdXM#(yrNJjT*kX2__DTGn`SwE$f)uIZxmTI>GeD{> z>Sk@zMBDw`0X&Ztfo56yJ)_uDM{ck&bOAmb;Nq{do5kM-!)#2%4*_h|@@7k8O|A{U zLeHj40^qf&IL5^(QAKtLXh=#S?-y3KsqMcX$5X$`j^`ia+^IW-X!9~5=XYcV@P!*WDxS3o#qg|&~Q9xyL|^)TI3xyAU0XsrvB zI+OM!Al-#K68avCJ`b<4#}eD5Nn0U60(*%Z+t>h@f+4xo7KWb?_HF|l5UYd7g4W%f z1S(qfp|4*9USUs%Pm9;4*R_{)iV+>UZ~ilxo$ixk?XD}>^q0~hN!MC%ngqbPByh!- zh;T1c4eyth=iBM8DP>*w4gQ=MY8t6H8}$%1A#gA3ubbqVHu7 z-$n3Z1M>ZEcc7cTBSBqz^^u6Uwztj=kBW-fYc;uc=H?Rq&^-*p=J?T zWh$ipHoq6gS5F6tA4qlOm}66gT~tHvLvg{#as%Z0VlC(`kB!6R{x44pZtCJpK z`x=ZK*;SJG3dk`SU&XFEV@!PtBavOwM^Z8h`H-jS>U_r<_T!q>fsBcI%&D1$?C0SCX?mG`fZZkvsX9j~5QmcEd>fjA`jEsbnIU0!FRKnGs2psm2jC=s{7=j%LK^bejY zs#)@UpLMe6eo&B?$c$vwMRJ40FK|Nn$nVc0mHvcgAgPNqxWDwwa%)#m#sd(6c5f^R zSRhGhDBaL%ZIfr%{#Vo5H)O{!X$zqe443y5AN)RMunuI!4AIp7_8C?g3hU0_m=L!> zT8bH1oqlUUG(y9`7?v}WEd~!KG092s9V3$Ea-MqnZ18<1V)q;(jvOR9GHNrrT&>Kd zT>T${k(-2Iu_uW3t+t=fTg4#br%qSS3>LaHN*40n7l z9_F{R$@kYkrkx_bejCnHyts?VV#;(8IDc*wrcZ|0tMaDZ(fvi}3aXQ94o~sBZ>)?Q zIeM|<{T(6(O@l`#Z#%l~&0wDw`Ao?yQsC>7HeI&#MWxYRbx_;iMNbiWmHH$*`SL?e zhV9mU<()~)-OdAlgxRk7&b^U(cJCr^Qpy*DVl+O`^H+CtCn z*@-s(V67I}h*6Iv=zS47d;=fKlshSA@})tudI@m9@^$cd!Tt~{l=L?>xq^vlDPaD5 z@W4~psNAkreE-@f*Q{#MSDWsk*FN0UB?5Q!MW5}+LF~EX{Up+!u4QGgGwibdghUKy zhFTcDck+j~R~c`PjV)$!Ku2~A)^zP%lym=JZY;_MZ{-ql4a2`@qP9hX!MpuVwXD|H zFh&2>Idd#bn*tc7&{U|=C096aZ-2QlsbDKx2jH$(Aar@o$z+)l{^Ew~I_^YkM^M*l47@&;J_g=851!uNqG z+DwS*jy{>LwX)z_E=@XgU zPKR)RxFxCUZaqEz8>FJV+R5#C1Zs~#_&VLTsOO&lBLV;-DYrb~?8a#FDHiJmP)^nd zyv^Wx2$AvnvA|)wHF_OwapdIOT|}Li#tx+*+9O^ywoXf}FT3!Ln2AXSojXM!! zfjeYTL#(>5L!R83e9b~sr9mrrDC-5wL#I+ZI6b}iKlmogPwg2R|GoLAnedB4<>Bzu z+`Oj1W#AMFKtOwf?Gc2Idm&#i5zuex7( zS2KQz#foR%K1GZ9bE^)9Cd|EuJ*}KQ{_wN$4(^^76+H~$j`ei2K*PtEHK9AbkcRWB ijoj~9Q*5XW|M?`&y5|**#8!X#-~R&~As-tyX8-`OQMlp& literal 0 HcmV?d00001 diff --git a/capa/render/default.py b/capa/render/default.py index ac2c2eef5..3dac5ba2b 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -6,8 +6,10 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import gzip import json import collections +from typing import Dict from pathlib import Path import tabulate @@ -85,16 +87,18 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): +-------------------------------------------------------+-------------------------------------------------+ """ - def load_rules_prevalence(file: Path) -> dict: + def load_rules_prevalence(file: Path) -> Dict[str, str]: try: - return json.load(file.open("r")) + with gzip.open(file, "rb") as gzfile: + return json.loads(gzfile.read().decode("utf-8")) except FileNotFoundError: raise FileNotFoundError(f"File '{file}' not found.") except Exception as e: raise RuntimeError(f"An error occurred while loading '{file}': {e}") subrule_matches = find_subrule_matches(doc) - rules_prevalence = load_rules_prevalence(Path("./assets/rules_prevalence.json")) + CD = Path(__file__).resolve().parent.parent.parent + rules_prevalence = load_rules_prevalence(CD / "assets" / "rules_prevalence.json.gz") # seperate rules based on their prevalence common = [] @@ -106,17 +110,14 @@ def load_rules_prevalence(file: Path) -> dict: count = len(rule.matches) matches = f"({count} matches)" if count > 1 else "" - rule_prevalence = float(rules_prevalence.get(rule.meta.name, 0)) - if rule_prevalence < 0: - raise ValueError("Match probability cannot be negative") + prevalence = rules_prevalence.get(rule.meta.name, None) - prevalences = [rutils.bold("rare"), rutils.bold("common"), "unknown"] - - if rule_prevalence == 0 or rule_prevalence >= 0.05: - prevalence = prevalences[2] if rule_prevalence == 0 else prevalences[1] - common.append((rule.meta.namespace, rule.meta.name, matches, prevalence)) + if prevalence == "rare": + rare.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold(prevalence))) + elif prevalence == "common": + common.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold(prevalence))) else: - rare.append((rule.meta.namespace, rule.meta.name, matches, prevalences[0])) + common.append((rule.meta.namespace, rule.meta.name, matches, "unknown")) rows = [] From 430bde669c00d9205021e5b4df2df0b5f33c4db4 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sun, 27 Aug 2023 20:46:52 +0530 Subject: [PATCH 09/32] Update default.py --- capa/render/default.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/capa/render/default.py b/capa/render/default.py index 3dac5ba2b..e2276b42e 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -88,11 +88,11 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): """ def load_rules_prevalence(file: Path) -> Dict[str, str]: + if not file.exists(): + raise FileNotFoundError(f"File '{file}' not found.") try: with gzip.open(file, "rb") as gzfile: return json.loads(gzfile.read().decode("utf-8")) - except FileNotFoundError: - raise FileNotFoundError(f"File '{file}' not found.") except Exception as e: raise RuntimeError(f"An error occurred while loading '{file}': {e}") From 7f1566d5a07be12574fed27f186ca23a6985e5e4 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Mon, 28 Aug 2023 14:46:53 +0530 Subject: [PATCH 10/32] Update capa/render/default.py Co-authored-by: Moritz --- capa/render/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/render/default.py b/capa/render/default.py index e2276b42e..acdc2d111 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -110,7 +110,7 @@ def load_rules_prevalence(file: Path) -> Dict[str, str]: count = len(rule.matches) matches = f"({count} matches)" if count > 1 else "" - prevalence = rules_prevalence.get(rule.meta.name, None) + prevalence = rules_prevalence.get(rule.meta.name) if prevalence == "rare": rare.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold(prevalence))) From 67875555f7da3f0a3dad127a071fe33f9759bff3 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Wed, 6 Sep 2023 20:12:15 +0530 Subject: [PATCH 11/32] updated default render --- capa/render/default.py | 89 ++++++++++++++++++------------------------ capa/render/utils.py | 18 ++++++++- 2 files changed, 54 insertions(+), 53 deletions(-) diff --git a/capa/render/default.py b/capa/render/default.py index acdc2d111..088d24b8b 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -6,11 +6,8 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import gzip -import json import collections from typing import Dict -from pathlib import Path import tabulate @@ -78,66 +75,56 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): """ example:: - +-------------------------------------------------------+-------------------------------------------------+ - | CAPABILITY | NAMESPACE | - |-------------------------------------------------------+-------------------------------------------------| - | check for OutputDebugString error (2 matches) | anti-analysis/anti-debugging/debugger-detection | - | read and send data from client to server | c2/file-transfer | - | ... | ... | - +-------------------------------------------------------+-------------------------------------------------+ + +-------------------------------------------------------+-------------------------------------------------+------------+ + | CAPABILITY | NAMESPACE | PREVALENCE | + |-------------------------------------------------------+-------------------------------------------------|------------| + | check for OutputDebugString error (2 matches) | anti-analysis/anti-debugging/debugger-detection | rare | + | ... | ... | ... | + |-------------------------------------------------------|-------------------------------------------------|------------| + | read and send data from client to server | c2/file-transfer | common | + | ... | ... | ... | + +-------------------------------------------------------+-------------------------------------------------+------------+ """ - - def load_rules_prevalence(file: Path) -> Dict[str, str]: - if not file.exists(): - raise FileNotFoundError(f"File '{file}' not found.") - try: - with gzip.open(file, "rb") as gzfile: - return json.loads(gzfile.read().decode("utf-8")) - except Exception as e: - raise RuntimeError(f"An error occurred while loading '{file}': {e}") - subrule_matches = find_subrule_matches(doc) - CD = Path(__file__).resolve().parent.parent.parent - rules_prevalence = load_rules_prevalence(CD / "assets" / "rules_prevalence.json.gz") + rules_prevalence = rutils.load_rules_prevalence() # seperate rules based on their prevalence - common = [] - rare = [] + common: Dict[str, str] = {"capability": "", "namespace": "", "prevalence": ""} + had_common = False + rare: Dict[str, str] = {"capability": "", "namespace": "", "prevalence": ""} + had_rare = False + for rule in rutils.capability_rules(doc): if rule.meta.name in subrule_matches: continue count = len(rule.matches) - matches = f"({count} matches)" if count > 1 else "" - - prevalence = rules_prevalence.get(rule.meta.name) - - if prevalence == "rare": - rare.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold(prevalence))) - elif prevalence == "common": - common.append((rule.meta.namespace, rule.meta.name, matches, rutils.bold(prevalence))) + if count == 1: + capability = rutils.bold(rule.meta.name) + else: + capability = f"{rutils.bold(rule.meta.name)} ({count} matches)" + + namespace = rule.meta.namespace if rule.meta.namespace is not None else "" + prevalence = rules_prevalence.get(rule.meta.name, "unknown") + if prevalence == "rare" or prevalence == "common": + prevalence = rutils.bold(prevalence) + + if "rare" in prevalence: + rare["capability"] += capability + "\n" + rare["namespace"] += namespace + "\n" + rare["prevalence"] += prevalence + "\n" + had_rare = True else: - common.append((rule.meta.namespace, rule.meta.name, matches, "unknown")) + common["capability"] += capability + "\n" + common["namespace"] += namespace + "\n" + common["prevalence"] += prevalence + "\n" + had_common = True rows = [] - - def process_data(data): - "joins combine similar rules into a single section" - if len(data) == 0: - return - capabilities = "" - namespaces = "" - prevalences = "" - - for ns, c, i, p in sorted(data, key=lambda x: (x[0], x[1])): - capabilities += f"{rutils.bold(c)} {i}\n" - namespaces += ns + "\n" - prevalences += p + "\n" - - rows.append((capabilities.strip(), namespaces.strip(), prevalences.strip())) - - process_data(rare) - process_data(common) + if had_rare: + rows.append((rare["capability"], rare["namespace"], rare["prevalence"])) + if had_common: + rows.append((common["capability"], common["namespace"], common["prevalence"])) if rows: ostream.write( diff --git a/capa/render/utils.py b/capa/render/utils.py index fb3932340..1896e2616 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -5,9 +5,11 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. - import io -from typing import Union, Iterator +import gzip +import json +from typing import Dict, Union, Iterator +from pathlib import Path import termcolor @@ -56,6 +58,18 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: yield rule +def load_rules_prevalence() -> Dict[str, str]: + CD = Path(__file__).resolve().parent.parent.parent + file = CD / "assets" / "rules_prevalence.json.gz" + if not file.exists(): + raise FileNotFoundError(f"File '{file}' not found.") + try: + with gzip.open(file, "rb") as gzfile: + return json.loads(gzfile.read().decode("utf-8")) + except Exception as e: + raise RuntimeError(f"An error occurred while loading '{file}': {e}") + + class StringIO(io.StringIO): def writeln(self, s): self.write(s) From 7c84926f79093ae6a198e2e2a3290edd0c177982 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Wed, 6 Sep 2023 20:25:29 +0530 Subject: [PATCH 12/32] Update utils.py --- capa/render/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/capa/render/utils.py b/capa/render/utils.py index 1896e2616..819ad7d99 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -6,6 +6,7 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import io +import sys import gzip import json from typing import Dict, Union, Iterator @@ -59,8 +60,11 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: def load_rules_prevalence() -> Dict[str, str]: - CD = Path(__file__).resolve().parent.parent.parent - file = CD / "assets" / "rules_prevalence.json.gz" + if hasattr(sys, "frozen") and hasattr(sys, "_MEIPASS"): + file = Path(sys._MEIPASS) / "assets" / "rules_prevalence.json.gz" # type: ignore + else: + CD = Path(__file__).resolve().parent.parent.parent + file = CD / "assets" / "rules_prevalence.json.gz" if not file.exists(): raise FileNotFoundError(f"File '{file}' not found.") try: From c1f9e7282787d24754f91bfa4d41ab426dba4d56 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Wed, 6 Sep 2023 20:28:48 +0530 Subject: [PATCH 13/32] Revert "Update utils.py" This reverts commit 7c84926f79093ae6a198e2e2a3290edd0c177982. --- capa/render/utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/capa/render/utils.py b/capa/render/utils.py index 819ad7d99..1896e2616 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -6,7 +6,6 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import io -import sys import gzip import json from typing import Dict, Union, Iterator @@ -60,11 +59,8 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: def load_rules_prevalence() -> Dict[str, str]: - if hasattr(sys, "frozen") and hasattr(sys, "_MEIPASS"): - file = Path(sys._MEIPASS) / "assets" / "rules_prevalence.json.gz" # type: ignore - else: - CD = Path(__file__).resolve().parent.parent.parent - file = CD / "assets" / "rules_prevalence.json.gz" + CD = Path(__file__).resolve().parent.parent.parent + file = CD / "assets" / "rules_prevalence.json.gz" if not file.exists(): raise FileNotFoundError(f"File '{file}' not found.") try: From 8ede526ca438efaf02707f9a9719c1d64cf6cffa Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Tue, 10 Oct 2023 17:52:28 +0530 Subject: [PATCH 14/32] Resolving path issues --- CHANGELOG.md | 2 +- capa/render/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91ecf8cee..a5ab9a069 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - ghidra: add entry script helping users run capa against a loaded Ghidra database #1767 @mike-hunhoff - binja: add support for forwarded exports #1646 @xusheng6 - binja: add support for symtab names #1504 @xusheng6 +- Show prevalence of rules in the output #520 @Aayush-Goel-04 ### Breaking Changes @@ -55,7 +56,6 @@ Speaking of new rules, we have eight additions, coming from Ronnie, Jakub, Morit - ELF: implement import and export name extractor #1607 #1608 @Aayush-Goel-04 - bump pydantic from 1.10.9 to 2.1.1 #1582 @Aayush-Goel-04 - develop script to highlight features not used during matching #331 @Aayush-Goel-04 -- Show prevalence of rules in the output #520 @Aayush-Goel-04 ### New Rules (8) diff --git a/capa/render/utils.py b/capa/render/utils.py index 1896e2616..687a2e4bb 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -60,7 +60,7 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: def load_rules_prevalence() -> Dict[str, str]: CD = Path(__file__).resolve().parent.parent.parent - file = CD / "assets" / "rules_prevalence.json.gz" + file = CD / "assets/rules_prevalence.json.gz" if not file.exists(): raise FileNotFoundError(f"File '{file}' not found.") try: From 4476b2c6200006c69622c07be4f05d484bc30b0a Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Tue, 10 Oct 2023 18:21:18 +0530 Subject: [PATCH 15/32] Update utils.py --- capa/render/utils.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/capa/render/utils.py b/capa/render/utils.py index 687a2e4bb..29268d80d 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -60,6 +60,35 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: def load_rules_prevalence() -> Dict[str, str]: CD = Path(__file__).resolve().parent.parent.parent + import logging + + logger = logging.getLogger(__name__) + CD1 = Path(__file__).resolve() + CD2 = Path(__file__).resolve().parent + CD3 = Path(__file__).resolve().parent.parent + CD4 = Path(__file__).resolve().parent.parent.parent + CD5 = Path(__file__).resolve().parent.parent.parent.parent + logger.error(f"1 {CD1}") + logger.error(f"2 {CD2}") + logger.error(f"3 {CD3}") + logger.error(f"4 {CD4}") + logger.error(f"5 {CD5}") + CD1 = CD1 / "assets/rules_prevalence.json.gz" + CD2 = CD2 / "assets/rules_prevalence.json.gz" + CD3 = CD3 / "assets/rules_prevalence.json.gz" + CD4 = CD4 / "assets/rules_prevalence.json.gz" + CD5 = CD5 / "assets/rules_prevalence.json.gz" + if CD1.exists(): + logger.error(f"cd1 : {CD1}") + if CD2.exists(): + logger.error(f"cd2 : {CD2}") + if CD3.exists(): + logger.error(f"cd3 : {CD3}") + if CD4.exists(): + logger.error(f"cd4 : {CD4}") + if CD5.exists(): + logger.error(f"cd5 : {CD5}") + file = CD / "assets/rules_prevalence.json.gz" if not file.exists(): raise FileNotFoundError(f"File '{file}' not found.") From 6077e9948e94909f02fe146ba2ffb6dfdfaa1a4b Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Tue, 10 Oct 2023 18:26:56 +0530 Subject: [PATCH 16/32] Update utils.py --- capa/render/utils.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/capa/render/utils.py b/capa/render/utils.py index 29268d80d..687a2e4bb 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -60,35 +60,6 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: def load_rules_prevalence() -> Dict[str, str]: CD = Path(__file__).resolve().parent.parent.parent - import logging - - logger = logging.getLogger(__name__) - CD1 = Path(__file__).resolve() - CD2 = Path(__file__).resolve().parent - CD3 = Path(__file__).resolve().parent.parent - CD4 = Path(__file__).resolve().parent.parent.parent - CD5 = Path(__file__).resolve().parent.parent.parent.parent - logger.error(f"1 {CD1}") - logger.error(f"2 {CD2}") - logger.error(f"3 {CD3}") - logger.error(f"4 {CD4}") - logger.error(f"5 {CD5}") - CD1 = CD1 / "assets/rules_prevalence.json.gz" - CD2 = CD2 / "assets/rules_prevalence.json.gz" - CD3 = CD3 / "assets/rules_prevalence.json.gz" - CD4 = CD4 / "assets/rules_prevalence.json.gz" - CD5 = CD5 / "assets/rules_prevalence.json.gz" - if CD1.exists(): - logger.error(f"cd1 : {CD1}") - if CD2.exists(): - logger.error(f"cd2 : {CD2}") - if CD3.exists(): - logger.error(f"cd3 : {CD3}") - if CD4.exists(): - logger.error(f"cd4 : {CD4}") - if CD5.exists(): - logger.error(f"cd5 : {CD5}") - file = CD / "assets/rules_prevalence.json.gz" if not file.exists(): raise FileNotFoundError(f"File '{file}' not found.") From bc0d129a941ce517282ee64d8589c40d27d3fd73 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:20:16 +0530 Subject: [PATCH 17/32] Update pyinstaller.spec --- .github/pyinstaller/pyinstaller.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pyinstaller/pyinstaller.spec b/.github/pyinstaller/pyinstaller.spec index 7d90e9668..da3943be6 100644 --- a/.github/pyinstaller/pyinstaller.spec +++ b/.github/pyinstaller/pyinstaller.spec @@ -17,6 +17,7 @@ a = Analysis( # when invoking pyinstaller from the project root, # this gets invoked from the directory of the spec file, # i.e. ./.github/pyinstaller + ("../../assets", "assets"), ("../../rules", "rules"), ("../../sigs", "sigs"), ("../../cache", "cache"), From 5a0a3a56f73e6dfc1812e047c6c65afcc27aeb92 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 21 Oct 2023 00:40:18 +0530 Subject: [PATCH 18/32] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84f24037d..fdfc58fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - ghidra: add entry script helping users run capa against a loaded Ghidra database #1767 @mike-hunhoff - binja: add support for forwarded exports #1646 @xusheng6 - binja: add support for symtab names #1504 @xusheng6 +- add com class/interface features #322 @Aayush-goel-04 - Show prevalence of rules in the output #520 @Aayush-Goel-04 ### Breaking Changes From fe4af5cc15dc996cf9b2a2d2b3ffe070861117c5 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 21 Oct 2023 01:03:26 +0530 Subject: [PATCH 19/32] render output with prevalence for (v) verbose --- .../rules_prevalence.json.gz | Bin capa/render/verbose.py | 3 +++ capa/render/vverbose.py | 4 ++++ 3 files changed, 7 insertions(+) rename assets/{ => rules_prevalence_data}/rules_prevalence.json.gz (100%) diff --git a/assets/rules_prevalence.json.gz b/assets/rules_prevalence_data/rules_prevalence.json.gz similarity index 100% rename from assets/rules_prevalence.json.gz rename to assets/rules_prevalence_data/rules_prevalence.json.gz diff --git a/capa/render/verbose.py b/capa/render/verbose.py index c3ec24425..e8a32402c 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -115,6 +115,7 @@ def render_rules(ostream, doc: rd.ResultDocument): 0x10003797 """ had_match = False + rules_prevalence = rutils.load_rules_prevalence() for rule in rutils.capability_rules(doc): count = len(rule.matches) if count == 1: @@ -139,6 +140,8 @@ def render_rules(ostream, doc: rd.ResultDocument): rows.append((key, v)) + rows.append(("prevalence", rules_prevalence.get(rule.meta.name, "unknown"))) + if rule.meta.scope != capa.rules.FILE_SCOPE: locations = [m[0] for m in doc.rules[rule.meta.name].matches] rows.append(("matches", "\n".join(map(format_address, locations)))) diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index 03ff8c843..fa0dc4ad5 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -278,6 +278,8 @@ def render_rules(ostream, doc: rd.ResultDocument): had_match = False + rules_prevalence = rutils.load_rules_prevalence() + for _, _, rule in sorted((rule.meta.namespace or "", rule.meta.name, rule) for rule in doc.rules.values()): # default scope hides things like lib rules, malware-category rules, etc. # but in vverbose mode, we really want to show everything. @@ -325,6 +327,8 @@ def render_rules(ostream, doc: rd.ResultDocument): rows.append(("scope", rule.meta.scope.value)) + rows.append(("prevalence", rules_prevalence.get(rule.meta.name, "unknown"))) + if rule.meta.attack: rows.append(("att&ck", ", ".join([rutils.format_parts_id(v) for v in rule.meta.attack]))) From 95bdf5d8f84e6c976e00ce7978bdfd0125531c72 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 21 Oct 2023 01:11:57 +0530 Subject: [PATCH 20/32] Update utils.py --- capa/render/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capa/render/utils.py b/capa/render/utils.py index 687a2e4bb..38527ad0d 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -60,7 +60,7 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: def load_rules_prevalence() -> Dict[str, str]: CD = Path(__file__).resolve().parent.parent.parent - file = CD / "assets/rules_prevalence.json.gz" + file = CD / "assets/rules_prevalence_data/rules_prevalence.json.gz" if not file.exists(): raise FileNotFoundError(f"File '{file}' not found.") try: From af57da85c1e60c78e218d7701e58c8980b1c3a57 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sun, 12 Nov 2023 11:53:39 +0530 Subject: [PATCH 21/32] Update RuleMetaData with Prevalence --- capa/render/default.py | 5 +---- capa/render/result_document.py | 24 ++++++++++++++++++++++++ capa/render/utils.py | 17 +---------------- capa/render/verbose.py | 4 ++-- capa/render/vverbose.py | 7 +++---- 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/capa/render/default.py b/capa/render/default.py index 088d24b8b..af4ee3c86 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -86,7 +86,6 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): +-------------------------------------------------------+-------------------------------------------------+------------+ """ subrule_matches = find_subrule_matches(doc) - rules_prevalence = rutils.load_rules_prevalence() # seperate rules based on their prevalence common: Dict[str, str] = {"capability": "", "namespace": "", "prevalence": ""} @@ -105,9 +104,7 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): capability = f"{rutils.bold(rule.meta.name)} ({count} matches)" namespace = rule.meta.namespace if rule.meta.namespace is not None else "" - prevalence = rules_prevalence.get(rule.meta.name, "unknown") - if prevalence == "rare" or prevalence == "common": - prevalence = rutils.bold(prevalence) + prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown" if "rare" in prevalence: rare["capability"] += capability + "\n" diff --git a/capa/render/result_document.py b/capa/render/result_document.py index e8c27a49b..6f857c6db 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -5,6 +5,8 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. +import gzip +import json import datetime import collections from typing import Dict, List, Tuple, Union, Literal, Optional @@ -22,6 +24,13 @@ from capa.engine import MatchResults from capa.helpers import assert_never +try: + from functools import lru_cache +except ImportError: + # need to type ignore this due to mypy bug here (duplicate name): + # https://github.com/python/mypy/issues/1153 + from backports.functools_lru_cache import lru_cache # type: ignore + class FrozenModel(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -501,9 +510,23 @@ class MaecMetadata(FrozenModel): model_config = ConfigDict(frozen=True, populate_by_name=True) +@lru_cache(maxsize=None) +def load_rules_prevalence() -> Dict[str, str]: + CD = Path(__file__).resolve().parent.parent.parent + file = CD / "assets/rules_prevalence_data/rules_prevalence.json.gz" + if not file.exists(): + raise FileNotFoundError(f"File '{file}' not found.") + try: + with gzip.open(file, "rb") as gzfile: + return json.loads(gzfile.read().decode("utf-8")) + except Exception as e: + raise RuntimeError(f"An error occurred while loading '{file}': {e}") + + class RuleMetadata(FrozenModel): name: str namespace: Optional[str] = None + prevalence: str authors: Tuple[str, ...] scope: capa.rules.Scope attack: Tuple[AttackSpec, ...] = Field(alias="att&ck") @@ -521,6 +544,7 @@ def from_capa(cls, rule: capa.rules.Rule) -> "RuleMetadata": return cls( name=rule.meta.get("name"), namespace=rule.meta.get("namespace"), + prevalence=load_rules_prevalence().get(rule.meta.get("name"), "unknown"), authors=rule.meta.get("authors"), scope=capa.rules.Scope(rule.meta.get("scope")), attack=tuple(map(AttackSpec.from_str, rule.meta.get("att&ck", []))), diff --git a/capa/render/utils.py b/capa/render/utils.py index 38527ad0d..8d62ba8bf 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -6,10 +6,7 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. import io -import gzip -import json -from typing import Dict, Union, Iterator -from pathlib import Path +from typing import Union, Iterator import termcolor @@ -58,18 +55,6 @@ def capability_rules(doc: rd.ResultDocument) -> Iterator[rd.RuleMatches]: yield rule -def load_rules_prevalence() -> Dict[str, str]: - CD = Path(__file__).resolve().parent.parent.parent - file = CD / "assets/rules_prevalence_data/rules_prevalence.json.gz" - if not file.exists(): - raise FileNotFoundError(f"File '{file}' not found.") - try: - with gzip.open(file, "rb") as gzfile: - return json.loads(gzfile.read().decode("utf-8")) - except Exception as e: - raise RuntimeError(f"An error occurred while loading '{file}': {e}") - - class StringIO(io.StringIO): def writeln(self, s): self.write(s) diff --git a/capa/render/verbose.py b/capa/render/verbose.py index e8a32402c..6d3f433c1 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -115,7 +115,6 @@ def render_rules(ostream, doc: rd.ResultDocument): 0x10003797 """ had_match = False - rules_prevalence = rutils.load_rules_prevalence() for rule in rutils.capability_rules(doc): count = len(rule.matches) if count == 1: @@ -140,7 +139,8 @@ def render_rules(ostream, doc: rd.ResultDocument): rows.append((key, v)) - rows.append(("prevalence", rules_prevalence.get(rule.meta.name, "unknown"))) + prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown" + rows.insert(1, ("prevalence", prevalence)) if rule.meta.scope != capa.rules.FILE_SCOPE: locations = [m[0] for m in doc.rules[rule.meta.name].matches] diff --git a/capa/render/vverbose.py b/capa/render/vverbose.py index fa0dc4ad5..07f22fe7e 100644 --- a/capa/render/vverbose.py +++ b/capa/render/vverbose.py @@ -278,8 +278,6 @@ def render_rules(ostream, doc: rd.ResultDocument): had_match = False - rules_prevalence = rutils.load_rules_prevalence() - for _, _, rule in sorted((rule.meta.namespace or "", rule.meta.name, rule) for rule in doc.rules.values()): # default scope hides things like lib rules, malware-category rules, etc. # but in vverbose mode, we really want to show everything. @@ -307,6 +305,9 @@ def render_rules(ostream, doc: rd.ResultDocument): # library rules should not have a namespace rows.append(("namespace", rule.meta.namespace)) + prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown" + rows.append(("prevalence", prevalence)) + if rule.meta.maec.analysis_conclusion or rule.meta.maec.analysis_conclusion_ov: rows.append( ( @@ -327,8 +328,6 @@ def render_rules(ostream, doc: rd.ResultDocument): rows.append(("scope", rule.meta.scope.value)) - rows.append(("prevalence", rules_prevalence.get(rule.meta.name, "unknown"))) - if rule.meta.attack: rows.append(("att&ck", ", ".join([rutils.format_parts_id(v) for v in rule.meta.attack]))) From 8057a73333e64945955a617ea3b52289de9e6013 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sun, 12 Nov 2023 17:52:15 +0530 Subject: [PATCH 22/32] Apply suggestions from code review Co-authored-by: Moritz --- capa/render/default.py | 4 ++++ capa/render/result_document.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/capa/render/default.py b/capa/render/default.py index af4ee3c86..53e859948 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -73,6 +73,10 @@ def rec(match: rd.Match): def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): """ +render capabilities sorted by: +- prevalence rare -> unknown +- namespace a -> z + example:: +-------------------------------------------------------+-------------------------------------------------+------------+ diff --git a/capa/render/result_document.py b/capa/render/result_document.py index 6f857c6db..70d6fe2bc 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -513,7 +513,7 @@ class MaecMetadata(FrozenModel): @lru_cache(maxsize=None) def load_rules_prevalence() -> Dict[str, str]: CD = Path(__file__).resolve().parent.parent.parent - file = CD / "assets/rules_prevalence_data/rules_prevalence.json.gz" + file = CD / "assets" / "rules_prevalence_data" / "rules_prevalence.json.gz" if not file.exists(): raise FileNotFoundError(f"File '{file}' not found.") try: From 5102ca169f260a5c08dea39fb8199d79f40e87ea Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:54:23 +0530 Subject: [PATCH 23/32] Imports, Paths, Comments & Exceptions handled --- capa/render/default.py | 9 ++++++--- capa/render/result_document.py | 21 ++++++--------------- capa/rules/__init__.py | 14 +++----------- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/capa/render/default.py b/capa/render/default.py index 53e859948..a2bef6bd1 100644 --- a/capa/render/default.py +++ b/capa/render/default.py @@ -73,9 +73,9 @@ def rec(match: rd.Match): def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): """ -render capabilities sorted by: -- prevalence rare -> unknown -- namespace a -> z + render capabilities sorted by: + - prevalence (rare to unknown) + - namespace (alphabetical) example:: @@ -99,6 +99,9 @@ def render_capabilities(doc: rd.ResultDocument, ostream: StringIO): for rule in rutils.capability_rules(doc): if rule.meta.name in subrule_matches: + # rules that are also matched by other rules should not get rendered by default. + # this cuts down on the amount of output while giving approx the same detail. + # see #224 continue count = len(rule.matches) diff --git a/capa/render/result_document.py b/capa/render/result_document.py index 70d6fe2bc..7b13f9201 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -11,6 +11,7 @@ import collections from typing import Dict, List, Tuple, Union, Literal, Optional from pathlib import Path +from functools import lru_cache from pydantic import Field, BaseModel, ConfigDict @@ -24,13 +25,6 @@ from capa.engine import MatchResults from capa.helpers import assert_never -try: - from functools import lru_cache -except ImportError: - # need to type ignore this due to mypy bug here (duplicate name): - # https://github.com/python/mypy/issues/1153 - from backports.functools_lru_cache import lru_cache # type: ignore - class FrozenModel(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") @@ -512,21 +506,18 @@ class MaecMetadata(FrozenModel): @lru_cache(maxsize=None) def load_rules_prevalence() -> Dict[str, str]: - CD = Path(__file__).resolve().parent.parent.parent + CD = capa.main.get_default_root() file = CD / "assets" / "rules_prevalence_data" / "rules_prevalence.json.gz" if not file.exists(): - raise FileNotFoundError(f"File '{file}' not found.") - try: - with gzip.open(file, "rb") as gzfile: - return json.loads(gzfile.read().decode("utf-8")) - except Exception as e: - raise RuntimeError(f"An error occurred while loading '{file}': {e}") + return {} + with gzip.open(file, "rb") as gzfile: + return json.loads(gzfile.read().decode("utf-8")) class RuleMetadata(FrozenModel): name: str namespace: Optional[str] = None - prevalence: str + prevalence: str = "unknown" authors: Tuple[str, ...] scope: capa.rules.Scope attack: Tuple[AttackSpec, ...] = Field(alias="att&ck") diff --git a/capa/rules/__init__.py b/capa/rules/__init__.py index 9e39379b7..12ced0a5a 100644 --- a/capa/rules/__init__.py +++ b/capa/rules/__init__.py @@ -16,18 +16,9 @@ import binascii import collections from enum import Enum -from pathlib import Path - -from capa.helpers import assert_never - -try: - from functools import lru_cache -except ImportError: - # need to type ignore this due to mypy bug here (duplicate name): - # https://github.com/python/mypy/issues/1153 - from backports.functools_lru_cache import lru_cache # type: ignore - from typing import Any, Set, Dict, List, Tuple, Union, Iterator, Optional +from pathlib import Path +from functools import lru_cache import yaml import pydantic @@ -43,6 +34,7 @@ import capa.features.common import capa.features.basicblock from capa.engine import Statement, FeatureSet +from capa.helpers import assert_never from capa.features.common import MAX_BYTES_FEATURE_SIZE, Feature from capa.features.address import Address From 07553a63d656c18143cc2fb4f7bfcc4667563c6b Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:03:58 +0530 Subject: [PATCH 24/32] Update result_document.py --- capa/render/result_document.py | 1 + 1 file changed, 1 insertion(+) diff --git a/capa/render/result_document.py b/capa/render/result_document.py index 7b13f9201..d61c86f19 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -15,6 +15,7 @@ from pydantic import Field, BaseModel, ConfigDict +import capa.main import capa.rules import capa.engine import capa.features.common From 2c4931d13ebce8ffc4e661716565af03bd121e96 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Mon, 20 Nov 2023 17:27:25 +0530 Subject: [PATCH 25/32] Update result_document.py Comments on loading rules_prevalence and warning if file not found --- capa/render/result_document.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/capa/render/result_document.py b/capa/render/result_document.py index d61c86f19..49672292a 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -7,6 +7,7 @@ # See the License for the specific language governing permissions and limitations under the License. import gzip import json +import logging import datetime import collections from typing import Dict, List, Tuple, Union, Literal, Optional @@ -507,9 +508,28 @@ class MaecMetadata(FrozenModel): @lru_cache(maxsize=None) def load_rules_prevalence() -> Dict[str, str]: + """ + Load and return a dictionary containing prevalence information for rules defined in capa. + + Returns: + Dict[str, str]: A dictionary where keys are rule names, and values are prevalence levels. + + Example: + { + "capture screenshot": "rare", + "send data": "common", + "receive and write data from server to client": "common", + "resolve DNS": "common", + "reference HTTP User-Agent string": "rare" + } + + Note: + Prevalence levels can be one of the following: "common", "rare" + """ CD = capa.main.get_default_root() file = CD / "assets" / "rules_prevalence_data" / "rules_prevalence.json.gz" if not file.exists(): + logging.getLogger("capa").warning("Rules prevalence db was not found. Prevalence data will not be available.") return {} with gzip.open(file, "rb") as gzfile: return json.loads(gzfile.read().decode("utf-8")) From 61e745978ae9e9775baa9740010045387dc498b7 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 3 Feb 2024 19:47:52 +0530 Subject: [PATCH 26/32] Added prevalence to verbose --- capa/render/utils.py | 1 + capa/render/verbose.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/capa/render/utils.py b/capa/render/utils.py index 6b6c23c41..642b45a3b 100644 --- a/capa/render/utils.py +++ b/capa/render/utils.py @@ -5,6 +5,7 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. + import io from typing import Union, Iterator diff --git a/capa/render/verbose.py b/capa/render/verbose.py index 44024acf4..1eef6c003 100644 --- a/capa/render/verbose.py +++ b/capa/render/verbose.py @@ -265,6 +265,9 @@ def render_rules(ostream, doc: rd.ResultDocument): if ns: rows.append(("namespace", ns)) + prevalence = rutils.bold(rule.meta.prevalence) if rule.meta.prevalence != "unknown" else "unknown" + rows.append(("prevalence", prevalence)) + desc = rule.meta.description if desc: rows.append(("description", desc)) From 66d0ab7216b810ef67886d998c05a4a9cbbd4fb2 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 3 Feb 2024 19:52:29 +0530 Subject: [PATCH 27/32] linter checks --- capa/features/address.py | 3 ++- capa/ida/plugin/form.py | 6 +++--- capa/render/result_document.py | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/capa/features/address.py b/capa/features/address.py index 0edf4cec2..800cefcd3 100644 --- a/capa/features/address.py +++ b/capa/features/address.py @@ -10,7 +10,8 @@ class Address(abc.ABC): @abc.abstractmethod - def __eq__(self, other): ... + def __eq__(self, other): + ... @abc.abstractmethod def __lt__(self, other): diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index f62191864..dcc1ca462 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -932,9 +932,9 @@ def get_ask_use_persistent_cache(self, analyze): update_wait_box("verifying cached results") try: - results: Optional[capa.render.result_document.ResultDocument] = ( - capa.ida.helpers.load_and_verify_cached_results() - ) + results: Optional[ + capa.render.result_document.ResultDocument + ] = capa.ida.helpers.load_and_verify_cached_results() except Exception as e: capa.ida.helpers.inform_user_ida_ui("Failed to verify cached results, reanalyzing program") logger.exception("Failed to verify cached results (error: %s)", e) diff --git a/capa/render/result_document.py b/capa/render/result_document.py index 1581650b0..d5433adfc 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -165,7 +165,8 @@ class CompoundStatementType: OPTIONAL = "optional" -class StatementModel(FrozenModel): ... +class StatementModel(FrozenModel): + ... class CompoundStatement(StatementModel): @@ -685,9 +686,9 @@ def from_capa(cls, meta: Metadata, rules: RuleSet, capabilities: MatchResults) - return ResultDocument(meta=meta, rules=rule_matches) def to_capa(self) -> Tuple[Metadata, Dict]: - capabilities: Dict[str, List[Tuple[capa.features.address.Address, capa.features.common.Result]]] = ( - collections.defaultdict(list) - ) + capabilities: Dict[ + str, List[Tuple[capa.features.address.Address, capa.features.common.Result]] + ] = collections.defaultdict(list) # this doesn't quite work because we don't have the rule source for rules that aren't matched. rules_by_name = { From e3ca32b0acc308770598e10273aabea9b1017e19 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 3 Feb 2024 19:58:59 +0530 Subject: [PATCH 28/32] Revert "linter checks" This reverts commit 66d0ab7216b810ef67886d998c05a4a9cbbd4fb2. --- capa/features/address.py | 3 +-- capa/ida/plugin/form.py | 6 +++--- capa/render/result_document.py | 9 ++++----- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/capa/features/address.py b/capa/features/address.py index 800cefcd3..0edf4cec2 100644 --- a/capa/features/address.py +++ b/capa/features/address.py @@ -10,8 +10,7 @@ class Address(abc.ABC): @abc.abstractmethod - def __eq__(self, other): - ... + def __eq__(self, other): ... @abc.abstractmethod def __lt__(self, other): diff --git a/capa/ida/plugin/form.py b/capa/ida/plugin/form.py index dcc1ca462..f62191864 100644 --- a/capa/ida/plugin/form.py +++ b/capa/ida/plugin/form.py @@ -932,9 +932,9 @@ def get_ask_use_persistent_cache(self, analyze): update_wait_box("verifying cached results") try: - results: Optional[ - capa.render.result_document.ResultDocument - ] = capa.ida.helpers.load_and_verify_cached_results() + results: Optional[capa.render.result_document.ResultDocument] = ( + capa.ida.helpers.load_and_verify_cached_results() + ) except Exception as e: capa.ida.helpers.inform_user_ida_ui("Failed to verify cached results, reanalyzing program") logger.exception("Failed to verify cached results (error: %s)", e) diff --git a/capa/render/result_document.py b/capa/render/result_document.py index d5433adfc..1581650b0 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -165,8 +165,7 @@ class CompoundStatementType: OPTIONAL = "optional" -class StatementModel(FrozenModel): - ... +class StatementModel(FrozenModel): ... class CompoundStatement(StatementModel): @@ -686,9 +685,9 @@ def from_capa(cls, meta: Metadata, rules: RuleSet, capabilities: MatchResults) - return ResultDocument(meta=meta, rules=rule_matches) def to_capa(self) -> Tuple[Metadata, Dict]: - capabilities: Dict[ - str, List[Tuple[capa.features.address.Address, capa.features.common.Result]] - ] = collections.defaultdict(list) + capabilities: Dict[str, List[Tuple[capa.features.address.Address, capa.features.common.Result]]] = ( + collections.defaultdict(list) + ) # this doesn't quite work because we don't have the rule source for rules that aren't matched. rules_by_name = { From f084040b4ba4c253527879f544156031038cac75 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sat, 3 Feb 2024 20:33:36 +0530 Subject: [PATCH 29/32] Update result_document.py --- capa/render/result_document.py | 1 - 1 file changed, 1 deletion(-) diff --git a/capa/render/result_document.py b/capa/render/result_document.py index 1581650b0..eb4166909 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -18,7 +18,6 @@ from pydantic import Field, BaseModel, ConfigDict from typing_extensions import TypeAlias -import capa.main import capa.rules import capa.engine import capa.features.common From 10d214005e7e40dc40a73e0050e0dccc8aef2113 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:51:27 +0530 Subject: [PATCH 30/32] Convert database to python files --- capa/render/result_document.py | 21 +- capa/render/rules_prevalence.py | 603 ++++++++++++++++++++++++++++++++ 2 files changed, 610 insertions(+), 14 deletions(-) create mode 100644 capa/render/rules_prevalence.py diff --git a/capa/render/result_document.py b/capa/render/result_document.py index eb4166909..9a5e26291 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -5,9 +5,6 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and limitations under the License. -import gzip -import json -import logging import datetime import collections from enum import Enum @@ -23,6 +20,7 @@ import capa.features.common import capa.features.freeze as frz import capa.features.address +import capa.render.rules_prevalence import capa.features.freeze.features as frzf from capa.rules import RuleSet from capa.engine import MatchResults @@ -164,7 +162,8 @@ class CompoundStatementType: OPTIONAL = "optional" -class StatementModel(FrozenModel): ... +class StatementModel(FrozenModel): + ... class CompoundStatement(StatementModel): @@ -593,13 +592,7 @@ def load_rules_prevalence() -> Dict[str, str]: Note: Prevalence levels can be one of the following: "common", "rare" """ - CD = capa.main.get_default_root() - file = CD / "assets" / "rules_prevalence_data" / "rules_prevalence.json.gz" - if not file.exists(): - logging.getLogger("capa").warning("Rules prevalence db was not found. Prevalence data will not be available.") - return {} - with gzip.open(file, "rb") as gzfile: - return json.loads(gzfile.read().decode("utf-8")) + return capa.render.rules_prevalence.RULES_PREVALENCE class RuleMetadata(FrozenModel): @@ -684,9 +677,9 @@ def from_capa(cls, meta: Metadata, rules: RuleSet, capabilities: MatchResults) - return ResultDocument(meta=meta, rules=rule_matches) def to_capa(self) -> Tuple[Metadata, Dict]: - capabilities: Dict[str, List[Tuple[capa.features.address.Address, capa.features.common.Result]]] = ( - collections.defaultdict(list) - ) + capabilities: Dict[ + str, List[Tuple[capa.features.address.Address, capa.features.common.Result]] + ] = collections.defaultdict(list) # this doesn't quite work because we don't have the rule source for rules that aren't matched. rules_by_name = { diff --git a/capa/render/rules_prevalence.py b/capa/render/rules_prevalence.py new file mode 100644 index 000000000..3bf4461c9 --- /dev/null +++ b/capa/render/rules_prevalence.py @@ -0,0 +1,603 @@ +# Copyright (C) 2023 Mandiant, Inc. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: [package root]/LICENSE.txt +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under the License. +from typing import Dict + +RULES_PREVALENCE: Dict[str, str] = { + "reference anti-VM strings": "common", + "contain obfuscated stackstrings": "rare", + "gather firefox profile information": "common", + "log keystrokes": "common", + "log keystrokes via polling": "common", + "capture screenshot": "common", + "send data": "rare", + "receive and write data from server to client": "common", + "resolve DNS": "common", + "reference HTTP User-Agent string": "common", + "send HTTP request with Host header": "common", + "create pipe": "common", + "create two anonymous pipes": "common", + "read pipe": "common", + "get socket status": "common", + "initialize Winsock library": "common", + "set socket configuration": "common", + "act as TCP client": "common", + "encode data using XOR": "rare", + "create new key via CryptAcquireContext": "common", + "encrypt data using DPAPI": "common", + "encrypt data using RC4 PRGA": "common", + "hash data via WinCrypt": "common", + "hash data with MD5": "common", + "hash data using SHA1": "common", + "hash data using SHA1 via WinCrypt": "common", + "accept command line arguments": "common", + "query environment variable": "common", + "get common file path": "rare", + "create directory": "common", + "delete file": "rare", + "check if file exists": "common", + "enumerate files recursively": "common", + "get file attributes": "rare", + "set file attributes": "rare", + "move file": "common", + "read file on Windows": "rare", + "write file on Windows": "rare", + "enumerate gui resources": "common", + "get graphical window text": "common", + "get disk information": "common", + "get disk size": "common", + "check mutex and exit": "common", + "get hostname": "common", + "get system information on Windows": "common", + "check OS version": "common", + "create a process with modified I/O handles and window": "common", + "create process on Windows": "rare", + "create process suspended": "common", + "enumerate processes": "common", + "terminate process": "rare", + "query or enumerate registry key": "common", + "query or enumerate registry value": "rare", + "set registry value": "common", + "delete registry key": "common", + "delete registry value": "common", + "get session user name": "common", + "create thread": "rare", + "resume thread": "common", + "link many functions at runtime": "common", + "resolve function by parsing PE exports": "rare", + "persist via Active Setup registry key": "common", + "persist via Run registry key": "common", + "contains PDB path": "common", + "reference anti-VM strings targeting VirtualBox": "common", + "receive data": "common", + "start HTTP server": "common", + "access .NET resource": "common", + "manipulate console buffer": "common", + "load .NET assembly": "common", + "unmanaged call": "common", + "compiled to the .NET platform": "common", + "reference Base64 string": "common", + "encrypt or decrypt via WinCrypt": "common", + "encrypt data using AES via WinAPI": "common", + "initialize hashing via WinCrypt": "common", + "generate random numbers via WinAPI": "common", + "set current directory": "common", + "copy file": "common", + "print debug messages": "rare", + "create mutex": "common", + "get thread local storage value": "common", + "set thread local storage value": "common", + "create service": "common", + "enumerate services": "common", + "linked against CPP standard library": "common", + "parse PE header": "rare", + "schedule task via ITaskScheduler": "common", + "persist via Windows service": "common", + "encode data using Base64": "common", + "get file size": "common", + "execute VBScript Javascript or JScript in memory": "common", + "act as Excel XLL add-in": "common", + "act as Word WLL add-in": "common", + "encrypt data using memfrob from glibc": "common", + "check for time delay via GetTickCount": "common", + "check for VM using instruction VPCEXT": "common", + "reference the VMWare IO port": "common", + "get HTTP content length": "common", + "parse URL": "common", + "check HTTP status code": "common", + "connect to HTTP server": "common", + "create HTTP request": "common", + "get socket information": "common", + "hash data with CRC32": "common", + "decompress data using aPLib": "common", + "encrypt data using RC4 via WinAPI": "common", + "hash data using fnv": "common", + "contain an embedded PE file": "common", + "disable code signing": "common", + "manipulate boot configuration": "common", + "interact with driver via control codes": "common", + "bypass Mark of the Web": "common", + "get local IPv4 addresses": "common", + "shutdown system": "common", + "modify access privileges": "common", + "run as service": "common", + "delete service": "common", + "start service": "common", + "get token membership": "common", + "compare security identifiers": "common", + "access PEB ldr_data": "common", + "enumerate PE sections": "common", + "execute shellcode via indirect call": "common", + "identify system language via API": "common", + "hook routines via microsoft detours": "common", + "generate random numbers using a Mersenne Twister": "common", + "read clipboard data": "common", + "allocate RWX memory": "common", + "enumerate processes on remote desktop session host": "common", + "acquire debug privileges": "common", + "link function at runtime on Windows": "rare", + "linked against Microsoft Detours": "common", + "reference startup folder": "common", + "get geographical location": "common", + "parse credit card information": "common", + "get domain trust relationships": "common", + "initialize WinHTTP library": "common", + "read HTTP header": "common", + "prepare HTTP request": "common", + "receive HTTP response": "common", + "hash data using SHA256": "common", + "list domain servers": "common", + "set environment variable": "common", + "hide graphical window": "common", + "allocate thread local storage": "common", + "reference analysis tools strings": "common", + "decrypt data using AES via x86 extensions": "common", + "encrypt data using AES via x86 extensions": "common", + "encrypt data using blowfish": "common", + "encrypt data using Camellia": "common", + "encrypt data using DES": "common", + "encrypt data using Curve25519": "common", + "encrypt data using Salsa20 or ChaCha": "common", + "encrypt data using twofish": "common", + "hash data using murmur3": "common", + "hash data using SHA384": "common", + "hash data using SHA512": "common", + "hash data using tiger": "common", + "authenticate HMAC": "common", + "generate random numbers via RtlGenRandom": "common", + "debug build": "common", + "contain a thread local storage (.tls) section": "common", + "read file via mapping": "common", + "get memory capacity": "common", + "query remote server for available data": "common", + "create reverse shell": "common", + "convert IP address from string": "common", + "create UDP socket": "common", + "enumerate files on Windows": "common", + "use process Doppelg\u00e4nging": "common", + "use process replacement": "common", + "get installed programs": "common", + "connect to WMI namespace via WbemLocator": "common", + "encrypt data using RC4 KSA": "common", + "write file to startup folder": "common", + "reference SQL statements": "common", + "check if directory exists": "common", + "act as Office COM add-in": "common", + "check for unmoving mouse cursor": "common", + "create BITS job": "common", + "enumerate process modules": "common", + "bypass UAC via ICMLuaUtil": "common", + "check for debugger via API": "common", + "reference anti-VM strings targeting Xen": "common", + "obfuscated with Babel Obfuscator": "common", + "obfuscated with Dotfuscator": "common", + "obfuscated with Spices.Net Obfuscator": "common", + "obfuscated with Yano": "common", + "manipulate unmanaged memory in .NET": "common", + "get OS version in .NET": "common", + "resolve function by FNV-1a hash": "common", + "reference Google Public DNS server": "common", + "connect network resource": "common", + "enumerate disk volumes": "common", + "enumerate network shares": "common", + "check for PEB BeingDebugged flag": "common", + "reference anti-VM strings targeting VMWare": "common", + "initialize IWebBrowser2": "common", + "get HTTP document via IWebBrowser2": "common", + "encrypt data using AES": "common", + "reference AES constants": "common", + "get Program Files directory": "common", + "read .ini file": "common", + "get number of processors": "common", + "get keyboard layout": "common", + "get networking parameters": "common", + "enumerate threads": "common", + "schedule task via at": "common", + "get startup folder": "common", + "create TCP socket via raw AFD driver": "common", + "execute shellcode via Windows callback function": "common", + "create raw socket": "common", + "check for software breakpoints": "common", + "log keystrokes via application hook": "common", + "capture webcam image": "common", + "compute adler32 checksum": "common", + "compress data via WinAPI": "common", + "compress data via ZLIB inflate or deflate": "common", + "encrypt or decrypt data via BCrypt": "common", + "hash data via BCrypt": "common", + "hash data using djb2": "common", + "hash data using murmur2": "common", + "generate random numbers using the Delphi LCG": "common", + "extract resource via kernel32 functions": "common", + "get file system object information": "common", + "delete directory": "common", + "get file version info": "common", + "access firewall settings via INetFwMgr": "common", + "find graphical window": "common", + "enumerate devices by category": "common", + "get process heap flags": "common", + "get process heap force flags": "common", + "get process image filename": "common", + "query service status": "common", + "suspend thread": "common", + "terminate thread": "common", + "linked against CPP regex library": "common", + "linked against libcurl": "common", + "linked against OpenSSL": "common", + "linked against wolfSSL": "common", + "import public key": "common", + "empty the recycle bin": "common", + "stop service": "common", + "obtain TransmitPackets callback function via WSAIoctl": "common", + "connect pipe": "common", + "write clipboard data": "common", + "set application hook": "common", + "patch Event Tracing for Windows function": "common", + "get kernel32 base address": "common", + "inspect section memory permissions": "common", + "validate payment card number using luhn algorithm": "common", + "enumerate domain computers via LDAP": "common", + "get domain information": "common", + "steal KeePass passwords using KeeFarce": "common", + "inject thread": "common", + "timestomp file": "common", + "encrypt data using AES MixColumns step": "common", + "list user accounts": "common", + "get storage device properties": "common", + "access the Windows event log": "common", + "persist via IIS module": "common", + "persist via ISAPI extension": "common", + "forwarded export": "common", + "linked against PolarSSL/mbed TLS": "common", + "linked against XZip": "common", + "hide thread from debugger": "common", + "switch active desktop": "common", + "get MAC address on Windows": "common", + "send ICMP echo request": "common", + "decode data using Base64 via WinAPI": "common", + "empty recycle bin quietly": "common", + "get current user on Linux": "common", + "enumerate files on Linux": "common", + "read file on Linux": "common", + "write file on Linux": "common", + "create semaphore on Linux": "common", + "lock semaphore on Linux": "common", + "unlock semaphore on Linux": "common", + "get networking interfaces": "common", + "get kernel version": "common", + "create process on Linux": "common", + "decrypt data via SSPI": "common", + "encrypt data via SSPI": "common", + "get client handle via SChannel": "common", + "check mutex": "common", + "get system information on Linux": "common", + "terminate process via kill": "common", + "write pipe": "common", + "spawn thread to RWX shellcode": "common", + "obfuscated with SmartAssembly": "common", + "encrypt data using AES via .NET": "common", + "generate method via reflection in .NET": "common", + "mixed mode": "common", + "execute syscall instruction": "common", + "block operations on executable memory pages using Arbitrary Code Guard": "common", + "protect spawned processes with mitigation policies": "common", + "spoof parent PID": "common", + "get ntdll base address": "common", + "bypass UAC via AppInfo ALPC": "common", + "bypass UAC via RPC": "common", + "bypass UAC via token manipulation": "common", + "manually build AES constants": "common", + "reference cryptocurrency strings": "common", + "encrypt data using XXTEA": "common", + "modify service": "common", + "save image in .NET": "common", + "gather chrome based browser login information": "common", + "reference WMI statements": "common", + "decode data using Base64 in .NET": "common", + "find data using regex in .NET": "common", + "load XML in .NET": "common", + "enumerate drives": "common", + "access WMI data in .NET": "common", + "resolve path using msvcrt": "common", + "send file using FTP": "common", + "enumerate processes via NtQuerySystemInformation": "common", + "disable driver code integrity": "common", + "install driver": "common", + "get Windows directory from KUSER_SHARED_DATA": "common", + "set console window title": "common", + "map section object": "common", + "hijack thread execution": "common", + "check for time delay via QueryPerformanceCounter": "common", + "compiled with dmd": "common", + "read virtual disk": "common", + "create Restart Manager session": "common", + "delete volume shadow copies": "common", + "detect VM via disk hardware WMI queries": "common", + "detect VM via motherboard hardware WMI queries": "common", + "manipulate safe mode programs": "common", + "read raw disk data": "common", + "query service configuration": "common", + "get token privileges": "common", + "obfuscated with DeepSea Obfuscator": "common", + "delete internet cache": "common", + "check for PEB NtGlobalFlag flag": "common", + "implement COM DLL": "common", + "act as DHCP server callout DLL": "common", + "act as DNS server plugin DLL": "common", + "act as Security Support Provider DLL": "common", + "act as SubAuthentication Package DLL": "common", + "act as credential manager DLL": "common", + "reference public RSA key": "common", + "get domain controller name": "common", + "self delete": "common", + "extract HTTP body": "common", + "clear Windows event logs": "common", + "lock the desktop": "common", + "hide the Windows taskbar": "common", + "manipulate CD-ROM drive": "common", + "power down monitor": "common", + "swap mouse buttons": "common", + "execute command": "common", + "overwrite Master Boot Record (MBR)": "common", + "linked against ZLIB": "common", + "compiled with nuitka": "common", + "contain anti-disasm techniques": "common", + "terminate process by name": "common", + "reference anti-VM strings targeting Qemu": "common", + "decompress data using UCL": "common", + "get CPU information": "common", + "create device object": "common", + "enumerate minifilter drivers": "common", + "find process by PID": "common", + "inspect load icon resource": "common", + "capture screenshot via keybd event": "common", + "decrypt data using TEA": "common", + "encrypt data using TEA": "common", + "encrypt data using RC6": "common", + "check for sandbox and av modules": "common", + "check if process is running under wine": "common", + "create shortcut via IShellLink": "common", + "send TCP data via WFP API": "common", + "copy network traffic": "common", + "register network filter via WFP API": "common", + "allocate user process RWX memory": "common", + "free user process memory": "common", + "get OS information via KUSER_SHARED_DATA": "common", + "compiled with Go": "common", + "decompress data using QuickLZ": "common", + "schedule task via schtasks": "common", + "encrypt data using XTEA": "common", + "compiled with Borland Delphi": "common", + "load NCR ATM library": "common", + "encrypt data using RC4 with custom key via WinAPI": "common", + "act as password filter DLL": "common", + "schedule task via ITaskService": "common", + "reference absolute stream path on Windows": "common", + "linked against CPP JSON library": "common", + "access PE header": "common", + "compiled with cx_Freeze": "common", + "acquire credentials from Windows Credential Manager": "common", + "execute shell command and capture output": "common", + "encrypt data using OpenSSL DSA": "common", + "encrypt data using OpenSSL RSA": "common", + "enumerate browser history": "common", + "linked against wolfCrypt": "common", + "compiled with Nim": "common", + "compiled with MinGW for Windows": "common", + "disable AppInit_DLLs code signature enforcement": "common", + "persist via AppInit_DLLs registry key": "common", + "inject shellcode using extra window memory": "common", + "gather 3d-ftp information": "common", + "gather alftp information": "common", + "gather bitkinex information": "common", + "gather blazeftp information": "common", + "gather bulletproof-ftp information": "common", + "gather classicftp information": "common", + "gather coreftp information": "common", + "gather cuteftp information": "common", + "gather cyberduck information": "common", + "gather direct-ftp information": "common", + "gather directory-opus information": "common", + "gather expandrive information": "common", + "gather faststone-browser information": "common", + "gather fasttrack-ftp information": "common", + "gather ffftp information": "common", + "gather filezilla information": "common", + "gather flashfxp information": "common", + "gather fling-ftp information": "common", + "gather freshftp information": "common", + "gather frigate3 information": "common", + "gather ftp-commander information": "common", + "gather ftp-explorer information": "common", + "gather ftp-voyager information": "common", + "gather ftpgetter information": "common", + "gather ftpinfo information": "common", + "gather ftpnow information": "common", + "gather ftprush information": "common", + "gather ftpshell information": "common", + "gather global-downloader information": "common", + "gather goftp information": "common", + "gather leapftp information": "common", + "gather netdrive information": "common", + "gather nexusfile information": "common", + "gather nova-ftp information": "common", + "gather robo-ftp information": "common", + "gather securefx information": "common", + "gather smart-ftp information": "common", + "gather softx-ftp information": "common", + "gather southriver-webdrive information": "common", + "gather staff-ftp information": "common", + "gather total-commander information": "common", + "gather turbo-ftp information": "common", + "gather ultrafxp information": "common", + "gather winscp information": "common", + "gather winzip information": "common", + "gather wise-ftp information": "common", + "gather ws-ftp information": "common", + "gather xftp information": "common", + "impersonate user": "common", + "linked against aPLib": "common", + "reference NCR ATM library routines": "common", + "read and send data from client to server": "common", + "set global application hook": "common", + "change the wallpaper": "common", + "disable automatic Windows recovery features": "common", + "listen for remote procedure calls": "common", + "encode data using Base64 via WinAPI": "common", + "get outbound credentials handle via CredSSP": "common", + "encrypt data using DES via WinAPI": "common", + "get user security identifier": "common", + "create registry key via offline registry library": "common", + "open registry key via offline registry library": "common", + "query registry key via offline registry library": "common", + "set registry key via offline registry library": "common", + "set HTTP header": "common", + "check Internet connectivity via WinINet": "common", + "resolve function by Brute Ratel Badger hash": "common", + "resolve function by hash": "common", + "load Diebold Nixdorf ATM library": "common", + "linked against Crypto++": "common", + "receive HTTP request": "common", + "register HTTP server URL": "common", + "send HTTP response": "common", + "create mailslot": "common", + "read from mailslot": "common", + "compiled with exe4j": "common", + "resolve function by djb2 hash": "common", + "inject shellcode using window subclass procedure": "common", + "hash data using RIPEMD128": "common", + "hash data using SHA224": "common", + "execute .NET assembly via CLR host": "common", + "load Windows Common Language Runtime": "common", + "create new application domain in .NET": "common", + "get HTTP response content encoding": "common", + "get Explorer PID": "common", + "obfuscated with callobfuscator": "common", + "capture network configuration via ipconfig": "common", + "connect to URL": "common", + "decompress HTTP response via IEncodingFilterFactory": "common", + "create reverse shell on Linux": "common", + "execute shell command received from socket on Linux": "common", + "change file permission on Linux": "common", + "get memory information": "common", + "lock file": "common", + "get Linux distribution": "common", + "persist via .desktop autostart": "common", + "persist via shell profile or rc file": "common", + "persist via rc script": "common", + "execute anti-debugging instructions": "common", + "reference DNS over HTTPS endpoints": "common", + "deserialize JSON in .NET": "common", + "compiled with Zig": "common", + "get proxy": "common", + "check for Windows sandbox via device": "common", + "check for Windows sandbox via dns suffix": "common", + "check for Windows sandbox via genuine state": "common", + "check for Windows sandbox via process name": "common", + "check for Windows sandbox via registry": "common", + "check for microsoft office emulation": "common", + "check for sandbox username or hostname": "common", + "64-bit execution via heavens gate": "common", + "hash data using CRC32b": "common", + "persist via Winlogon Helper DLL registry key": "common", + "compiled with ps2exe": "common", + "run PowerShell expression": "common", + "crash the Windows event logging service": "common", + "capture public ip": "common", + "compiled with perl2exe": "common", + "inject shellcode using a file mapping object": "common", + "encrypt data using HC-128 via WolfSSL": "common", + "read data from CLFS log container": "common", + "rebuild import table": "common", + "encrypt data using skipjack": "common", + "get session integrity level": "common", + "decode data using Base64 via dword translation table": "common", + "encrypt data using vest": "common", + "get logon sessions": "common", + "discover Group Policy via gpresult": "common", + "create VMCI socket": "common", + "capture microphone audio": "common", + "start TCP server": "common", + "open clipboard": "common", + "compiled with pyarmor": "common", + "create TCP socket": "common", + "act as Exchange transport agent": "common", + "execute shellcode via CreateThreadpoolWait": "common", + "hash data using MD4": "common", + "check for OutputDebugString error": "common", + "check for protected handle exception": "common", + "check for trap flag exception": "common", + "check for unexpected memory writes": "common", + "check process job object": "common", + "reference anti-VM strings targeting Parallels": "common", + "reference anti-VM strings targeting VirtualPC": "common", + "get number of processor cores": "common", + "enumerate disk properties": "common", + "inject APC": "common", + "inject dll": "common", + "check for hardware breakpoints": "common", + "check for kernel debugger via shared user data structure": "common", + "identify ATM dispenser service provider": "common", + "reference Diebold ATM routines": "common", + "resolve function by FIN8 fasthash": "common", + "register minifilter driver": "common", + "start minifilter driver": "common", + "simulate CTRL ALT DEL": "common", + "get session information": "common", + "create virtual file system in .NET": "common", + "invoke .NET assembly method": "common", + "make an HTTP request with a Cookie": "common", + "obfuscated with ADVobfuscator": "common", + "execute shellcode via CopyFile2": "common", + "compiled with rust": "common", + "list groups for user account": "common", + "references logon banner": "common", + "enumerate internet cache": "common", + "log keystrokes via raw input data": "common", + "register raw input devices": "common", + "get routing table": "common", + "inject pe": "common", + "rebuilt by ImpRec": "common", + "patch process command line": "common", + "inject DLL reflectively": "common", + "impersonate file version information": "common", + "list drag and drop files": "common", + "encrypt data using HC-128": "common", + "compiled with V": "common", + "encrypt data using Sosemanuk": "common", + "compiled with py2exe": "common", + "compress data using LZO": "common", + "decompress data using LZO": "common", + "obfuscated with vs-obfuscation": "common", + "execute shellcode via Windows fibers": "common", + "check ProcessDebugPort": "common", + "check SystemKernelDebuggerInformation": "common", + "bypass Windows File Protection": "common", + "continue service": "common", + "pause service": "common", + "persist via GinaDLL registry key": "common", +} From 9bebffc7c23ebcdd184abf79477f61fbfb8ddbb1 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Mon, 5 Feb 2024 21:40:12 +0530 Subject: [PATCH 31/32] Lint checks --- capa/render/result_document.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/capa/render/result_document.py b/capa/render/result_document.py index 9a5e26291..719631335 100644 --- a/capa/render/result_document.py +++ b/capa/render/result_document.py @@ -162,8 +162,7 @@ class CompoundStatementType: OPTIONAL = "optional" -class StatementModel(FrozenModel): - ... +class StatementModel(FrozenModel): ... class CompoundStatement(StatementModel): @@ -677,9 +676,9 @@ def from_capa(cls, meta: Metadata, rules: RuleSet, capabilities: MatchResults) - return ResultDocument(meta=meta, rules=rule_matches) def to_capa(self) -> Tuple[Metadata, Dict]: - capabilities: Dict[ - str, List[Tuple[capa.features.address.Address, capa.features.common.Result]] - ] = collections.defaultdict(list) + capabilities: Dict[str, List[Tuple[capa.features.address.Address, capa.features.common.Result]]] = ( + collections.defaultdict(list) + ) # this doesn't quite work because we don't have the rule source for rules that aren't matched. rules_by_name = { From fa89f44563543639efbbdcfe51e69bcdfbc3cf70 Mon Sep 17 00:00:00 2001 From: Aayush Goel <81844215+Aayush-Goel-04@users.noreply.github.com> Date: Sun, 25 Feb 2024 06:10:47 +0530 Subject: [PATCH 32/32] Delete rules_prevalence.json.gz --- .../rules_prevalence.json.gz | Bin 6056 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 assets/rules_prevalence_data/rules_prevalence.json.gz diff --git a/assets/rules_prevalence_data/rules_prevalence.json.gz b/assets/rules_prevalence_data/rules_prevalence.json.gz deleted file mode 100644 index 96aba1db5e7e064a2a5932ed941cfdb736f223e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6056 zcmV;Z7gy*XiwFokGwWmm|8jL~WpiI}a%Fa5Y-Mg^WiD!SZ*BmsU2SvYHj@6nzXFv{ zyS3+R;+aY1?wc&x9w*XRkvuP(+Nu;tLJ~$G!2qNzW$XU?bps$JO8^LGZ!25t*i8@w zx}Sdf1=xT8g}{HlQEbH&t8zwY)$prNGol;CtF`{-f5rfGo|oCl4s9SSP-w&4wwm;_wQ&C$*9*E*Rm}N~)4J90tC4g^+8qWd|5hZkZnX zJK1qcYAFP~-ZNcVH?3ks=ZdjPZ)9^Rqghpuf;RMg@`~k*?{HBH@~n8nj1*ZZSrX0E z4nC}rBo~}jo?#W!Qoz%b4@=)4yA^-`^fV_QVYsW&8pd%eADE>PR_ynd>4rS>W<%af z_-Vsv!IbCg6@wijHLqFlkIl1$50urRl&v@Qt9D$;3T4QS zDo!&I&?oyFVxXc11PN?e0o#)|(x*lR6AUD9go}qa3>1*B?C@;QwQtI-f{#(ujiJHl z@>Xd;199-r*^EXTu->d~*aZ;kVUbeqjx+YypLraX1-7M-&litdgc=Y!e%89>1R<(b zxzFP9lh%sX8=g}EDu$yGzagM#!Syzd>b;Z(9Ppr#*_9I3WpD5n_9gJD4Io}c%McWS zUts0wRL)kz_u9uL0WjCzc{^RB=(}L0v^BR<7JS9Q`91vOk!)bXs3XQtm^X!?*bNfi zp)*0<>Kc%r6)_cc!oGcMm@0X7B6R*fYnxJ$<8X?t!6@LjfQKPt_CK(rqGdkSzm)LW zYy8dULV>Z28o-z0-nZabiC-KJMg?XB322pschykX2jdsAaJG@Rcpl zr;>%aKqpPI0|1*ca67*RWv``bF1DsdodoxeSUiH~V1OlS+InD_P9Iye)qu1kbeha3 zH$B*f$jTW#Bn^N*Yw#;Ran-a`+{nG}dCQ|Zj& z!G7?%6}YMJQLx~QWbJC@8&F8VQ!^X|vnewlsJ9giNY`NFfPlicaAgf(TObDnl52;n z;q{0-u|>S0n*H!LNc1|k02Wp-`(xZ0d+5dJcG*4TWh5tz481=-5`PC!Q)lV`azKB< z^fOSi=u6POJj=3nO-j%J*o_cg8qeX!0&B!XNc3bHCIQ2Q zutUF6-n|_am-}?`h>=c2`@sa%h_2BL8#12Hkx5qtRYjO5BYH5zjwAPffw|z=iWV%O zq2O|HmmZ$rr=|O~ilc^eO|j29C)QS}Bh<+6_SqhI6|Byun`N#zDDh8pN45PC4B#jC zTL?HyR(jRPg#owwoC)$}IwiCyu6Q_J_$*;rKS$&|<^=DCpf9tL-x~HL4>1X_eS$6k zlvYq1Ox&~PiRas~Y<(fh@iBl^`!>`9)mlICX+FOF@)Sc8gq}~c&#?F8;gO&p@o$0w zSOj}CN5QJK59zH|eOydK6SXjH;yi*X;PodJm^z|j6w{sMoJO?K;4(@s&w`=xV*KXX z#~%>06}pm>&871x&ys_-7~q`;QmbkgPN2|=fa75WP&+hZKL7$@#wf`wHCWKP@_LLD z$CpVt7&rr_0wm=wflw%pi46QMoSKrtyv>7C*`Wq+NoG`SiCh`Q`^+-`rdjZOnD5@A zov8ph)tj~fps5^d_k0=B>;rNBAKbwXmTB$HPyo@P5yW!bG5B}OLV|xbenf7W!6Ec~ z8(#Y<0UDylV39Wm!V9oA3>1_OvYPqr4H1R<&CiaG4g{10#x-0sjH7zPgfQBI8wxOP z{BD53xY-@Lph?hb-5O_ygww0xQf`ToEl@s}c`JFYq?Rjm(+vm?3j@#~ zPtV|hd{|-paV~hBNpy<-m4Z){n2;ok&tHPx!V@!LET%vl6ADak8(H_!1BRNueKUz6 z$4g>LX~Kk_^B)6*NceX~24?;8PD0CjbOlC-Av{<~;Y|mcn4ZaFSZ@~0ixJil07e%a zwFee`IJlQ3;u^Rvjq#o^qG%_B8A0O%f=tJ3z7YwfIRVm**~NWH7lk+}UBm2HQa0@R z-4DLK)%`bHAE*6vGA-@30b{&KxI~oPM^fXF8x;p2cz|M3=x~0&3pqT&#Qw)0AW{YX zpaU3G{eM213HiL@dXpqPh6rB>PSdiqx68CaTiDg>ckjOc?dF; z2cH%qv}P?AMT9~5BelPN+X(%)&R52Asm6cNy7rPPgf#0#0RL*5gF6r9!kqVTB_ZVU z5X5j{n48~MIr6I*!iMwO-VcSpH$bC066I=7fy{v$_2XpjNb_)7JC+-5^)U@Ok#)_) z`d{sL-?6vV+WY!bc|c@Ab5?rGGHXNU!CMJTpTHx4d!>$s5}#)eCXJ_IC+x<3&R97` zER-sly#KuP6kJa7=J28+A9_U8GJc!|gVSTU z*jA<78DNmO@K~LY;V|pw;b}>}$}E8qVQ#w>kpST1XlyeI#P{~FbE05iP@uy znF7#G3i$VE&8-_#Z-Ce7n0jIKbi+VJCRWb?+67(z$sD#xqeR*214O(x3gD8d>qwrZ zka=qfAKFRu89!@^eE^awOmH{Ui#s@pxy4j+XFxuZd^OJSTlm000s;rKZW}{>hY(T) zUb|%n8=(n$fFLn2oZVaeP+;wV>5Z2nhWvXHV_Mk66#xn0V zyCbu&2-nd#-et)mfYFb7bV4f`%xz)c*E84wIFY6?XlnowTT(Lh`c0LyKV}sM$?KR{3F%sPhlWnXHBoIx13VF8314m6+~a2#Du| z73RRww)(Q`hvl>g=_VrfAp4Q3<)25(2tWuy-p` ziU?&^97l(6bR!EJ{z2s7++#4)Qu00KZM<{t!^49{wsG3m;mNHTEk&u-nN2pK*B|0c$IrtI zdn}zD^jyKInON>b-UcE;anqCeWi|~CMS3() zOn#s@i9LlqKllh3N9pw5D}MAGE`_sLj{Iqylla|r7>eMb49&!f*zF9bLQRP!lZCID zTrjFGgI^s8FfdXM#(yrNJjT*kX2__DTGn`SwE$f)uIZxmTI>GeD{> z>Sk@zMBDw`0X&Ztfo56yJ)_uDM{ck&bOAmb;Nq{do5kM-!)#2%4*_h|@@7k8O|A{U zLeHj40^qf&IL5^(QAKtLXh=#S?-y3KsqMcX$5X$`j^`ia+^IW-X!9~5=XYcV@P!*WDxS3o#qg|&~Q9xyL|^)TI3xyAU0XsrvB zI+OM!Al-#K68avCJ`b<4#}eD5Nn0U60(*%Z+t>h@f+4xo7KWb?_HF|l5UYd7g4W%f z1S(qfp|4*9USUs%Pm9;4*R_{)iV+>UZ~ilxo$ixk?XD}>^q0~hN!MC%ngqbPByh!- zh;T1c4eyth=iBM8DP>*w4gQ=MY8t6H8}$%1A#gA3ubbqVHu7 z-$n3Z1M>ZEcc7cTBSBqz^^u6Uwztj=kBW-fYc;uc=H?Rq&^-*p=J?T zWh$ipHoq6gS5F6tA4qlOm}66gT~tHvLvg{#as%Z0VlC(`kB!6R{x44pZtCJpK z`x=ZK*;SJG3dk`SU&XFEV@!PtBavOwM^Z8h`H-jS>U_r<_T!q>fsBcI%&D1$?C0SCX?mG`fZZkvsX9j~5QmcEd>fjA`jEsbnIU0!FRKnGs2psm2jC=s{7=j%LK^bejY zs#)@UpLMe6eo&B?$c$vwMRJ40FK|Nn$nVc0mHvcgAgPNqxWDwwa%)#m#sd(6c5f^R zSRhGhDBaL%ZIfr%{#Vo5H)O{!X$zqe443y5AN)RMunuI!4AIp7_8C?g3hU0_m=L!> zT8bH1oqlUUG(y9`7?v}WEd~!KG092s9V3$Ea-MqnZ18<1V)q;(jvOR9GHNrrT&>Kd zT>T${k(-2Iu_uW3t+t=fTg4#br%qSS3>LaHN*40n7l z9_F{R$@kYkrkx_bejCnHyts?VV#;(8IDc*wrcZ|0tMaDZ(fvi}3aXQ94o~sBZ>)?Q zIeM|<{T(6(O@l`#Z#%l~&0wDw`Ao?yQsC>7HeI&#MWxYRbx_;iMNbiWmHH$*`SL?e zhV9mU<()~)-OdAlgxRk7&b^U(cJCr^Qpy*DVl+O`^H+CtCn z*@-s(V67I}h*6Iv=zS47d;=fKlshSA@})tudI@m9@^$cd!Tt~{l=L?>xq^vlDPaD5 z@W4~psNAkreE-@f*Q{#MSDWsk*FN0UB?5Q!MW5}+LF~EX{Up+!u4QGgGwibdghUKy zhFTcDck+j~R~c`PjV)$!Ku2~A)^zP%lym=JZY;_MZ{-ql4a2`@qP9hX!MpuVwXD|H zFh&2>Idd#bn*tc7&{U|=C096aZ-2QlsbDKx2jH$(Aar@o$z+)l{^Ew~I_^YkM^M*l47@&;J_g=851!uNqG z+DwS*jy{>LwX)z_E=@XgU zPKR)RxFxCUZaqEz8>FJV+R5#C1Zs~#_&VLTsOO&lBLV;-DYrb~?8a#FDHiJmP)^nd zyv^Wx2$AvnvA|)wHF_OwapdIOT|}Li#tx+*+9O^ywoXf}FT3!Ln2AXSojXM!! zfjeYTL#(>5L!R83e9b~sr9mrrDC-5wL#I+ZI6b}iKlmogPwg2R|GoLAnedB4<>Bzu z+`Oj1W#AMFKtOwf?Gc2Idm&#i5zuex7( zS2KQz#foR%K1GZ9bE^)9Cd|EuJ*}KQ{_wN$4(^^76+H~$j`ei2K*PtEHK9AbkcRWB ijoj~9Q*5XW|M?`&y5|**#8!X#-~R&~As-tyX8-`OQMlp&