From 41cca2dd4d3041a77e8624c8c47c235f0a0a6f4f Mon Sep 17 00:00:00 2001 From: Evgeniy Lozovan Date: Tue, 16 May 2017 00:34:32 +0300 Subject: [PATCH] a note on flaky annotation usage (fixes #5, via #45) --- docs/2.0/features.adoc | 4 +++- docs/2.0/features/flaky.adoc | 20 ++++++++++++++++++++ docs/2.0/images/flaky_failed.png | Bin 0 -> 8389 bytes 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 docs/2.0/features/flaky.adoc create mode 100644 docs/2.0/images/flaky_failed.png diff --git a/docs/2.0/features.adoc b/docs/2.0/features.adoc index b88af02..bc16901 100644 --- a/docs/2.0/features.adoc +++ b/docs/2.0/features.adoc @@ -3,4 +3,6 @@ This section describes the main features of Allure. For example, you can group your tests by stories or features, attach files, and distribute assertions over a set of custom steps, among other features. All features are supported by Java test frameworks, so we only provide Java examples here. For details on how a particular adapter works with the test framework of your choice, -refer to the adapter guide. \ No newline at end of file +refer to the adapter guide. + +include::features/flaky.adoc[leveloffset=+1] \ No newline at end of file diff --git a/docs/2.0/features/flaky.adoc b/docs/2.0/features/flaky.adoc new file mode 100644 index 0000000..6ea6d2c --- /dev/null +++ b/docs/2.0/features/flaky.adoc @@ -0,0 +1,20 @@ += Flaky tests + +In a real life not all of your tests are stable and always green or always red. A test might start to "blink" i.e. it fails from time-to-time without any obvious reason. +You could disable such a test, that is a trivial solution. However what if you do not want to do that? +Say you would like to get more details on possible reasons or the test is so critical that even being flaky it provides helpful information? +You have an option now to mark such tests in a special way, so resulting report will clearly show them as unstable: + +[source, java] +---- +@Flaky +public void aTestWhichFailsFromTimeToTime { + ... +} +---- + +Here is what you get in the report in case if such test failed: ++ +image::flaky_failed.png[A failed test marked as flaky] + +Additional note - you could mark a whole test class as flaky as well. \ No newline at end of file diff --git a/docs/2.0/images/flaky_failed.png b/docs/2.0/images/flaky_failed.png new file mode 100644 index 0000000000000000000000000000000000000000..77f34f2e0f11c36a28ead6b93ab0df61ecbab36d GIT binary patch literal 8389 zcmcI}WmH_v((Vul4DJ?eaA$A^cNp9W?iw5hcNkoPy9Eo71PJaH2oNMd(4do`!Gi_@ z+HIYq9s%hY#`Is#~e zvp!&NjGz3tB8SPpR}l96O*5j@B%Msn8BxZ*T_Ln?f^hP8(A#z-a^l2OHmm%69xH>x zi7RQCM_35jNoXTK`@sk@jOb9yBPHgz2sO$>kn=e(5-28#Qpx=l)*ztZZXKNWf!hXJ zFS_Q~@fcy}`G+aFHqMi)ZDSFC`V8^xctp{~qllY$o$2+-%J+}bM2MPqbA{ct5vhnc zn@QL7vbDABWZr3Oh>_Kt?*OfRQ?%ks6^#y%y9lJLBzxua+`t@v!xZ^Ap<$tWJw^BLHkN zgeO?IWvDn2?31H-5ETC{+-CzM1ei9mM0uDmB~Z6zrhNvUHIigS@HdjVqL~ijHB!w- zp>6@4(73m7Ao%qWJf6gB(Y~|z77_Gkc*!83oSYMoW>yA9$TA8yXZ9LF8f=m8OnztQi6Lzh9n4%`gF-;9B;*D`2kDj(P#!4bzTfED z08P-aqp0ZNX;3KZBz+bE2QV0O6mxH@?|`+3kthMD9CMcPWP-_t#Y@VI)Jy!F;A8f( zMu<8!OFHxg6Ltm$d&V%sPNpi<6nd}nR%HqLP>F9B*u=AvN)6Ryxfoa6#`-PwoBg-Y zZ-L+OMHxluMTKUS)XMH9YcDz_y{Eii?%eGtv3(&SPQ+l&$xty?T~QZ>iYj+%1u}xc z4d7D_X@e5heuc2+zt(?}euw_1tL4!5siX51Gf*(s`0%Dp_f&gx#!1k&K!iI#N6S>> zT}5$~WM!y!8K)5^oXwMjj8mUOkj+dlpqA~eQ{{dIMDL^aN4fKw9|j|`%%aS;%+ky` za|~6gRm7_8S+Us%U)sK;e-ZuiF6)NfQa7w#PM6ORQyWo3^6tn`^xf~g)LGp9Fqm*C zWhnjwb+2W1&8zYkRXydqKGZH3-QJ~E&UtAy}H~wI^F)-N0sPW!m{+LkD3)aKO&fcu>q$QdS-P|v#w|9srTV4$)nAhxH-C>7!=dpgRxidC zdkaH{9dwm+Nvz{`z4d*vs^bgITbZ1i^xI7<$+2iGfqLXxyxoZJO9(3pOMj$)z(6NK zC65-2#zl8Smyhj=MZ{v`9O9n77^m!{u%~e2@3OkwSRRH926TE=ocj>_W)ft|Wd>#5 zR}1Qaze#+1^^M9wL4drOtNGZ$e7B~vA}}_v>W=~T{Lpxmrx?S32pUi8E;#$_4#Tz;+N=_?$&(lBJLyJ zW?r?98(a`t_F!|*a{m&K49_f%BW^VnHtIRLlj@dgSyWVvrioFEUW8k$SJW=L>8<0N z;BCC)J)6Dt>#%FNb>ym|Fw?Yem9V{k?ZaC0+D)f)sVjfJc;MIVQQA9NuGi+Yy(Q{j zNxl}B^jZX2a5cPYP~Je_pg!&0_!=@9!XJ8l-*ee|PI^mw6MDM0+dk8(& zqm3Dlz5L>gS6iA_vQf&DZH&(;OxuN*9wHcEe)fUjn7|KeJ~l(L$BzBBB~Dk~R<1o( z3QGW=7%mi7fzTUTirak6o4BPcs1!ApHg=VcU#?#MyF9E=wdjRVtM{*OPOJORXUT2$ z`I>F6M|zU3Qp|~i!PA18Buius6w;D?JddGK;c|g;9zQCFV-j9w2xbPTXpC2G-){Gg zfBN>U+O23&dfh8%{IZ0SMPPTjLr~Ah;d(N&nZ)s<1C0iKB~#^*+MmfpJA>v}J$c>s zD!h*m%Z1Be$VL%v5lTHqd(l1D6ypbp;70i`5?=ew!QPx|Izg%JWuEi@pj{bWl+d13UyPZ33lT<`-tZOVn1dg>{A4ktF zEM8qvI^hmW0-FO{YBHhJ2Uh>v>~SqMaQDJ+I@plH=yoC`L@GFJ^;z$8HS$aDOXCg~ z!X`FfomHFYk&&ePTr}=lb5-Ml*2^1)LS(?&Cx~){$X&?6>|uZPX0)!Dv6!PWOefZJ zPmLlQwm$5eIrn={d}*jEnwzi}wkNeOefjaF`Qys<-jwv~dQH>y&d<&o3no8I9>16P zS{%Nb$ezHSpliUl+-!{Od;|Nmn$j$6QRi8+{4t?PrKQ&2XL+}pJJxj3{ZEgO?ctQ_ z<`?;(!`R4oZch0je zwGy>{-c#&fYJcceyv5eSZ{KahD^ZcY=x{S!?QKqQ^k~s?JCz_0I{EQu;KC7nYX%9O z{b@77+^)vT({1M7^8;4(=F`WYx2LK7bf>Uit{uNZT5Bg--*pCi-_*KTEKQr;^oKnv zjiqpkFuMEQ3}3DF2|XB|x=kPN`8S>RbeVekZ*OGW!|$u`vuO1t=7WqL@SDS_fBF8B zT-n=PI%nIR?HxTD1(yIzLr%b9s=+4b>Nm#gKHYoVsFA4V;p+DlPl_i=>4O>4*-vZN zbC)|G`+J^#QK(Y?3TF-wx(iyMN)ZwApxJ1{RmPPL$-Qvj4n4p~!EdEAm$JCcIy+by zEBadbwXi=g)cNtn-PNPh^u801+CYZcbKd2d7D0f;5d6X52Flm-AK0>qG{n?1}L z6yWCS?jsf;$?z`;F(my@H<$tRFA`L zl>P%pzDY7T!C;QS)wvGsQLggJY-gZ|NLZR6n!lVo7{$I$RbEu@3se>h-19$xVO z=te>%{^=Fdb`G$2HBxYPvv>DF+K}Ss=au-E{QtxGUyJ{RH2z;C1oGd&|6=?HC;|Rw z{Qu?8zk~I!USzbSuq43$6L~2tpO*=G$jP8~R?yN%w$J{V8D!x?R<{4NNE)jE%YPD? z6k<83D9Gvupd6V6s~e0{K1~a~f)|ugZ4wGbDT#Qw)peKIv`R(IRyllllk;j#aS&tQ zfmGYXR4}HCf)PvcoN&@%iQwf!#XH%Q+xKK4{>02!pY4ftY-`@*EAZZ2-lOALoD_GK z1a~+t28(B=`AioDKIUzPTu7eG!qm$$W?%@RhIv6%Vz2t+rTw!0J2 z8z7;>Kr~W>`+C=N`8d27TFIbBO09hU7;~*ssi}IfWgk5F=47x&r(!t1Mg$9miV*l> zNHX{uXJ}~1YBG<{^Y^yO{=z$Q1w}=^{knJW5aXGwyl&qGJG_3P3knL#d@Yf5gBu8a zARqwANXy{nWEB*s_FF8CrqaA)9K0MGgI+7&WGSE4B&%XYmSRVirF(u3Yx3SvWn*KD zK&qlLJ__2Mq=c%eMIW!W`;WcE$3n>mqR}9LJx52b9lXz%fdc?2WIxBd?ijGHqn8%X^L0h z)QKa=OizzED>wJkND}#*`g%NCiGYDeIeB^Ou{8SQ@K160am&j_`*T%laiR-LOY+v% z^swXaq&Z;AjW7uqbD_vL=>xAge7e{|5f3>i6dKnyC9!Y68TIJix*i z9d$ZKwgk_8D>!oP$T~Z-Q%i?Ymq>@ZzbcEyz(?obV`XL@ER&7inypZld1GQ=VDMf_ zF^eEDA&Y5+$FdKHX{60fKHWo4cW#Qs3iX4rnl7Gi2c$5CMWM0PWvR)N<6R!n8;7Za z=OiWeX;V%~Q02&RyC0UVuSmD|_NW?MJH?2@*E{``pU_(@LxD4_M4)G=buM&mZ7=@Dhbc=k8a>KVmyM>&af^g>gfO$%D3aV{?MhT7Wm_{m*)>(DzB# zI8M!~b5eZby-QI)-#^@+mXY7JA&lv%Kc(Az`I~@4$-O(7|0%?>{#cjyd^mylO?CBW zdEL&OW^WH_FmgM(RdpSREdt|McEXRIbE~hVSiD3Jx?}M|3Gb&&5rgikYYho8BBV$O z+y!_RBTtfq27v8<(5ByXV-(zpGMRY+h*Duf8LpQC6pZO!QK`X{l$4WWvN{hW2VVEr zZWZyC^$K0sUN525C5;dgtq-70=6koh7dRwbPa6I7;Kca};k0IpM?)ndXH{bQSP>vvPQn5@zo8 z6B{BxR}6ru{6|>RI_GuPXU68usW2&&p@1bkelYAxh`jaz$-j zT^oreP8#~921|Sld|EW=cPZ~5Vz9}F`a{Q}XA8xm_h7Q^jWxuYRj?VsBo_A^y zi|tTWFI)H+MK?c{JD=z&RlX6($^2&tT0qIp!h*h^iN4<5ucPUIx-lj9!NgvM&VkV4~T#+`w z8_!zLx$iBynJ5sBhXSi)dX_>hn(=U*45H_xSM<6(eI3zOCFHU=ki%`7tZZ7p4Pw<@ zEmkMysRFh*nE@`f@Q=quyAaSq9kH3<-{GwwYeTL*{BVw6rlp|Sp*3!J$;ef#iJ4VE z6kA}Dhk92wQ9{ncbD4R76xnC}g3?Hfn7)A?4W-mhPK4w^sP!sNE_n8?i{y0{ZG8%W zVQsotJTk;esT_c~yoCKskFnON$zD&8`^2U`d2*VS$6Un1D~;_%kpENfI>J`>(BByD zD+<1hEwp*27nl!E8o~W3AULztXCu}tWB%I%n%1uB3hQ*8-s|8DkfhJ2>Oy?J+TFI# z!lK~ZfnyRInK9vpYz;(Cjyh1Nu-Qn9iM_Ds4|AKGSUgOn5w|*+t7^NpF7XW6F@r#8 z5$YYp@M8paFBx~K@iP_rBu~1%$m{#p@Ql&b+sHVTI4o#eNS14;1m^skyo9(25cP;K z(Rh(2YhgyP6X}S^+)}e0^Y2gQ{bB2iCzHgfr&gr+)9YgK%8llQve=|c0rymEv;9T5 zcQ-{Dy1*NZnLd?|N^NcQhp)d@bZ>tx!n87c#SXDSKimA1M@0ZLAL3!;c%df>~Cpp))cB30+AMd3T+-6tZ4li18@l6EbokAOdjcH1iX!j`(lQ4Sc7hWKQy zjTkSwLT&LJGxDzL+qz6v1eLl(cr=1G<;e&wHUG35VI}b+R=*2V<+}q)wajsgenU_R z0c(ZIv_TLTb7<_zk?|&DwW#flsiBt_9}_F<{RhdLST4fUL*B3?dvy?vf@~WDV~QvJ ziUwqUow}wwkOe%troU!q11ZE&&i(fLd~AAQz44(qAXy6e6#_DHun&(n=U-DFz^V9b z(AGMA-i88Bho-uG4?!X#OKoqUjMP&g;gUd{WxFxUg{V=jmv06C>`2L&gVDni2+nE! zr-x$fL=Xzw&NT`LIfFEuGW@?n5~z#uRrDj6>UirAA!?6bvP&#%brpR+JH$XJ&Asal zF>n6KCZ3+o4s1*S03gvnC(dIt4`}vFil^ROZ!2PD`8=X)S!RoRm>zELW84n<&Sxv%%-R zHR2C@A79sybPygk1*`h*^_; zd3Brk9aaoKoi;YjQ6DP=(+sBzD|a?ojVKu<Dw# zebyFHlG>nVXS%FxvnBX3U=p7?#$x!1&hA1jJ?0JT`4Ky>z=*Mkx?;11_b>Rq6qXgu zDy|y69GZ0G?S~0=aeW)13=2xl-WG^6GfL#oT(ehK28%ETGvp*|mPpjL3lF~diurN_ z=yG8CWl)GxuLX4+f|$>ow1XM;5?jY<`b)&zK8D7da$wrp|58DXPiyd?)%huk*qthj z>-IlYVnf_~Lhk#S&a%WzezBb-wbE`xYRuJQSuLO3`ry%soP z=5SK0)WJy-Mu!u#sm4zIXhw)9F89?dd8m{97@?}Fnp~ed1RD9hGB+1BH#cWgFnn1r z0ylm3t`=pHtMyfRKXFGW>W^JZ8H@*=rt7&$Ns-w6Riu|L5U&fa;p5Rp-G#OT)sbTs z5H-gySSblYFleNy1dsioGtR}`{qy1nQ(uZdx{q6fsFV2uqeh!01e)Sym$aM*z_*xj zihE!Jp?l)LVTRwTRa(8xIo)X^m*1`maz^?7eYebjLargnx>lk5IxRapCOw^yytU_@ zNlyS@y?M7oi@YVK$h?YpyeY}`EWouKrRFR?>;s|3m-JLU@|lEK;r z2KFuT;h1k%nl>L9aVtAjyIWCJ_7u(0-v>Pq_uA9xGTy5h`O?+&syrFQ7k8ImPQ@$) zKs)X62~^a|pYWEdjxOm7dZ=2hy!_ueRzsZyW zo_o_Y$WuDS@h0(*v&eHNtG^obEB751Sm3FIeH$kpiiE;?%*3hW-SM=u?5iO7o)yRB$D1>V zyODjH0n{Pqd(T%tb<3qqzqy@v1HTzMv$D8h)8X}f>OzZ_9iyTRtD2wJhxz)_3+PFI z>t|car* z+gw-pVK9aD(}cD!Gd>y`xX8o|SLHPc6&*s*boRQrYRr8bi->Rgi_So;)GHYfYa_X8}%6)0X z!be_KtT7)QR&=a<(%06G5OACZ5dZ_`T+HBlL7WV7X?`1KCl4rs(IT!dH$LG|ahAYP z2d4?+b5%sNJ2$F3*|agK@LNB{@p7 z<3#@C>M;C*?9}dr+Zo6zo8cmQa4=me{WQdz>F6fzf1+S+PGh(68YzQ%wS6Q|3nY)- zNB*{u_szp8R3bx)eBmZJ&;N+GfB$Z>y(3EA!-uV2hCzbKtwQP* zbM&>Fn~fR_2XAs>`n2L`nf2L(4c*1j2Q7HkTy@|Z$i2X@7RKAoY}XA1pwSMjnYrJy zd@1Wn9m33YCwJ`~e|K#|+er3do zoR~*xV5lL<@6MVjCaE`%+H1|al7^C{hz56B;8`r8)et3m-iW|6KeRkg zKS9V>Acax{rXz(F*;n8)lDRUwCBbM#fx=Q`9bt^5l#xv3=!@hCYovTEioYG)XP~#{U13zh4ugd literal 0 HcmV?d00001