From 3e8a0e8c88dc00eb92520da72d8e847ed04cf08d Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Fri, 24 Apr 2026 07:55:41 +0000 Subject: [PATCH 1/8] mla with hd128 --- aiter/ops/mha.py | 4 ++-- hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv | 6 +++++- hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co | Bin 0 -> 39168 bytes 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co diff --git a/aiter/ops/mha.py b/aiter/ops/mha.py index 5c9db14959a..434c0fdf18f 100644 --- a/aiter/ops/mha.py +++ b/aiter/ops/mha.py @@ -1293,7 +1293,7 @@ def _flash_attn_forward( swa = (window_size_left > 0) or (window_size_right > 0) def is_fmha_v3_fp8(): - ret = get_gfx() == "gfx942" + ret = get_gfx() in ("gfx942", "gfx950") ret = ret and (hdim_q == 128) ret = ret and (q.dtype == dtypes.fp8) ret = ret and ( @@ -2074,7 +2074,7 @@ def _flash_attn_varlen_forward( swa = (window_size_left > 0) or (window_size_right > 0) def is_fmha_v3_fp8(): - ret = get_gfx() == "gfx942" + ret = get_gfx() in ("gfx942", "gfx950") ret = ret and (hdim_q == 128) ret = ret and (q.dtype == dtypes.fp8) ret = ret and ( diff --git a/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv b/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv index cabc8f8b274..f418e10cd0f 100644 --- a/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv +++ b/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv @@ -6,4 +6,8 @@ bf16,128,128,2,1,0,256,64,_ZN5aiter32fmha_fwd_hd128_bf16_causal_groupE,fwd_hd128 bf16,192,128,0,0,0,128,128,_ZN5aiter25fmha_fwd_hd192_hd128_bf16E,fwd_hd192_hd128_bf16.co bf16,192,128,2,0,0,128,128,_ZN5aiter32fmha_fwd_hd192_hd128_bf16_causalE,fwd_hd192_hd128_bf16_causal.co bf16,192,128,0,1,0,128,128,_ZN5aiter31fmha_fwd_hd192_hd128_bf16_groupE,fwd_hd192_hd128_bf16_group.co -bf16,192,128,2,1,0,128,128,_ZN5aiter38fmha_fwd_hd192_hd128_bf16_causal_groupE,fwd_hd192_hd128_bf16_causal_group.co \ No newline at end of file +bf16,192,128,2,1,0,128,128,_ZN5aiter38fmha_fwd_hd192_hd128_bf16_causal_groupE,fwd_hd192_hd128_bf16_causal_group.co +fp8bf16,128,128,0,0,0,256,128,_ZN5aiter24fmha_fwd_hd128_fp8_gfx950E,fwd_hd128_fp8.co +fp8bf16,128,128,2,0,0,256,128,_ZN5aiter31fmha_fwd_hd128_fp8_causal_gfx950E,fwd_hd128_fp8_causal.co +fp8bf16,128,128,0,1,0,256,128,_ZN5aiter30fmha_fwd_hd128_fp8_group_gfx950E,fwd_hd128_fp8_group.co +fp8bf16,128,128,2,1,0,256,128,_ZN5aiter37fmha_fwd_hd128_fp8_causal_group_gfx950E,fwd_hd128_fp8_causal_group.co \ No newline at end of file diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co new file mode 100755 index 0000000000000000000000000000000000000000..ec3157afc17d5d5cc610f781d71b9a0a4d1ce9d9 GIT binary patch literal 39168 zcmeI54}28Wwa4%7CLv@Y`w1!5A?>M1-B)01*KrpdzB1O?Hzk*Ds2S`AvY^|g8Z>udX7eXVV&?d!hZojVg|Vdc+` zzE6E`HlLI4oHO^Hd*<%lnLBsx%tU6+onkzZTTb} z|362Dku;cTUK?9|4x@@QmFY@4YF5isOz{}6s4BCVPV*rILR9-5%uW=3SjKdfM3axk zdirs;{`A3k2AE3HKc_4x`OHu+Cf{IwCHsP3hTU}gF19?i&hJG&THd98XjyW7(tzpb zEDmHdvr4t@hrs+A)>%P*Ucv05`v_^;@2YZFgj_k5o?x}d7ka$?=^R(EJoFxEZ{45c zDh=1Vd`?$ssV5ZL6ZE*;&gDUG*n>50cMvvbDBvpf>?-&9OI*IEbE;ico~Hxhpws_M zj=!ud?u4oPM@b*t%c$wyl88|tuudx&ivTh(H+|* zeT%z7&V`X89<()ZqD8NRwR#=kczMa~ z0F_X9rV=`m@DNuCoAf%^tk=PoBpjd;>}M*WD+v#Am9SH`TJI(;;VhjZ57j zoH&q#i?~{NQLlp@y$+-#9PAD`E93TzC!$GsiK~d0^*VS(uY(gwIKTm-QoCz-JqaIi z)$pcX2k+>0@Lm!QdmCgz}EaCQ6sXvsHCTkMD;%j7#ZV%&hdzg@f2Q-no1Qn8-gsb=p$!c}=gp>HH2Ij-Pw{wI3AVKLD(dh5eHszhog1+=LD3fx@wf!pJ~m=yXS6@07LT(i z&5_TX%bX>@OETx__K>gJ!?Ywk#8t_et2igkwj`Xz?WN7SPbtlXx?@P@lu}bAm2s`4 zNOw4kbbDB=+r!c%Jj7Lrwv`ko;VizD=sv+TSLlu*nG;M+mDI(xl7Q}T!n!@w>-Nx) zgon5)(Y6vX31{)GMEB{Yd9CgkF6nghXij<1UmL*ic6pVjI_xa-)_dGedTIUp#4M7- z@&q-^OdCw#@_Eav)4R{H(<~>=9X&Z!u6k#g&lPqq_XjH#=khwjW}Lk{*12T@>Y9*HD{i zC05Khq_#r`mpyTh@YDkqEK{K$b^0OGSYC2b+j()RP#es(7+;c2nYXEytL61juD1*$ zEeLXO9%{_g8jE3tDJ_cs2s|w+yRf81tyE7*i{?@eF-MC48(2*Bq2_1+5TT1;6zl*= zx;fefh*%#5Nv1jaI$#3_z$olRs5B9JF&G5{pbd5rB%gyHs)JorPfd#wY@!He(M?T5 zTwoh9TieVj)>ul)O)+dw%xq>PL}n{Vv-V-xO!|5)B-)!oj`vzp8-#5nklhrh8NHeW zN3RZs#`c81>}?@|(Jg_RvCXhG21D6Bp$*>EiftPSW;X{zqhrBPW)>!mt|8&<)^P3E zS4ixQp3ra8Xu77rnz7Tn?48@l+HCvU$Gp=U?HlYdd#7k6#4g&!>5Yz7$77E54m-4t zr@XO_ZZByfq!D@71ZuJy1EFl{U&Q7$BDPqdCew;9=xEImFVbYMh2C@Ih#V(P+kv!4 zp<{V4E~>{4AMuhD6Oqerv3JsP?GBU~o83wM#b(bIX}Qyz5l_s~N!v=yQ7F_l4 zwH%8?+xxDHT_v}PIruIS61h5dwXCmG^pUZVvVKC*Z!$}fY;$i!B)fZKCW)yq$^ciQqCox|+I#7r@>X_$kNp91@2M`xBjOT1jXyeZ4U z$Xo55Gp=)VI_wV7cD-F@x+7g2Dh_SpZ%3YdM<=bL8K@5$k7F#;Q|u{Xs+igoBW=B? z0x?F`fo3u^HdNkr|g*2TzrW&H+ueP%3EUXR$;Tziwf(~k8q(y64`D5tqRcDbAe zajv;Go2KbRnpP#v8aYjBELBc}7}mtb+U3~nNF#r@t@ibDn`&{am%rck8ZX*zb_?3` zn83!&?}avc$26n8b$*4l#kjUGOKuAv=ckdicl;K@*^S}a(QDAwzpu3QrY&C|E9Yyp zx8je*fO6y**4Ln|2D6(GgPIrl$G)!S?fS;p>8tJQ>F=+HYp z?NP57cwZ0$zk_@L`4Dm%@^=V{h=CMHI%F6m3o;xs0x}9>g^YoWgG_+rLh>N_kZBMb zqySO~nF}d`EP^bCEQJ(9+>w}BjKpG*m>8&s-hlS}a_{t(l)hi3wDg(ChQ2Q~+1J~* zZ1Xm4^^T5h-G<*9A>Xq+5qZ_JIWkLZ5J}d<1FsRYI4jm5(tQQhPbfN_hi}uaAI@~T ze!RIQGCnr5RgA@A?^%vTno&nBfv3j?E^F`mNUF(v5kG~Gq)_II_mNnC%`@Kq(6ipz zN_nS9#CcA6CqmrA_xELWa|H;0T%y z97WTC)`FQSecOh%^hLhe-j_%ApBg^YY?kBRqr@FM6E`u-dB#{7$EkTnjqFdM+d+6% zZ!~V(6>{1M#f-T$Z2|m`fq!|Mv5%m2Ot(q2%4vspF^*8vP8fMbs_ge(R^QZbG*?bL zYJky7(=Jltmi^)$0Qz0Yes5s$b}`EF(zK&SE=bk3gXlyS>j0x1t6Pbc)%DBo{up&g zOfo-D%rZw1i_B{xFSqy6^b)PE4UB~w8RvYJaqd%$^S3b;?O<5e$rx=&L&FK3vWA#rN13zaB z{u;CdA||jLre<#|e4kcq3<`$ChiFEY==NFp_6Ga4K)o zz;%;U_k;I?i^0X<5^xE)6kG~E0zLv3gT-J8SOU61H&_OiffZl{xC~qd`amC8 z4OW8zFaQR@AV}L~9$z6$;+F}Nd7m(auNLz8fN(n>6sGcF;SRn|n8w!&ck(NR>3oAQ zgBJuFFA6;0B-r^D!NIo*1^ilJCf_E^;@gGU{Cc5~?-1_dJB2y?CgE=W31Ke3S(wK^ zDa_}$2>0+?g(AL7xR>89EZ}zv3;A8bB7V1UAHPSqpWi1u!0#6p^9O_n`9s1I{zc&- z{;;r=?-3s6j|q?PlHlZfgf&BDERnygev|=LN))M;OBoL1o#1=hX0umZ>~OoB{?Oo8M>Zih^T+yR*exf3!SG6Q0R@DMx1 z0V#mYgv^4>h7>~Xg3N*34Vep>2bmAK2T}yN7qS4d5V8nzALM?>1CYg#2O&!!4?&hf z9)>&uaYBkAE=UQa6yk<>AZ3tpNCm_TSq7D4QtTfYlF7tL77Hh( zrgGAdA)GXHC?|0oCuL@G(q)%%(&d+P(v??o(p6V+($!aU(#VmV^o1{Q(zVxe()HJK z(&*8gbmNVjGEw^yet+#U0ZMSjKq)D7KWeO+Vemf`K zaR(>ec_$~$n1Sz(=Ol-NlV;B3q}j7M>8`st>F&EZY2G|ey5}BFy7yjATDXvt?z@kZ z9(aJ09(<6K9(stA9)6gUoK8-1xj3n`l#@IjPAV_QwyZ>3z;A|Z@6W3=lRVq@{>gqb znFKvQ1VW#>+)O4zpY|vSeR{-9ra-qfgV613%(#1Sdw;1wx;{&5Tpj?fpeNK&>!hD zlNr#9zXd`s`L3DRpu1lKp_l!@j6?4B{)$r|^kr|Gi5;twcUXjj^)h+)Y+?oEv9l$GI=jJT&d775%AQ>^L`} ziyh~3y4Z0pw~HO;CepMm^cd>BBF)D94!0@CPn&Z5w5i8WCGCV2SENOi<7ZSkenyq! zXH+?UQoozzco(D1?hw95n(1%LU_ahG0C$Tcd_n|cXLfb;tCn;t; zxBsN{QRRM)i^bc;cy9km>A%YTry3Uj0OPs+Crw{dEM>8FF~;?uq9GRF0OPrRCSGn! z!--wy=ZQn+2vQ(3zR%>6ZKK2znb(P>GOeOprqO3I`b=i~eI}#N^v~C4V#m2L zR(71b$;ys%^PUx#g22iUF^sB-*_>oY0w7Ax@{@eU~Q4rt;X>@%_PrNMEgxIWWa zV@|Zs3}zvzv~6^)Aos#@N?eHP3n93O(rIx3za_F4@rtxs2RI^ThZxQ zJssD2N6~YKG5uZW;fox*Q~T(YTA|~ZblmSk<9Th|S$$D#<6&hR4=dYvSlPzI3AXWk z<9u|R)p0&r2j?5-lj}zr=cDy=zHvU9)_NDKD_VEw8|R~GbLX+Nv|h#Yjq{;hoo{?E zQJW*m`3f<~{ZC`O595-?c%L!eXN>n5<9*ukKJ^;YKl8XBJ$@Qver?8h-wpDyJol3t z@7pu%+&`UdKt1uMyHdv2pE`UMn`Pof_9pjccd>H?Ex;eW}mR^;0^Y#>Qea*G`pjHDeqw zethae4p=ZgrSH|uv-(qX?wm0PO`b2u#sKC1>!&gYt)R6A^EX4@qpO=&(SFvq zy%m_ttkEl%VN6z|SNTY@R`*wvwOYLa}6APOC4kBOO|OMLl_3t3SGu zbZYfg4P>KMj|gP5R&Nx^*R*R;bNUeM~>wvryL z{*5kjRIBgUPL65yr+1Q$u@`9_WXX? zR;aZdJRsZX@a~|$7Y@m`Ia*uyi?Z!*t?kHR*;b^r9qo~Am==1*-%H12+XAiaUnJSK zP;2Y!m2Hc)w&PLR_Mq1Gt#8VH&Aj%+K|+P;T73{Y1tt?ji}Wm}2X zcJhR5E7jV5fEnA!TcNeRftlE_d9}7v{jzPD*7nw$vaM2Ud;5p7Eugi%`;Kg@(b|6e zBiRXj*7nO^$hJqdwqO5>7;|8aIk3hY zSYr;XF$dO|18dBI{fy3m#W~!A#vEArJ|KFJlYGwh366iCeczCA&Sspm8Ru*vuW`;6 z@*3xCr`|B`TQu%lH11n8?prkOTl{}`-=cBt4fk2%zQwcVhF|E~+vvZtYj3|#JL}q; zF<+d{v7eCWc@Y2G^TmzxJmWmiIL|ZA^XT@c5F83cBd%9;^pff-AuWumKc60Te+I zYyz9W7O(|u1zW+j;99T^Yy;cDc5pqo9_#=+z)r9e+yrg{p8%f#H-nqOC&4GdE#MY# zE4UTx0=vNN;C65)xD(t3?gDp%yTLu+9&jJH58My#2M>S;z(e36Fia}?!@o6?TCnc- zW>N>%|ItjAgDd}HCiP&$-^^qMD8$TUB`8`f7P0}{gSY8)fcqY@kjKIOP7CP- z50qNSM(|KM(sz%DZ0Now(%Ef|Z0x>4E-$M3Z)fEVsCtJ)C$k2cI41AH1 zV<00GrYiI)?*YnCbb1dN{ru@4F+b%vWj%d_2K~%cbf%s8Ki6Ab=?PA?^79I;bLSRV zC+6nh&pPfFpTD}?6U2Ki!xh!8D!fq1?+jPq1x>*~RmfTDs&+^(e2?a}U@%ieiZiZ!m*sjUmra^7$y4Gg&3EOyrcBIpyGm}Ic$+(SvPY?tGtr+}GRccWNR-|X zr}@xFec&ZQq=4S=r}8?CetK)XozC^7>C?Bx+h3rnIh~w77;kq&P4u3qHrNxrCyL}$`@VD!fEdMzc^Q-yQ{ej9ca_Tew)%=Q?jKl)#zv zA1bX%LUp|wzsl>eo*tpq_3HjgrMkaT+{oLYE>qI#q1&bASKoW4(uQ@~k4>v!`4$Mx zukKF=-={X7CH%0AaqvG)tocmBGR;f3fqvBA-)ZIdOz%0RA9a7M)?XGg<5~CYJ5lpj z9AYafIvF=-DhU;wRXm}OfTXtE_+&)Q|NL2YHNVRHv0kk^ZPF1NnU=34q+!r@rpB+f npGRO*V`n+o|DsKd>h`Pa)N)n50t<=qN0t0|pGDl~?EL=&ZxJg6 literal 0 HcmV?d00001 From 8076e5228a579e37a6b25605c38859d4940623d8 Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Fri, 24 Apr 2026 07:59:32 +0000 Subject: [PATCH 2/8] Fix GPU fault --- hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co | Bin 39168 -> 39200 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal.co | Bin 0 -> 39528 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal_group.co | Bin 0 -> 39536 bytes hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co | Bin 0 -> 39464 bytes 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co create mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co create mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co index ec3157afc17d5d5cc610f781d71b9a0a4d1ce9d9..ced92d63d4be686efa87f1a34830a499c4774c87 100755 GIT binary patch delta 579 zcmZqJ#I#@&(*zC11rs%w*Gr@`fB}qV5Pg0>PAblmoEf_IIjI8a&1?1C zlQ;jE=)frXBpB*0D8&P&*%>k>2Tm53d;|40OpOv$cEjXGpsYnGOdLX6KxH!~Z=5Xd z3CJZ@=NvS#U24-d!P^OWyt22~o z?C4|wWg0pf!xWfUz}1?XnZVSVnHVCNM$QbA1vOPSdrlQ+VvE?_blO<=^Gg?fJ1d;`lXU?$ad@u##yFEaIu8Y7GDY delta 550 zcmZ3miK$@|(*zC1hKZWX>%XKifB}qV5P6K{>lD~f=Yz#*xbaZr@Q%zo=Y+d$hyrElN=Z&4+KN4hEhCWnw`OAa^Ylg$rI_Q zvPw|dp2-V=vLd0VvKDZ?7blB*X+S*dX{#K3}K zvY@8w=FF+$OiW+WCNoZ#o4jDU02e&2CQjZp-ApiJ6U22HlOXhr$&54188aqZ&ah{c Xm|O`Y119gBVb6JB3PgSn&>(pL&s1k^ diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co new file mode 100755 index 0000000000000000000000000000000000000000..602860715820b9321e676186a0df82ff7d42e55f GIT binary patch literal 39528 zcmeHQ3w#vSxj(y`gvXK^6f0QNMZwUDA!2}l2s@kb5CJ2gBBGm3c9Sf5>}Gcp0*2Yi zgAgGF3>2wSiGYZJh+-h%++?xVT7_1FR&Bj)ZlB&hZn=G0+uObWbIwdM3(F%rdT+gV zHoq_b^L=y9cfR@N%$#q|IWrTUId7KPWU?m_{`MBhtvqp8GL>bMf%yLg zGK8c+Ov~E&=5sKjC{uBul8=hjIu%h2*l*&jiX@Hmv>Z$zWE|(ov{=k5=Ddp9a@5wm z&C2$}vFPV4IW$B`DD|1aUP?K&yt;o2e;#tv{kzoqRG;6CaJm^2-t^|17Q)o=Rt#r$Y5YegaywUU2IyU!t=-u6Fdo)~}w=KZ@-;fk*CtSYchI@G*8wM`1T<(paI#Uaf@Zx6&|mhu9l!~5 z`kb&S0Sz%u*rHd#HoXeAC!hdMaP&E$EddQNPS~wi!Ct)z4kV!9>7cW?(xqMyP991? zMT{3-(yO3DuL3y%1$%?e(wHmb$w&fPVjS_RUInk~Rd6x^1?V73wU>ss63`LjhIjNT zcweuA4--(p+@QHQ^dz7o#tlEytKcKO3VxG-0$dz?{!s75;Y=x-|^ctzk$48V=y63`Xvkeetvw}yEMXh5u09TRJvGUX+pCe|5?b!%9nTf?#hG$7EZ z*2G(Jy$8b0ighQyBCA+?z~F`4N0RuuO&w;o=5f1BG! zaf+Sy2T@l&#i{xdEFK%D*m*5hE=C7~YW28_()4`hOs*{POOiQD zw}x!p8m1+nA;u+rvpA>B_5_s0T&2yrZz;`lblZ@~Ev3dKr7^7}Pq#UXb!%9nTf?#h zG{m?>+e-2iP!`)tbl+f_OLW_i$PK2(CDk#l#IM_&kZuh%x;4}$pdrR3+EyYZpe(kP z=)T=FH|w^cU$>jbGKvGfDnEv|i_1L~A!m`d#^ZLoNXg~#n}hroJ$u>bYVGL;MQA;%1d0%q7`mu ziTjob+0G(=wzJSx6~yAV;-Z=<6EkPNoZ%06t6d>auK|4)l9w_9o=U8Zi>YWc!XcAs ze@3v_A8;1>swzVH&twEwmKXTSJ{aGhsxwO6SS!@41ydl;XRL5ldx`=+IEwDCU0uaj z?Fj_EzKT|Li6LJIc-DWq@p21R1FGvOmF*e8p>zeLx*k)J)BGx3jkJxBG#dm|R`U9v zDrY`*{4opCl+>k&bX&}?KRZnNlNsKLsptZXq*|xs8G8P_*J0CjmLa4OJEgAkJb&)b z#}-r6)D(@vR#QrZS!_;;u%)k-l!%QAl2Rg>6bG0ic>p^wpYnsu5xVA80xtn0z)e6h z&>U$4NZ1|$l2mi#Er1=^1B^hPhlU`5&j&_;exMz42}r&GJ(LHwQ9d~(LU4#AU>ZGC zDXsD2y=xTN4iZRj2n0t(1HsfZESg2;2cj;NzmvJv8t9Mbfj-yAJoiy(z z0Y}iA3bu^OOi~4)!*o7rS&>;P-k>@E7cdBJzJvnPH%uc(VSL$8e}w>xIS zHh<81+rEw&9m(0^vE(fAYVt&ON~!fjaySZ;A)l1fn&wE8hDt;0 z({eZpHb?7>Yjaw2966HxItRmnIRm9Z(x7@_3d&^Xv{E0>b^JHre_$-dAL_S%V+<=-=V-zow*mDq8#Yv;JqOb3VS`#0<;T3GmTh}# z%=9&m4fGuCAX#sBtRaM+r?kC~@mHlDt2*YD{2z&u|2LpBpg)2>2K^mG!jeA;G!Qfd zlm;3G8V(u>vVlf}#)8IyGC^6OY|u229h3{21DXfQ11$zE0WAaNgWTb$Sqev^;i%-V z0bh$Y|7z#-#-y&_CN*}M$i}X()H^mfw(s!PKj|G6eR2oRM?$`1*&2SsvMoGIs+CCE zgFSB&votGOE79u-;yvxanw_dpq-7(IUjx}pcj+>IY z95*@EIHEI~q-Zqyq2+kE0sd?BKRw2OSxeU^a%JjEIF&w;gQ+jwL!#Z4&v?6o&w6Jo zwwxvwX|~3ecNAW=rFXkx3*Aoj;tCA<#|Nj-L)bVDw5KtCNIu6jZN486GK}T}4yXBm zBWXUMEq7*8*PcO*UEx<+y0XZ@)58Xt&CKq7irvvZyNQ{V8ExY@RxLAfM0XP14`SjdhrYeJ&sK8_1xYzj?7+~cjSnL$=Ws&nZRxB;mB-tE4Ff9zxu%s;X`6# z_(fu7IGk7*ULAh5rHketH=;FJvN736X%xe2rRy2mq#GDspG+j>_-$kuvDAl8kkfBx zm8J}!ILXH`JIHbB3Xanr<~Tjf(cZw(v6f@*dXBR;a-6f7f)IHO^yZM=IDNtW6}3GmYn9e{5_6kKjc{PV~+lx zat!ciFSYZL}kNO4)VrZ-)cZ`0r?3+-ab)hHPOE5>UxW&H>H^&IQf`&I2w0 zE&%2M^MDJ13xSJ)i-Gq7?*%RaE&(nDE(I19yghnw(XcBXUW^tylPMjsQh_i(a;v8X=73Wd&PT%ed4{s0r5WJptwXh zB;GH)C@vLV5+4wbh|7cy@j>CZ_>drrPN7rG7b2odcttD_UKI<4*F?AQy66$!5Q~J9 zVzKaDu|#-F^a|bLa^W4ZRQSGFCcH0}3qKGmgbziZ@FUSL^oW(hPsD)mGchRqLX0%5 zOlIw*j<*vT?L;;WGM%tU;j3#_T4kXoOxusVdbKrc^=ivR&?L}gP&Q}^Xe#J7&@|BP zpy{9)AUj9^IY2p}T+mF=EYNJw9MB!0xu82i^FZ@K3qW^)@<4Zk7J?Rm7K82q-3z)8 zv;=fNXesCc&@#}2poc(CP(H{7DgYIN+#nCA2viIz0eL~oL8YKFP&udqGVC?QaIlzx$8D@_(KgEdSy6gXKT{aj^WCKM$7w`te}-Z=Vd7|LgCA z<>)^K%O;amwpgrkaZm`N@##rU_bgTU3FI(lCZnDZZ-)xn~kGIOV++vk) zz11pDoM@FNPqxZardZ|MZnMg_-)@y>%)qe|ta46{Rh~K1D$ky6mG8L2D&KjhRh~cJ zD&KXNRlfUftGsBDRlesQt9;*mR{8$>t?~m8Smg&Fw8~DWRd%_oa$%uW_IRvvaWVE~ z71{#M8M3oGtJF-g>^r+B`OIV@`0OAM{L~d@G70>&hk@Xyhs|U%czXj7yko5yPhalr z&Rq`#KWn3zOaVV_@PS{NaXnqvMPq~T>n$v1qI`(b6=rlXx@=4yOY~^KR2$8_j4I-yr0W#|F0KyV8H!mHyMN z_Mb}LaVsxRi75SNMCm^xO8*&A`cJBN3$u4&O1@%mzG82_VsF02UYd90%7rOCioHFG zy*-M(JsNv=GkaT8h`{3!8^__{*IK&hws9+4Q|x#1_#>aAO(MBRX$XDM z^q+R6|Fo<9r#5C%`p<~ce@2x4GotjLF)@>3Z@yx0zG82_#@^nTNwK#_vA0LDw?|`d zZ_LEUmwLyUVq&JV#++!(YCFd7KIyDVZKva6ohwq?F)sFQ<#XP5gU@^4t$M-x?ux_SIZFKY zeUc>2(TwZ8r|@(hfR1^+uke}leJ9{Q03W)G@5{z=CKFE-Y<&U#jyihVqy z?Bfw-ACD;ecqGm~UT6%E?z1`uNPTdjF+k=YWekw|=R#wEG_UOr?knoM3ylHNyqWWP zUg}rrLSukva~B%>i`O2BaxX(nEH*X90x@Q3j0GBFfyP*%F&3yD3smQy{+Y)FY5!@A z2d*>50Zh)5o%-13yl%7_(~ZgWbxGn#t2z_rHM@=7Z@XC z@s%cSO?Zru--moEy}_tLfDUb@t?w@vKF$q1Q4D|_w$ zljUe-&mUy6Ia=8ZhnS2G@Am3D{34Uh)ympmVzN86vZF_sEKe&t*1=?0GTNu_<>O4Y zP%HZvnaLJuWnG<2wnQsC5n;0XwX$!%!emRevTwc0WDjU%-+qnB^0l(>;OPbU%B7XP z`392}Xl18PGFhQk_C2ihM%far>}{;zhRmy#o$h9`tl`evBvsXV|}c#KGs+t`#D`7i+i~HjrFnY86o=o6T4^o6vw~Mex}H{XEW~EjC;1A z*SKd3dX0Ow({CHkG#bw|8qYKu&omm(H2yz)rqP&t!!ww8rtz$`;TM~G8}(N{_x8J# zv*zB6_2P7m{kVAViuj*iFK*oD8TWa{eV%cjN9X^H`#j@5&vsk{rucz2Dm#`03%U4yx~YRkFzc*gT8B=(&)pW=L7yysPndpYA?&bXH|?&XYo zIsRTyW6sW)voq%Gj5)hR=Io4nxzG8%oH1vY(45^Bn6uNq6N;ugm3M1*;=S6Pm8&hC zGecqe&Sa^Uzccwb-kE$H?@YcSRAoJGsSbziFPsilS%y~wtARDZ8sIA6DqtyBd`(J1Z)B}1Dk>Cfa`!Qz!u;J;0E9(;3i-zuobukxCOWsxD~h!xDEI? z@NwXF;CA4Xz$bxiz&7Ac;7;If;BMd^;2z*!;9lT9;6C61-~r%4;6dOa;343Pz!!lb zQqmpzwV6}_tAA%E)xerRn8^y@s=t^?4Y2laX0j3}M$KduP_kIa!@&9h7E;~*k+`D$ zSK_MnGvdSTzZa$Bc&AxfZBIwYB( z-$bbE!X!8Qo%u3HtHKXgFj>LcukgfRg?~uFKm8->qseS#JAHiy{Vh{?&h`DD=dCF9 z1g6@A`MI`v^KxwyGBfaJ8#}(tS5fQ<;QOJWk_uNjzEI_JhDz|^tAM{eh|gG6ID=Jw zzb_C%j>1w`F+Pln?H+vEv7oBh>8&X8Ig5M&r>m^Y=?gmj0bh}~%o7|R3>1zpE2}QQ z8It9m!VqI@MOm5IlZz%5WKGB}%JfVsnmBo)r@&R1?aFpdo{;5s72Go6R(IwkkK&U) z>-$nBmc;j`?&*`!6n)JHK0ZVm>4SZm-&HIh(k4gRI`xSos1)zPK$`PS<(T{O&fE=>2zsrpyCL-r-&mOSthr-uJd1g*@K( zwn#>WFXSN^?v)kzTo}nHuBgf=aRp0=@}J5>0g@5$xm_U_$q0FBLM*qd+*=5-r!3%8 zKPF2u3Vr4DgKC6+dMxBB0G6OMPjMt=t^!Y)^Oh-@nMyKqQg)V-oRB*~P2MWZ%vSRX zlhwRg*@Bv!*jH>))rpSUm(oCgCECO-zUP+_KDU>Uk%BB zYC^#?>FNfK>Upb5X<2$LP>ZS82Nmh_sPsp#540}&DjfP#uN#*k{pV=Zr31UPzB6|ci~+FPsJ)$5mv>h()e!}dW5;p`=ypw zzbB`n9ow`&yOu$EJcyQ8udlt|sT;>L<`r}7{ZBJ%eiM$B=VytbVErsF@AspTWe`5aNo+xdlxXU6}tswebykW`l0n}a9R^3R_oSIetNFI&C7 z)8zGU;Y_xYF%>LrXR7^b`=Q52wVjvX|Mz~4qq_g9aQb_?30TIPOz`4m{t+fiR2A|d2=Jx6B{O7-C&&)Y<=FCiF#+;dElgXY!_{TzeiAh1N;+y*x2c%oV84#5DoF;{X+DHN$T+T(sqri<*m zDZD)3+3CG$eE%??WMTTNe9O-nWwpA&j077zXOzPeWcoa!tWcx0CD`COqa2$c)8`Ro ztr}%^f(@QC%G?B*&O6HUte8*Gis=h^)IjYCwt3D#3le1d)IgQ0y*pIm^0|WQb_=RY;i-1|JXLBb6eZzBTMBNS`O9_Y$I*@+*e>W- z+#PZ*@KlG?A_yemLR$o3o%!o@=C4m8|E`d8VP659kdpAAErAowdL69M>j3TL1=j-< zLc!@m*qDTexI);h*TGi34z?xX0EOT6U3wkt(d%G;5)K{@Im@bD>H*=z z!6aP7mBMp+9dzn-ASdBqPsmvjcVs*fO~OlDLAGrTB z2@mLK)N1-$Crw33IEgO`w{8#Rx;^-k@W6`Vbf4>_DUgJd_@W5w_E4wWLwyn+u#5Qn z?IKbVZsH50S+|EZx;?Ze;Q{r+AGeEaOu|WgQEb-jVXJNr+mi6WibA`Kv?t*tz9@F- z_OM5{hy54gA-St9d+5-6HAa9>_t6gWtzOuLp_8W01lc}y@LWf*6>-5v6KwUImkLgr zUQUqhQ-f757U1B6)1A0I?ltb_f`*#otJxd6J-nma!}}NFA-P}fpjyqGxR~hnR+XKu zZoPc#7h2suNRaJQe?c8V)9Q9+&>tq)`gv5lj}m12jDvoPm9*3Cp_jJ$uBe?flLVVS zcf{s_39@}25m%fUaYlkopGU;Q6J+~5BCb6%Vq1bupGU-F6J-0;h?RrUfsnd+oJDC` zK65sAmi!^foTJ-Au5J%glkgB%B&V<9oHW~$a29uzHtRm6G#BWOA(>N3O_5Z@)sjNp z;VjhcVUcbROOo&qS0vh6Qj~+Ax?{MY)6JtjU2XsDKNo*n&4s!W(TVCE0r`k;7~dvQk|cv*lBuI zF2}NMphVZCP}|4DR}OoaWL$5-G8Ouvc6Ev(Gthq*{?(Jng3{{SAQq&4Q4H0tj9Q$B@q#0jAo#z>!;)|!5Vy2c@3~`&%qAYrIT9i$FwWLLD)R2-E&88e+jurxTu!!n| z%uzb$Re~;oQE($j2AZSofQ0o?kffWVuK{+j7mUJQh#y~qUIa$L0BDC@0?C))hw5NE z)l<`=1V51kX3(E14RL{MiP^T+oMMZmq+A=r?-4VbISFBGA!)X5ESpJpmxV;TQpoWx zOKQEiodhx)1J$Eek>IFR!O)n_(AT}qBrvKuP(7vzwuWFRvoo~8yGpTbC&A37U}#h< z7)sB;q}kOZoY@kt8S^rUz1|u6O&U$t7+5`KTDzldJ6V(ISo5HFT7zSQBj#w6Y=k%@ zhcvAruO;uny!CkwXd55(#x{0%Nh2W*$h$gFo!JlwWm5kVzFq@jiv_CFZKz;JtB-h* zCUXt+&Lc-yoHXr5q&*59%YAWCJ$Cqrm+oS6z z!9FpsEyIx^4V8vAX5?`SHb>j^EA!g&9C?!cDhK1hyn)gnX;7mu8F_N^+GrU~M|seA z^2TsI#gQVVN~w)8vbGCFAjQZ!&`bu!2C?ss{Hw3Tx)@ontlz-ar^nLSdc?N+ifbKh z4y=!nHYLp_mS$*dC`*GlS6`7y)3hN?i;`wFOOqN)WoZz@>ev_ui_L*F(^0QjxjA0I z@A^U$U3QGd_d_fW#GZ#V?DubRtY`JEId46y1Fh9w)bGq@)cMhYP3hkcZSsz8LY-{; z2CuJi^<^fjFQ4Y8f$tO8U&5IU;hIsau|NDk*&iCWeQONM*WhTuACCdsVKJ<)Mtu%u zHX;T!FY=FhP0ic>?J?62_S_SmlV#Xpv&E*!z}aktUR1bKvnYfs0zZKbEW0pTn;7u^dW& z?oJZxseZ!S6ME7+ONr$au}HHtvAm(^YAk*0m00L{>K8{~$Uojai59}fInbWL^dUu@ z8QOF|B4ij%2M(v{z>zc^Xv?3G(!G07b9dy0*6ti~;MA}|W;2U>uM&6cblk+u@{G1| zj#cxF9MO|PzXuU`nbNrJ7qhhEia4`r+I;vQ4gc&r;}}EBpMEFF#?lUJ=Nzu49XI0Q zROa^%{##JL(QKA>WG|-TzgP2k+c{ahH0{U{^Ha5TBs!kQ+RMpe zbt|#*vVQU1_fUq!#Q0@mW*kl|jF(4VZ0)A$$Bk%9m2Av5QX0j0rF1o;O}d8h>Qo{r zzuyMNsHHJ-oSb?+ry^|tuJtKhd37>;au`O=l$KBMc?5p{vN0MRnF2Ma+aUsT>3Vr z?>)|{pKu0##u@w-XbD72tX!n-;~jy>@#?0_9ORYPzT1S(1? z2b=@W1LuK-U?DgkoDVJp7lLxt&8mE=~}ZiW3E&I7z4ybA^C7SqO?#gs^zCP%BOq z>cm@w72-6ZUYst7qFss3hUhytrzj(KBKwKmo z6z>t96&DN7iT4VJ#U(_62Qn8j4{|%C5ON1(K4bx8A>>ZTU68vW ziy-$v7DMiZEP>nyxgX+$6hT~&Vn_+Z4e>xqA!U$qh!?UHQUUQnDj`)6KO_LDh6EuY zi2Uy#43>ZM>%sExemhwH{eKLW|MSDa@*n>&SpM^$2Fri_%V7C$9}SlO{_$Y>zdji( z$Nn)`Hkqul#bT9HQ?2rV0akg?AggS(TIKX~t9;Q#R(a@9t9;2NR{7FPt@34;S>+KU ztnydBVwJDB!YW^Nl~o=!$|_%Tja42q#wurKTIH{P)hb_iomIa6daHcH4OaQa8?Ew9 zH(BKg6Rh&2NmhCCWUGAh%~tuATdeZ*>GQ(-1K_8mPF{bn)&dTt1WK4qDiOoTr50TBAMh?z`+Zf^piJ64-<_vMbB z{BXDa~=hu&)aS$H$yMn2|}O$gc+ywJ9-xG1EJsbw3*xj zebFHh`r;#IG7b8Y=RxTAcbmy{=tbWFp%;J8OzhCzuY%A^e`rRNyQ8Q46bOCk+h*c` z?t2e}UiA|*$%7vF83;Z2D>II#YrARRAo6l6>oZY3OxsUyPpXZuHfpquw{zFpcsn=N z#@o5C(L6Nm$mKn$?Yx~E*UsCytajecWw-NoZahufOxsZJ#c6i#cbHviKkZ8UX;<4% zCGEK77pFy)_A{!qpHZd#j4JIX^}CtHJ3p;RiML3Jw@8V%NE0tjJ97E_v|c6NUM1dM zCEi|5yt`PuZD~Z{{Sh1I@W@N8-E`f!XDc)abLXZD|zzAfuNRrC0JInV4rY5JN{36Hg% zGp_#>4e|JTInV4f39K#+BM!zdlRU=ZB%d+9&*WmZky0_^l~M_#O>#3DeI}#NWb~Pg zK9kXBGWtyae0?U~&W*P5cJ5jmZ|BC^csm#0XX5SLxOU#oWwrBmF1ww#bMbv9rTw%k z?WbL7KkaJ!sqHfe2lM-*Q5^t{(Z?7ia zzCIHlU+NoYit97=8*`$4CfDB^G5(>9*|qQVHtwhx728qmCFr6y1}d?<^M7UhZZ#Xf z+sDW6x~kJVXdkXCl-_~yyN|nS(mUw5Sl6=j4vdSvS^bpv&Ct`{H*22pzPap>w?OH? zy+u-_0?oMI+lo%F1JE(AcN9IFKJNtjyU@euIj*Pn-zl|1$3N+q;JL>7)>`}ZQ}G=S zE8p?3@*NK=-|=vQ?|8N`K>D54F+f@dXBz`#<)e%N((*am7$8k+yOoy}ExWUg0n)VD zb9q`?uF~1Y08!`8HujgOJ`&|zhL~92)EEoIn58imXp99KV}Zt4pmr=!z5eviJSIrn zPh&iAtuYpOHEYYWKdZ68y+h9IV_ir;@@!*-+CJvl#t2#eN*N<${j0N#5wiZ3rf(WO z%NQZ+UupWSNskfo^N`PEe2`w_NnzIyX`k4*u1c>X8`o8h>#D|e)&CpURgJ#X7w6h4 z9c$xbF`DbD%9xumMi@Unb*?eO1sEgD#~iy?();zN=)5{(ej1xg$HxF!|Mhd3pO)WJ zjd`CT@6lCFD``LLyWVollh)`JOEF%n(JOtVNvr#-$QrF)9Uu>C^-z$k)#^22vQDcn zt0fz?`tmyRkXCdjE9=Q7tsW7{R;}J3k#B1CrbhCJR$tXj9@XlrTS&WBe{c=? zwpL%amh8~#UvDKlwfctjWRF&VXd`)2t8Z!}d$syEHj{l?{oyU-pjQ9pR`QHifAkUZ ztXBWlHgZU-Z-0z*YV~iolcQRF=MHjAt3SSrJg?QC*iE8Z{mDIaZn&3D_3Wz?`>->@ z=Fr-n+RtoxTHDhHn5{r-d*&ds(c#@be}|rBw%J-+$8*efo7Q&ZFtZhEZAUwq4O2!> z`+NQvv(4Aq{zYcC1zKBo7qczW+Kxw=?H;Y|J1;QXVy*4FFEZP`THE(tVzwf!?fbZU z0cGXV+FpHy*^0HclP8$1L~HvYW_lxUxz_eNW^lvi)!I(=FxyhC?aeostwL*i`z>Y* zXl?Jl!)(=B+j~D^wxHJbukSNkNNfAakD0AbYx}oeW?Qbc{p_dAwnA(B#m|}T0j=#< zza+-|SYv*yF+bLrA8X8yHRi_}^JBlD^J8%icaJeYmfa&n?|)+FY@gxycmI2ejB_^Q zoXt3A3we!mwvg92XFK(}aZjUhPor^9qj688aZls_!+RQyYj3y*6ZbUsn;U+vYj30e z#;?8oKCR!iH)Fmyont>P(X%4{r{{|s=Xu6?o^hUMoafQ&f5v&9ah_*9yT*8Sjq&W7 znKqx{^}0m&s~G2U#<`qv zE@zy}8Rv5RxuC{1JL8(2am~)SW|z!0JL6pL3w|zVT(e8+n%yP1W~Y576kYC8o~_-5 z=W2IVue5Z{2uJ8Mlcjq8%;aNuX7VvSGx?fujrEwNHWIc!b1Gb88D0z4f^}dWxB^@O z)`Rt+2#TNtN?;?{2sVSwU<=p+t^wD8Yr(Z(E7%II2iJoe!Hr-W*amI}H-lTiE#Ovg zEBFZb2)GU020jKp2DXFk;0|yHxC`6`?gn>*d%!*5UT`nCAKVWf01to%!Gqwl;Im+u zl=p;xVyrddXPZ)eyfwX4f@CaE6jo2?eI5^TB1LPT)QB^D9^*Is5J64?3` z3uyq?UuD6oL&|$LUSlCmU|S~AcYGv9IzAQ~J4mFd!yJjKzUxg=49i8hkA`hmGD*$I zv-vs;-se!>v*iX0X#uz1WWoDX%6lG}WFc$7Z8uxUgWzM+EqK36c~5(ug{%X2%tHE( zfsxf6gClD@G9v3bE{;(DlIp*iZ?9d|*O;VMRark+(U&Ux^FLyKx>%sBr;pE|AD^Og?ezaS-l_^uaEeWs zn{S&lC*L+cI}3ldu{ZeqRb`$a-X9t+uX0u5-Bf;OxEwFO3I-}ec#Tz+GgK1@_=91j zD5-Fj;l-#}@4>4bi)+f9-l|f+v(z7Sx_myTKjaJq{iR-?Cv-z7SaO5US6g{KEK5Bl zVW!whbFyr}E{|dD8UmcDB&>5AVmoYhwnI!b5!~w&2G9MDN#n81_W(*CJU} z{;-E+xtCYrMPVeXtg0rf+!ZP(%71DP2T4}Y?{^3)KjH=~&oR43;A|U*aS_SFy+Eym4}Nwz8N#F*iq99G^d4UA#${k*lT^CaGz2 zas_p9!s%=ib0(!su4nqhuYNq>U-o= z+D|8COa1j5V5Rx{jBh0-q_B*$?|;7lCMbF7cc342zbxFub$TBv{iw%jwfyk- za`u^q|J0`Dk1G3hRLO7W2P~c#|NCv8(8oejTXtUxzM$rRx}ROmuaX|V`pQm|Hn5p1 zxk|!xsI;D`@vHUce%RF5c@F-6&o?;L@2{>?x2x*qSV)wA_!gexEj6LS{`vnKD(+wJ literal 0 HcmV?d00001 diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co new file mode 100755 index 0000000000000000000000000000000000000000..30f94913416d1683ab39a768dae68379d0026cbb GIT binary patch literal 39464 zcmeHQ4SW>UwZFTYgpVaPDAr(67X?EL#s~o-BJAvj4-pUn6%pNRvYTYd$8L5vAz+vt zk`N+9h=E$FR8l}hKtwSRa9*-l+ggQIgI2A+Hm|j9C z)%RxeJNe&p=H7G9oVhdi%)NJJA~WXBG@DHJ6vBTN(o0NAnu1+P?|GCb?ntGwJTegf zzeI+RG>B zP&I{D2E03c6ULuu=F`kfe`Wc~3tCyFR+y2X!V6lNnV`_;VP%pERU-ly|DG` zDDDoq7I~{estW>1sL;9~th0Qr&hm9hl;0I{E$(x`2`LE;S`VCP)T^LbuLAU!{cZfsAy--4mGMM02`zDscv-K4SM(}4k%R(t5M|m+!|O@th;zf6 zdKJ8*SHb&9C}3{TTpW6n&=Kc`5A-VdP_KfIl2Cw)gFg`NyEuHDgo-#X{86uhPxLDI zdlCxZ1x?U?(nONbp>@MaQ;Kd419fW{l7xo+xH{CF8AqQqWh9|0-XX(uYsl2CVMG!d zo(Z|C%3P&%SiDDkc z2@P;a6?aHwwWfvGlTa1!kbK=5X6x24Hwg`hwQAyGt&^q&NvMf;#$w$Xmgv^7EC~$= zG-@>Q)=5)A5=!D-;nA(3RJVrmBs4HroQ}CpngU5EiFZX(^@0Q>@zm{8Rlu1$4jTS`exKlnn{AHFKn@SV1iWi>AGeNP>!{VB=Ew&}7`XVeIlc3l+Emkf@2SRG~xQo*CeC8akEcr{4 zIajxaJlz_mCZQqDC8uX`PMYmWD2uyFn|0q(nrG{_A(>lBjZ4bnTFC<4<}B8&VTo=H z%aYI#=MrrzDM&(Dd@Ip?gJ~|+Z9_6Qm>QSV#I=%uZgaxAHPq_XP?v;;IG1Q!iIjx0 z_*SC(cGKLf+lGGKZXV4p3Hqx87~U?a@K%Oh#lBjv$3>^se@M&{Im{NQVP#rh26wry zq;g=#S#nzBq`9>-yTV=TDlT`2T`T;-GDSJtM)-=ecgMPx&Y$SUa<<@voZ^a7x2t%C z$5raNX?&imIFRS!MM{cmZ=RSl_oe$<>@=D|v#J|&m{c{Y26yT)4_^utYbZ|&+Dz8Y^Z z6E&$kp1(Y9Tjs^e{cnKH{G zVS5xv(#_G=0d`<7FbeqsGz1BJ0Wb;-0PT=VK=LK%p**mi@~LT2foq?ib+ztPb~>vcF$XKYp*)fbyJOQQ8oYk)m5XPZ3(GUsfGmWi3yN8sgH zB-+w_dF*o5Cg$K+!X$D<>K(i#n-q(dNMR7->`TY-V|e#)h&yuyfs2 zSu{@@@-!)V*0DUPu~e1^Hmr+{b}-u<$TJ=7iuuj)5{~Oj4K(c-h5dsq4%qHQ9(Mej z92;4?Yjkd8ZJ?#fhxVP-h&DeeusQt)q0PQg4QP{XU+3*LuD#4;?d4p0*6?!z=Sw(i zO}KjGI-CzbRL+O`ZQmNr%B^uU;g8#ZdYBCxtI(cXQN=ic;Vs(8r)Zf<6KL9Yi8hAO$oKGz63Z8V1S) zjR4s|qd;Ro<3Kr}Tu>fpD##AX2h9e}1uXzA1}yD1VzJo!mSd3y_^2`PcrN3|&iS<-HLnV1=75(~pCBQLjf)BNLxx1~xpCL1A*WO%i7Ekm1h z9m8u=iKHCAO$?)!`p9u|>W$p8v;h<+`8no=I8Ir?aq5E{r$sp08#p@Fam?SqapoqD zvo~{`^9_!3ALlrKJI4h(IWByP=W)o_J_y!!fg8xTN1ew>gu zj<2zrX#aC%y~WmP2}W}52T$cr?7eyta1t;NmC9E zC9zO=Su7I1CwhceM6d9wSS*|nON1YarNZl?Pv{Ys3vY^L!jHvr;T^F;_=#94yf6BN zpNavYSF93#E(V1U#E|eSG1{;)m9>*Kyq(BsC$edf>9|FTTv@x)Dhss{+J5BKtF5`K zS6e26CV?h{@<2C(rhsk%O$FTwng*H_-L^FyZ;<4|JTQZu>7Y# z4wnD&=fUz{KN&3l?bE^XfB${39Q(&$*<`ZH7K>F*O|{Ac23X}mgRHXEYL(N|t@0(8 zSmmKZt@34;S>?+wx5`&sVU>pux5{7nidDYqDyw|WHCB1#NUMC^byj)wXseu+WtG4B zRjWL9tX00@2CIDIjaK=lo2>GL308UHM5{b`vQ@tMW~+S5Emrx~Tdnf+={RroON#LhG z2n0VZVkVQp+Z%x39qY_^!g5DX{stiUnVZbyX7IB&1HsSvhM7zOKlgDU`1#w-;f2ME6K`(|PX z?|BUfzW7IGbh$fvN>2g7FMr!i9N^2}1A?#onVC4j2YvwrAN;i$*VFagG&YF5(!$~* z%7FY_g5!OeIvhjZIdK>TO#@Kj2_cdCE<{h!JC$*jTbK}~1KbPIk`?;KU-p`Gv zc^hdT>bo?}&h-wnEB&Wk=|Ami|Ec61xAM}osM3E%mHsoT^q*0s|D<}iFnbrK6)5%= zDE1a8_7-UDrFloJT$t9Y*xRev+pE~ytFd<%v$rjc2s|FKamKF^P?1d<>@5eLIB6DPw=lFu+cW^yyx z2&s_a)lv~do8)0=#7suaWW-EH%w)t&M$Gij7c=pGZj_DpbJyE=KR3q4`?>g-iT88k z+Ic^h-Ol^DoOa&N#m7ua|7lnHPrK58+SUG38#5{WXH@Awqe}l7Rr=4km`Sm>K(V(# zv9~~DZ(q!$*xRev+pE~ytFgB)X5!;ZedA1VG1D1iPBdn6|E(V5AIi8~`%Yi|j8FD$LPAM(mQCJ))h+cz!=@9UDfFwbnL5ZMS2IuzTT>O z#`jj}S>IdL&-vb3amY7YiP3&cQl#0MvAeewo{s0!ajkb0K8L=y1pK?;!xuSrr^e_M zwM54;>A2s8#`D%&&xl2_kB60gJgn^FVPzi=C)mgHjq}lcR>%3M56(Bv$NZy=^HKkt zZ=8?jwcW;jMSXX^aXy+iXCBW>{VJVroDc2leB*nG+8j~tSBQzlpT>9}#wCsMK4ZMk z81FO2`?TYI>KxNQ^SB@FKaDZJ^~QMLwX84C`K-qK_6|8a#_CTTdA@N#ZH#%oaX=Pd zDdT`FzBM+@$mOP^6A6!M+wEXHivRTVVM6y-OuaU?%wR}T8c}&Z% zZ6uFt`E^aCUCTezOunt@W+wEV*@WT%$jw2|!5@{hEVr?vd%HnLaCe|-ztr{%x# zC^@L*zqyq>r{y1ij6AR9zqO4V((>D%Ae~zN+wJ72mfyL99Mkeo?jkR0`KNZ1sFr_v z4_)K!qsuw_+QdGbjF36BvS;=)nNur!_5hR3*2^`mR``=@-0TTG`1HOje|o{Rk_zQMObodjl)6A@gZvr+S!d zxmNbpn@m=wmA(CACJSg~@4mxiRa)75KVh<uYpj7a*1#HTV2w4fU(hwMxQDyfSOd$R1ESA3 zv3s`9aQyqs=Z1`XHshYnxMvIbjC;0_&$wqh^@j1>qVe3K@!X>E+@kT^;{U_v7LB<# zJZFjL7SC84exbRyk$>fLZ@)`BWA4pZFHYCkk4yAEi2v>N;>LZRai3@0=Nb2TbpFq{ z&ol1xjQ7(R@24@|Pm|pHX^b_NU+^`S%KKi7_tPkAEHC!`G?<&Kv7CL6XFRV$;@?~I z8P3-wdS1o2mox6=jC(obUe36e9G+^KN2 zC9?)t1FQws0#^Z70qcNuKoKYcC7=YX2i5}{fsMc>U=y$z*bH0`Tn}skwg5K*Hv(IM zt-v;58*mG73-D3kqrk1et-!~Cj{&yMw^E)%C0oMM(OjZC_{l!dbfpvc~la)X* zW+tnElEp$E1lA9*keZGU#T6aD5m$A5EI!!rdr>-u_m*YU^>&6$QoEY2HA!{I-)Oav z)xf4pEJOr0UuGc@;QA{pL;|*a#X{BqH(q1GCqPPjTCcN^24Gtj@^^e9Mmjzf>pMuK zp~D=Bs=Dh;QVi(=JV(Rol}u72%4`{H!FL%-dmg>fLYjbECs^=Zl+vEZCR<1|aN8{w z@(}Qe=@xvarL?ErX(1bcJ7yt&$H2(Cj=_=j9T|}g9hXL^eo58e$m?rY`DT;Uf^xfV zw~&W{yYH}&jleyNEMyaK?>!dM3fzC6g**a0;Ifc5;K3pb*$jNX1o=BMBbz#|jI?#w zBAYv|W%Wf>{Vlw{UX^b(Nrcev9<1rcBscqg{8f%th0jzlRl&Ni@x)+-U#8%n{t@%j zWS+8}zCMHgmMJ{vPXC|lt1R;dr`Uvf`L?-p^KIjEvhimdb7Q%`vcwz2cR<6XmF^0B zYRc~lm*N9f!9YdGRphR8g{lJqe=v+3MP=?1eBcz@z4%09VRebiS6S?L75jrOcX_$X zA94kP{$gLbH*{ktSaf4~c}>L)kSzBWg&AWj&dtf2Ts)~TcYI!Pj(1Y=#K{xAh3=v} zcb>@TTz-JUNrRicd~ke>!Di$pPep#iwO7MPKuQj|h=c`e2{tcLhriARo~O z(KNqnS$Y6DSfddlNxw%$_2k{n5`F*jSPJ?Z`$Q~3pBbe2=eDs#-@o}RHGqt0?-Q{E zUFS*jk85X%zJF*G`25R2l6k04#1bC-PxRfaR>%{5H;ZIf`omt5?O9og?|zZ&lFI7r zQg^77DF3NE932z|u zvp=8a>Q7$!8827|5v|ZxsX7sg>{HG=q{3uJ^z)?M4RVgh?uK{W? z^}3)UeGZlW=yieCHIOp+sn?53kp6Ql=2y$B*98?vvD~NitK}6j8IABP!aI zjOupPeig65cG^p;+turqit2SsQN#8@O;e=qq5GwlSHB0Rq8;0`Kf9Je`bH2fuU=RC zzB4zDXUr_&*!Q1i*8C~5X0P93L`~Uy| literal 0 HcmV?d00001 From 8d28651b4e6bde2378134b636ee6aad11c9e5436 Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Fri, 24 Apr 2026 12:31:59 +0000 Subject: [PATCH 3/8] Fix register clobbing bug in s_load --- hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co | Bin 39200 -> 39208 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal.co | Bin 39528 -> 39536 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal_group.co | Bin 39536 -> 39544 bytes hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co | Bin 39464 -> 39472 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co index ced92d63d4be686efa87f1a34830a499c4774c87..d68dce12202d2dc78ce868a2d0b0b0fb5643e54c 100755 GIT binary patch delta 506 zcmZ3miD|_qrU@F1D<*0#cT`Ac00S7!AOPVr0x32iR+t3gD{Mlg3m{yE2a^RE#RV}` zO}1oISHuv7ngQ3lxskD5lZT06|2}4h1N&GQ4(^-Gp?zcX8|_O>T-^H^fNpU5@6@{a zfZmK0$v44J8=({rm}X}vm|QqnT=EOl8!$CWP}v=m7XoD+LSf<%+5#$DF!>@-He@2I ztRGZXWisOwaY>W;sIr?uTqX_%g~^6fc1Aff~P^Ph?lLeG% z=x72{U}6DRYiec+Q)^~ogkTz*FiaNIRNb68Rh)@YVRGYieNK1;Ox`)&TyO$3C?`Op nr(iPU40FZ_lPzc1GdfJJ1dOoy5`pU|6;!uVuz z;$(5j3`eMaP)Zz3vopMzJP|0np%hhC7bVSq>2Q(fkcqeyHmz}IWLz1y$ Yvf~VUMvKXfK(b)+K_K~HGUH5j03KFnbpQYW diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co index 602860715820b9321e676186a0df82ff7d42e55f..a7b9432dc9ce8bd0a114ddb457ce336936ce81b4 100755 GIT binary patch delta 489 zcmaE{h3Uft#N8=Xr`T-^H^fNpU5@6@{a zfc}LPMvlo7r-)1TI6>`&QsQ8mok3^vL7?o1GE`YzsBFdLk3iWKEvT~2P}!czic`fU zV-}#wMnYwCCI?OxXN;JfIdx~01w&3^Qff}TftiIflxgH*1Y;UIx*0%44IRy4YE3L0 zVN6qmHZv1bm^L#LGiQd$f|@#$S85q!GxRy(K|Yyrrn%4wXyBcIMu-K^ lHcCS@w(ylO1QN0{|pEZI%E4 delta 498 zcmeych3Ul>rU@F1FD7a(cMM2p00S7!AOPVr0x32i4www#2W&>A3m{yE4U+{K#RV}` zO}1oISHuv7ngQ3lxskD5YqE&W4IUEeXXtan1Aa2+Omo2l&>%bjjTVQ=fiuk+4@~Zy cY0p?N`6Q5BF`09gJ*UJpi0e;H4xAwm0A00a;Q#;t diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co index d232c71f584b63abeab654685196737795884f2f..73b02902507b1cd7746402ba20c06d855145382a 100755 GIT binary patch delta 492 zcmeych3Ur@rU@F1KPGA}cZ^7900S7!AOPVr0x32ij+hMLM{Gu=3m{yE6_W)S#RV}` zO}1oISHuv7ngQ3lxskD5i-(C}|2}4h1N&GQ4(^-Gp>t#N8=Xr`T-^H^fNpU5@6@{a zfc}LPMvlo7r-)1TI6>`&QsQ8mok3^vL7?o1GE`YzsBFdLk3iWKEvT~2P}!czic`fU zV-}#wMnYwCCI?OxXN;JfIdx~01w&3^Qff}TftiIflxgH*1Y;UIx*0%44IRy4YE3L0 zVN6qmHZv1bm^L#LGiQd$f|^E?_iLGNemdsGxRy(K|Yytrn%q=XyBcIMu^4a kz?tTZCnk5!v}dfCd=f}*n9MoLo>O5O#2?ot2hNZO0GwoPVgLXD delta 504 zcmeydh3UfA3m{yE4U+{K#RV}` zO}1oISHuv7ngQ3lxskD5YqE&W4IUt#N8=Xr`T-^H^fNpU5@6@{a zfc}gW$u}WT8=({rm}X}vm|QqTT=Gi>s;m-JcE{v}Kv{<{R9OqS-itulkV&Yreo$GJ z$&6FQB~2Ef%I*honK*y}V>oqZlruw4Vp3{Oyn&gy0hDRv>}CRG8ap~!K$(V)CNKpi z7I3wuW~MN;W+p}mrm+dbWI;{s&H2+LnV1yPCp%6NojiYrBqux;CLf$(E;Ioeo)e&P mRKPR2dxq>}^_h~46DB*(v}bgf+z2EqCLaWnFD5h2QU?GDp=Wvk delta 484 zcmdn6g=xhWrU@F1D<*0#ca%tH00S7!AOPVr0x32imY59TOKe7^3m{yE7n21U#RV}` zO}1oISHuv7ngQ3lxskD5YqE&W4IUDDkclgG-vFX>^ak( a(PDBZkSv&da;81!gQ*aIoR};)LmmLR^=8We From 92dd9be925a48946a444d929807c0b219588f852 Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Fri, 24 Apr 2026 21:22:51 +0000 Subject: [PATCH 4/8] Fix a bug in group causal attention kernel --- csrc/cpp_itfs/mha_fwd.cu | 4 ++++ hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co | Bin 39208 -> 39288 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal.co | Bin 39536 -> 39632 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal_group.co | Bin 39544 -> 39640 bytes hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co | Bin 39472 -> 39552 bytes 5 files changed, 4 insertions(+) diff --git a/csrc/cpp_itfs/mha_fwd.cu b/csrc/cpp_itfs/mha_fwd.cu index be7f94b8486..1673fa6a05b 100644 --- a/csrc/cpp_itfs/mha_fwd.cu +++ b/csrc/cpp_itfs/mha_fwd.cu @@ -183,6 +183,10 @@ std::tuple get_grid_dim(const mha_fwd_args& a, int ts_qo, const s { tg_div = 1; // do not merge the head and tail in seqlen_q direction } + if(arch_id == "gfx950") + { + tg_div = 1; // v2 asm kernel processes one Q tile per workgroup + } // batch int gdx = ((a.seqlen_q + ts_qo - 1) / ts_qo + tg_div - 1) / tg_div; int gdy = a.nhead_q; diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co index d68dce12202d2dc78ce868a2d0b0b0fb5643e54c..01e967338f47f7296d4163337d80d2d7461ad83f 100755 GIT binary patch delta 632 zcmZ3niRs5CrU@F1KPGA}cg#s=00S7!AOPVr0x32iE|>)27i>bKAzX$DlLZ;Y1u;}j zwq#US#1Ms<0n@v=k@1NpFCWu^4M33tKw5xtbCR|k6RS1D>V4LeTXnS~8yF7lvufhp z-`L>X>g4RyVAX=e?m%KsfU&I_98H}jIaoROJDWI9;&I|>;BDY-;&F!1j^PQ*{m?hfick{1nMd%#RI0<8TL4$ii<Zy6bLCU;KP=Y+@m=`FaZUmAiCLf$(&uK9g;+s8_yQa%AG5`Q? C5sVoC delta 533 zcmeydiD|_qrU@F1D<*0#cT`Ac00S7!AOPVr0x32iR+t3gD{Mlg3m{yE2a^RE#RV}` zO}1oISHuv7ngQ3lxsmaS=H^muX(m=#hSmFICr{PYW_2`mn&dF~udc*qNxk5d%>|Pj z7!%(FL#=~SJYbrgp}-MUTpTL?1?nr9tP)gwM=4AMLhC}s9VQD-5tl4zMwNAj%7#pK z1j?$+N0p6)%9>0roFdMsFu8Ne_Gk-NXNH`_q|}^v12c02DAUN<%>>Fcc673UG7TL~ zUae!q@}g__0+eFKzBup$S~INPXFa)9Pdl=K;m|&- zCeHnh4bH7j&Q1+hElBJRB=!Ut+p59Q)M=7~m2fiZcVwcOq>u*XA6eOf|@#;H%wDtV%#$M;tYLGcoLe- zIn!K7V++U~3=A4mAoLCH$w4#Cm^7wL?wV=Gcw_R(nf6Qq(k delta 613 zcmcbxmFdG4rU@EMAEr&zTILv$&Hx55nn3`NLq=@?SlP&64`fOpI?Q z&ozu^F=}*bn0&!iW3qwK7N%vKoBJZG7&j-FoM2}>wmC388^rEq+w2=*&Nw;3H3zJG z^99!@zMFSUc3_m`2!*;9O7Vbcc7~qGf>Xuobuv+9m7ua8%2361q2d)`sNxn-@f9to z;?7XSEb0BQ60EjpUBA7UU!GCe;j%Wi5hMdHt)SP$&GYe-Z z)5ygL#x!^K#sft<|1aBA|# G8S((oLx8jZ diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co index 73b02902507b1cd7746402ba20c06d855145382a..1506903dede3ed5abea421748fff020f4222c4bb 100755 GIT binary patch delta 749 zcmeydh3UptrU@EM5z{AXEpyzG&Hx55nn3`MyH0!7hE+a8yIb2TE@A#FS3epbArhUcE)3y1JkoX z>|VCbz7gh(lOtSnz{)pYaDC#tdBEL0gevX_6^~f}W=vi_RZubq!UhX~h_fJqi31oJ7pLxs zHn3pGNlZ%3i8nB_aE3CCT#R5$V@EdwsHmZ%IZUmIg(HkmSgjdW z@3WrVs;3>}72NHV%jBVB6XzDb{!OFSc*~EDgj}uP= zZv$@=k28dJG>6J7relbDp6 z6K`PVzX8;2j%^(2bGXg0#AXbY%bN2W@43PSiMhn@>D%-R!38( zNe+|$>Pc*t)DKSCTrk;zG4V|Z)N&}r1E$#-3Y<{I#i8O~GEl{ppyE5qP{nnj;trDq zr;1A!w4lm5LuErII|5}@7NE*TLS;=R7fuytRG8d3b$g73t20ARVp3{Oyn&gy0hDRv z>}CRG8ap~!K$(V)CNKpi7I3wuW~MN;W+p}mrm+dbWI;{s$t$!BHh-V?k`ZX%!x{RV z@Hm<*In!Kl0yMNIKx43Aa^g&L#tD;W&a`KAn0ymRR!o+hWzYFyD#RxzCMV922LL|# BXz>65 From 3e986ddb0cea1c21155d828a4a6d7eed52afe496 Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Mon, 27 Apr 2026 08:55:12 +0000 Subject: [PATCH 5/8] Fix GPU fault --- hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co | Bin 39288 -> 39288 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal.co | Bin 39632 -> 39632 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal_group.co | Bin 39640 -> 39640 bytes hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co | Bin 39552 -> 39552 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co index 01e967338f47f7296d4163337d80d2d7461ad83f..e50fe8ee2eaccd90d3176bac5032ebb626ed4732 100755 GIT binary patch delta 22 ecmeydiRs5CrVT6f7_BF-)Kg}(-h5VXbrt}3Y6+(R delta 22 ecmeydiRs5CrVT6f7-c7~)Kg}Z-F#MWbrt}1P6=xO diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co index 7393984cfba80cb8f904bbd3ab4e1276571d2c9e..fa65f2c024d25e668f9558484261694362a686e3 100755 GIT binary patch delta 22 ecmcbxmFdD(rVT6f8LcO;)K_M--h5V{IU4|QLI}A4 delta 22 ecmcbxmFdD(rVT6f8D%H0)K_Md-F#M`IU4|OCJ221 diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co index 1506903dede3ed5abea421748fff020f4222c4bb..ed3dbd9f9fb63821024175a22915bef368edd11e 100755 GIT binary patch delta 22 ecmcbymFdP-rVT6f8LcO;)K_M--h5V{I~xFTnh4PV delta 22 ecmcbymFdP-rVT6f8D%H0)K_Md-F#M`I~xFReh8HS diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co index 280540df4d8628ccbdcac674903c7b09ab595b43..9eb09bccbe589f564dce0080c6c9954855b923b3 100755 GIT binary patch delta 22 ecmZqJ%G9uxX~PPAM(fEd^_3Z|H=ottoCN@6(Fn)@ delta 22 ecmZqJ%G9uxX~PPAM%l?L^_3ZAH=ottoCN@4wFry= From 0e3d32a4002f478e2242965f67c0a4a18e514485 Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Wed, 6 May 2026 09:50:02 +0000 Subject: [PATCH 6/8] Remove broken varlen variants --- hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv | 4 +--- .../fmha_v3_fwd/fwd_hd128_fp8_causal_group.co | Bin 39640 -> 0 bytes hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co | Bin 39552 -> 0 bytes 3 files changed, 1 insertion(+), 3 deletions(-) delete mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co delete mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co diff --git a/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv b/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv index f418e10cd0f..cbd24fcd460 100644 --- a/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv +++ b/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv @@ -8,6 +8,4 @@ bf16,192,128,2,0,0,128,128,_ZN5aiter32fmha_fwd_hd192_hd128_bf16_causalE,fwd_hd19 bf16,192,128,0,1,0,128,128,_ZN5aiter31fmha_fwd_hd192_hd128_bf16_groupE,fwd_hd192_hd128_bf16_group.co bf16,192,128,2,1,0,128,128,_ZN5aiter38fmha_fwd_hd192_hd128_bf16_causal_groupE,fwd_hd192_hd128_bf16_causal_group.co fp8bf16,128,128,0,0,0,256,128,_ZN5aiter24fmha_fwd_hd128_fp8_gfx950E,fwd_hd128_fp8.co -fp8bf16,128,128,2,0,0,256,128,_ZN5aiter31fmha_fwd_hd128_fp8_causal_gfx950E,fwd_hd128_fp8_causal.co -fp8bf16,128,128,0,1,0,256,128,_ZN5aiter30fmha_fwd_hd128_fp8_group_gfx950E,fwd_hd128_fp8_group.co -fp8bf16,128,128,2,1,0,256,128,_ZN5aiter37fmha_fwd_hd128_fp8_causal_group_gfx950E,fwd_hd128_fp8_causal_group.co \ No newline at end of file +fp8bf16,128,128,2,0,0,256,128,_ZN5aiter31fmha_fwd_hd128_fp8_causal_gfx950E,fwd_hd128_fp8_causal.co \ No newline at end of file diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co deleted file mode 100755 index ed3dbd9f9fb63821024175a22915bef368edd11e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 39640 zcmeI53w#vSy~oe)CLv_VH7HiFR2Kz9E5?Wc!b8~E4G<9!0TmJ5Y?4i~WRu=Cwbdos1~YRFXeTSx~lThI%3S)corHE&gSwnC{<&Zci=qJCKiV?_xi6TXK2Qfa&K9 zev-+|D%G+d0`o7i&Iv`M2HzuXEeEn( zMWJe!*Xb%MatDKZ18!Hbb5+0-a$`+#aR4@F(C;d8?=JQF3SHhOvMOB_?kD`AfYbM6 zman8F=nj!5vVxv9?$mu*H7;+p+ga|ZEPk3LcNSKcl(+-UEZa`Mt60sH8Yk19!qraM zh3YA^+V9@!xq0H5VLl~?>8tdvK5vv&>IO67Z1B8Mj*OG(^N6xsjnW!tgXfKMe4I?5 zN0c>cl-Y4Mc-|;;<7B$vC@-*LK1nO4H|SOawZ+-yc>~Rllj&0fRjT&xV3Et~3aHy% z#9Sn)6`39q=X0SwI+p6pzf5QT6$#{j3TFy;mDB64R7;^C0WaE8DAt+3OlN+a?dXZ^ zqQ1r5LFW>8RZuMge*!MFMG(@NzgB1dx&-p?3Obh#7Qjg{0T0>|IN7AvL91Q|XfH3i z9-t8NPZvUa0v=)tVT)b|TlG5FmVg5kg8g(MbSB^-rVw`Nb+AXTgZ&9Ocp~U5t#YX+ zgp&soa1m1qhxIz>*6Tn@z`>rNvpnX^crucJmzaWhQLlrS^g1}1fCDrT<=Rujs|oms zDTdeeI(S>JgZC0}AQyw?wckCvizE1RUVx;PZzDPYxd@;3B3J{;1c% zM|vH6l7Iu0f~M2nXCevs&=y0VDM`17RNWqiCE#H{&JHza_M`hu840+GEsznqJ&e@t zVRQl>o(?*z%AI9&Sfbcdp&lrGCTjw|VoPMKZV%&idzg@b2UL-oI0cfOfUDR7$JTLP|P3nWjshkV^0<|p6*eXW|9zE+=UQ37sa3uCEn56g6W zSdoASbTn!-{jEMzK>|)%FP0td~mrFbHu&M++5U9b8IzxUAKp~b$fX4Vmu`F%^g&$nG-h?i#?U4 zr>oll+xo>;xA)^@`_ykxN6@soogMUtakhQ|)$XG>**@c-pO#D7S?s2_wg&I0^_fYW zO`ktvb84JypGU;yXGfe7XVVuD@yIyYK97iN&W_j`XVVuD@%T8|J~d+HWON{?ZXQ=r znwHO;&738EN;2o@_K>UF!;Azx#1zTtyEuJjTLR8v&eCSxmz3sw-7zF`NvSE4@|aq( zNOw3(b$eK*+rx?kJj4`h@5V zfQOhO(bf_%0cWwbMEB*Uxm9-z7j?OLEUPr&tM+4fyR^bx8FH3*YTd<7dTafM#4M5{ z@&q-^OdCw#@_I@uQ@hTx(=2`F_U^0-SFN+e>k2tn`2yvNb9o(MGtS z0yj@7sVH+fOI8&-%ZhKBnCmR@=Q@jA)j>>tV>wGpYNt-hp8b55Kj5ixh1`Q-oQ{(m z&I-7zFhee8bDPmAGMV;e1xx(_XOXYEGF0$nR&aGiq0jryq>B$AtGpQVh6a7%cE~eX zt6Vkil7J6|M)%#O&giRg2ZA17WrsT5kWC7V^PeWT)PmW8>ikM2O%6Dm&WKd!XDW7@ zUX?4cY)uvEnj~ubc;wmw}(| z_4ZBnsJ%n95@Hwa;>>zSi{nAZMu#0*`(vJHdzXha5K@o4>-<%j_5NTc^)F(V>JeMi zUzKh}g*#St)Pppct^@{W`N0&NTOgMY5+SN}fV}f+w=KM86u? zJ=4iEJ#@RH&HnHhe-+nFl3(kdjoAD_Zu;KtS>4IG!m;EW;Y#ut^1skCtzl?yz36DB z`!=mfv)rm#ZdaG5iToschi#f;lOxB`G0Z+pOc&D|hB+AdDayLZj*bj_hB#at-jLy7 z!M_%vVN1iK0TT)uSb0AreA08uw#9cbSP;y%V~y3hs$XY z=ep^cG))K6v?yuT$!U_K$#NRRur4~zF2`m^npvomG|sQsUqGGUdg(IiX|v3+*iXb{ z$38faR<1)W_KkAgZE|ds>rq>k2ggWe6ONm){>|w>2yXU_ZA889_y()qFIJ*5R1?p>j+$Z2Q(YIbXfK1%E6CY)6h^V-=2xKxPACQ1c@HxL4J@ zo!=fev)R6p9_QU8=e6!;Lg=whkC}1)>hxpP$2_9{eL?j94)P)7kC2ZbpFl`h^d~`5 zA;TaUkP(oPkkJq;WGrMnWCA1`k^{+w%z)S+d60a_e8?imQphsM3P=H@I2<*L;b=4* z75%l)>u}t>*fX;!srQ4Vrd|`-)cdst`$qe=?Vg6mJ!7JeZ^toB$PX+JhhMgA4bKtl zM3Qmez$?To&WYBE^qfTXlZsA%|6tRuAHj6Ge)?BlsJBnF*PDp_sJ*q>vyz;6^@Y39 zwBxwzZf-W&uTSc=UvF==M`yQ)(P;EN%kgj{%CX7+#5n&YZM`2$Rq2OuSbi)8(+}TC zqWx7*disM;dFCoHogx--t|q3}6=}T~Gbu3=aAGJAG(ztc7hBozvUuf&iAqP&47-}}laqm^)j-HO2nB_cUt&HQ< zJflbTC(->NJS$rox9w6n?SulxY??L?{>Q?e6J{ z?;Y$rp?)LTa@x@Yj8>XyB7r)P-->2;N8WwLSqZ}_yJ9^aOWNlrEOk}YRFv_tO zE3vY&e({}mQHI1M^Gn1mb0o3IyfXY^TQ5yNVN^%5Xq9cF#W6Ck60ebI6|a?fO)?Rc z?{AaLh@~NXf}DCSr#xi{D#jTPFwP7!+8P<{>lpJkFwWV;n7^5E-Xo0j zA7fm&opI4l#>G!EF5Sm?*E5XE4lyo2%DCcr#`}923%k#Up6K$2PgFHtVJ9!Y`rSr!PGCPl$ZIF+ITLN?u57Sa zyDfomj_u&7oJj*$O$H}}xnM3h4V(tf0B3+R!I_{9w1IZe4(5S*;2dxcm=ETI^T2uF zd~iOv5L^f@0vCac!NuTGa4C2fco(<~Tm~)&mxC+772y5g{a^uD02YFUU@=$>mVhN- z8CV9c1XqGy&jmBM7+D@@@lg5c;CBd%`CY;iez&lc-y__~?-lOi_X~IP2ZUw( zLE#?$Ibk_}Sh$xzBCO!Mh5Pv9!u`A?IQbr-fR6|+{sp0se^Dsnzb6#)F9~k`Wub&W zDU|X*6w3Hl1rOgZtmI!8%K0}1FaNes!T(sOX9^^L2e8>XGLdflqMUXoniy=!OOCfhc?t|M+qykb2@j?8MDo6kl zgh>DPewy^#-=siM@)b_H`f5%ZGlr9{y_S>4jpL-uOiudhS2^kW>pAI$8#w948#(Ewn>gv_n>lIH zBu<(#g_EXE<)rD;Iq8;LIBC`_e0Mx2IUJlcdp0M{oy$qL-pWb0-Ns1^7I4z-w{y}R zcW}~@C7g8Uot$*{-JEpKJ)Cs!y_|I4eVpWUa+1r%Nkv7R(FzVa8&JO5`?btd+HM*I8LRH{QzHxv$YY zH0|is{mGrIotx0f+PSPw*3M;jvUYAFP1{7a( zM3nY3qO_k8rTvU3?I-oSMUHoIN`Vq@ff8?l5^sSfUYd6F>cuGoO1uL~yaP(S1Dbeu z$?n5!3NU`0)`X2?1R*~c#rG2QE(&F@?;`IH9gY`+QjIn(v zO~25#g!M@Z7|-rMDg9WvpW|Zjb~2vbe^UCma{sA{#XrD!cK=D!*A$CbteuQ8{ijHf z#W%oscAtrt>(U5fm-!{)kU5g%$&BqYxn$dDu~6n!Vv$U%SS-`%GZ}p*qt9gYnT$S@ z(P#R{>oc)-ZmgBHbJtl}J2&3S+PT<16Km%tbh36XtCO{J*`2JNi|sQh?Wav?KW$3; zX;a%zZJ$YLKO;)}8ByBLh|+$>^qG`+3zT>Zlz0m?@ecNxlz0b}cn6et2Q={x_LM zPgQys?aTE9)4MQ^_i;~kdKVoZ>sgiFh4HaBs-E_|5q!q;M)k9vH&z|;WiwxVa#XR1Jd2YToN$M@8JJf&9XcqknkJl~k#dhU$=D)#Y+vX4iU zeLSM<S~c&@QPy3guZAT5J)jRnf(ql^X8@;TR7AWdt%m6a7OyK{{N(zMwNSXx@H z;>8}#<<{m zV@&WGxh>EBti}ZQ4m-P#busi>=Vt43eyi*t{aj=8b1 z7|s1vW$euuD~uhVI^S5~5{wn*VXocF>1Xt(==?fko|-(Tj*S7z{nyWBo?2c@73P5k zJ;$0G*V2C0cRgj8d#%ySS7O{&qgQxIqgMA-l2)x=uqGGR^POd?9u8EwUei``sNO@SF3+x3)!dDA9H}}vfpM!N`9}Y&?>{{E? z`(>L$YkTH^Y|Gc$o;@hr=;cC_MI1G+j6b#yD!SNd$qRje^0g*Xl*~hFCtJ@ zF0JjAmt|X_*4B4YwiRh@Kf=s!seakEQfqtTb=g*~wY~MGZ1Zbv z@4PMBs6(b|6b3)%L7*7oaP z5o4aLF;CW*Cu_`;HRj10^JI;AvR}}7vbcu3$CxK8|5k|p4obde`wYjw&-`tXam{92 zvl-WHL633G7W5d`Y^Po`e%omLw$b=)qw(8DVqca>lis zaV=+D%Nf^l?D?R^Jv-x`opI03xM!EhJv-xC?hAe`XWX+(=$_qWxM!z*CKO%nQJ%5g zgXe4aRIRo2%npU=vy;U-_Uz>2cy{t}JUjW?P&IelQWFl@o;?++wv4O+YrtBt7F+|a z0qejzPyhu`1Vyj`Yyg|UCa?u;0b9XVa6PylYy;cCjo?PG9c%|Xzz%Q=xCML|d>Gsc zZUr9&9|gC8+rY=c$H7jp6Wjsr0C$1Az}?_(a1Xc#+zajn_k;Vv1KhdqF&oXPafz4RnW0VyC)XYZB{_zKOGtwP4F779xPHmsv;{Tz`dyh+x~7 zEu2W&NEF3)ukfn2YpX zso`~9Y2o!<8Q~3Gmxif-QT5-%w%4iZttPPz`F7oAAzue~-)2@ z>LGa54gE(yUt{1H$Vi3B3fq+RX^Os5;h+8<_0h$AWj%d_2K{&yooT24&-YZ8y93j# z{DM5|{P}s-iP>5BvyQ*f>#Hnv2k@TKP+6s`0`I``IYVW5!B)Uu5yb1VDxJYhx5W_?#uafYasmI(Kj16zc-_GpgMp$Oz22IN8(>-K zE(*ztwInAycS_0R!kme@CE4!DC6lI1au>Raa$UKuDHC&wU4=JIytz1gviqy2<34lA zBpZg16?ew2p^y5&i;c(t{c)e>bA`M-gtXEho@qYU$jd{>{8btw7H?4Ouiq7G&sFS6 z&9Qd+yFt3$_d8Xa7-|Mk%&b1LaRG-#3+it#_*d%PZjJ>Gk~ zNLHmUS3mw-NesvyiJ0EJJR##7MlZLbum>)6_{0Wifm5>}+LmVqR{pvUsySM@?`O zpO>RpbEamii<2hLR?|+-v7cUa?Q8UL zk?f-`DEuT-UBRdxzp9nyrRM}Sn|hv5Nq?J4KlD67w?!X|LqF>I;}R_YIU4n;`PK7; z%CU0l)Be@`ikXbU0`;xtSI;Xdtx7_5y&AvDtFfN8-0FJuJf>1Tk11~CeNdMvY4Ong zQuC|tt5a#iI_<}%Rj_;`gytVS&#Ldio4^ttE@d42PZMiCldw$l(tV&G^|QYV4ZqB;#SGsr#?4Q@5+?Wmt%pUsUqHsU}o- HX8!*MX7`55 diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co deleted file mode 100755 index 9eb09bccbe589f564dce0080c6c9954855b923b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 39552 zcmeHQ4SW>UwZFTYgpVOLC|0mk7X?EL#s~qzhp@97K14tSR77;M$!?M*o9t$H69UHB zNeBTUg&3$(rIG?70wRimfb){Y+SV$x8nkNlwR!#YwZ2zhKdtTSzW=>HLz2aLS`H=J_|WZS_Y{JwLUZ0ODPvni)qKV=$BzPJ-$n=PxJX5C`aq->qhHh{-g%e?JWE- zn~OEneBTe^x3SL(6z3PrUU(-V4F_En?y`_8r`!{)^!P##H$R!<3YLc6C(TUUqm)p5A=nZ?Y$L$V6<_rZ~#hyK-K7WzR_he3`tHSeSARKi1pUUx< zl!QEC@?=iPyV}!#e@>0dSM70@dn?_~@a)c_>XH&qa4j#kE8uc#h59Ebw6|!Llc~@& zg;xbUyS(Gaoo(jR%uIi!Z`DPutkNpXN>JfNtsI)5&}U&~xn`v`L4_B!a&&@1pM{k* znw7Z;D!izb`3VYLvXz(EFrT6g(--n+hT0O;c~L{>Bq(%VL)E6eCsge6xq@1~3%QCk zts&D>;$kZFXUCEh%P&o_{PHBqKaDGer^@N`RBB!*OhSv^3+@!lm!()9S37!P>(j5e zC*)l0sS0T>2qd9G?}BiOzerWV2dOIfFbM^?IQRqM-iyOWNvMeP!XHyr@NudN z{+@&actIDmpE8jobm-l1%9NI}hW;sQ7?gyD1GqZWoE=A>GG!&9D&8SOQr0juWevlV z(C|#iSyk>Vqr(zzZ-sWEoHALH&=v2Iktu5!owA0pNoYV5sY&3F+$2=RJ0ve<4f!c+ zn3jYFIHZa@q_SGqLTpK>ig!ps${OaRtYKaf8W3yM#Kl^tObe4x6Yq>ADQj4ovWDeJ zXh5J*ql>prnF^Co67LFk${Na2*5FG*19QchnCp}&kc5(WSAvDGVCnz)_UAdI<@`-VwTAfwm=Ol z(|R+weBRQ^{_W?;X^~UrmX4eXSFN+e=L$Pl`h(@Fa<-4~6=&~`aV}dh!G+~)!ST5z z6=g1G$x64g%zg8?d}m1@-^q)Vmefw3kUR6moIuc9;|hCv>px>4c_An0slw8@xO%oA z8Zw#o=Y&cFL1(eQx-wk&R8DABMUmh4ZcZO+&M9|erBJUDOn^L_v(i=LDGB=FCVI3s zZ4F|IsfVI%Pm+9sI8||bKD3VOjkf^>oHY1&9BkbT$V}KT2k3# zAGx3VM)b`U7NjX@OA+b5xUCOIO!|-)-ifQ`f{d(Lr{)@b{@mAM*R_^Gq!97W;SyqgkclOuy!GB zNw1T3kU(}rplZZg5*)EM7#h_P`kJ?q1V%Ilsz$AWtUegZ?g(x6u2p3_NHBX%Ff<|- z3}t3v5$!q>&Ta}j(U+N zdp-D$qeq#YG;a&?9s`ebuU(Xn9XaYHX(qzzZ?w14dhHI>8Jpco^~Gk-mTA4y*T9~b zqt%uLnPZMj%f!qZBJgZ15^e6fGIk|vBXjY4!X$E4>?+1@RQX}CVT?bi@;8{3NVd5% zB9pzHG4eFp6Eu;1oiaH9{uz}&*m;5+g6z4@dUCjPMr>Rw%{P_o@0?1W?WFY%HwPjk z0#!l>Nq?hbCTt6YgsJ;FW^|*0y7YQoxrS@K|ca6^`ZqiD6a&bY?W>aaUx z+qHIv{T=<~0rG$baWcx~J6fp?GvG^lERIo}PqU}V>2i8QjBMzH`{Wqe2sD!cu>s6) zD8KF+?2D02>i*4ae`YL`?T3Bqrruz0wPSybw5oZwusnlfgIONfxo&DU&C`lJO=_NX zEKhnYo#lZI>tdtq%r-mn%s`u@c7ECZ6552&Mbl`fYZ;Ejal$4$j=_PvtPM5UH?g+c z=-9;CQFE0S=SX%V&YO{eEt%gBZSjs=gLd8eb>8OV+T1MG=FX?5o}X7Z=fc_b;p!3V za8CU|J*OJBe{&QoS8s2^AGZPZFdH^i;hYF&H^2t1EXt31T`Sx6tx?n0+Bebjyo2Pu z(Xo~gdal!RW>la$^H}vUuN?S5k^>)tJ_7v_^fBn~AQF)SX`ud~L7*(q5YSN2aF7)= z5;Ph#7L*Ih1LcFJfoz}x&>YY_&_d7>&{EKHP$9@2iJ9d{EEb8$fm-l&IB#C=oZgt$ z^{ceTE)&_@_0J0kFyQk2bSUL8$WiftG zA;&Czz8@1ZgysVdrTKuvX+EH}U}jp^jscBbk(Zjg^2ou{Lk5`5%cH1szdB+xV%%yn?pnoLvvt!1!i29fwlWb*qhqQ4Vs^uL! z?DBM`_YUrFsyCX;@(%CeXr+0VsCF~Gc=m$+R!r}8+}<{h%wC#z_^?Ik`gRZ<$8GK5 z$ZU11wsK#;{O)`3Au%!hA~7=@N-PYoj=bF5Me~mx)|xI`nQXW`g5fptbquZY^$f2| zC$jqcZDts?G(=92({JRJXY`{u(a$kI#Bs_>j?*6CI6cDAwuYm99mj%=9A|CjIA;sT zxsP(3_c+G|J2)=f#c|P79GC3pc-ONWmmcQ0>?p_OFLJ!Ui(}!pITn47qx&_EB|qd? zcADdgw>kRW<5>Aqj)9+Z4E`Eu2}DfHFLKw3_CVxB)tam93z=^2V4ML09*)M2wVhQ1Y81K0=x@&7jP+XDR3EZ8E`ppIq-hq{lG$C zA+QKo1at%4z!G2yunbrRTmf7G^Z|XqN?;{001N@0?qvQ~qqyllhG*jFl%@UiX z+2ST?j@TmIDz-{<#jVn9;v>>Laho(>d`wy(ZkKKspO6-cZPFd$PHB<2TUspck(P*i zr8~ua(p};K>2C3$v{XDK-6K9PEfZgm?iG(n%f$}qKJmD8zoJQ0=g4)7wB%#QqVo1WuSXO%R%>n?gu$Rg&-HG2viJmgFK)TP${Skm`5d4B2X2jwPf?xEM8AAg* zyO-<-g1_roGr0x)(!)UT%Z{4Kbnwex1cJZ6%S>i~FZ?zTe9`yJ#0K8|8W4QR56$Ru zcXpSZ27+Jlwwc($``!bBul%W*IKT&f4g??kwHepb4P7)gh`iFw;v&k2Y5(c%PPY=) zM~$@de(nY<@8?Eac|Z46T88EwzN$ODjrVh7+ju{h)5iO`+&13NjiY%RX&>snJj2HI z4za2Ir%mlYZCd}S<{i80@{Fk3e@4~*GphEVQMLc1dbcur7iAQx_7g=U? zhp$?c(WBbiquSe}+S{YEcQ>=QHG_ye97u`2@RcnUr4jz9LawT)w3{1gpw`+7K@8#9TlEe#=dhF>HOhC@jK!}yrV#bm?fB8Jz<#SE>o zo1qai88MR)GZ`_H5i=Pv(?4I##QV9CR^HFuVCDVXXe;mM;$tS>&y8*4{aj8P@8@#c zcs~~(GpYTjP3=EzYX51|`cHk#r1qatwf~H&{by9|KjUI1)!stY-a^&hLY=+6F_UU< zk7{p^YHyFu-rksrk1zF(GsVSBXN@`0n923G28@5G<8p1gybU|cN5pnkc?m+)hCl_< zyFRlqx^--fZa*KR>#WLbr*T?mD6<`7bf0uqXSUO^ug;a3?HK!dtLho=TcKyYZ&g3% zeQV`m?;JHo`w>Z#=jg`n-d1@!CqT!w-ck8n`Ysgk?}86s;@F)Qqf^uq9mk~Oeis|h z+aR13i{coMsKph$NRK7 zrhn#fKiYp9V}2Wq@xJR=U!MDEjrZ*vbZ(5*mpJl5$4O&zgfu2L=1%Ecu`zdQ%$*u@r~fzRPK{XV z^D}=+$J6*&jBf5!9al5P0prJ~E;bIh7~_BiW74`_%{(ifqHE`jHE3+T93KN@@#|-@ z2Cbl}3hOsR-eYUmJV@iL?|93wW?ILWufUkBj<4{MHG1A(N!IK6ssMRN&xe9!gPyMr zlZ|?QWesW3^Q&sf!+QRK)udI=KUhb$==q35w(0qLnS4XfuW2BU>G`#dSVH|dfAC6lij12efuRQTc($N=Vd0lS1uYpj7a*1#HTV2w4f#v0hq=^9wv!`)-7fo0DD(Py04J=>=^{(bgy zL&iOuanEMlvxU6IJzL0Y+_Rm2!+37dcy7^nZqayd(Rgn0|KW3s#@rj8v&3_YXRQst z*xcKQzw)`a-({RN_hzgYr)%uTCVC&l|Mq%u<37*0&ol1xjQcz~|7YCi8TWa{`)Q2# z(-`lkN$&kL#v04d`5H_0eJ{rQY1B2Am->Dh%+1wU&b`Moo>w99@2&Y1=j#$ZuVUQG z8TWF=y_|6`XWYy2cYhjlcE+5YF=uDY*(Eb)XWYwu&hO=nIlH9h?5@C^o&KFrG~KDb zKf4p}&hD&w(9$_G9HH+;mh1R?k&olO$j9+swtAY2`B+&pbTsPHUJxejld>g6L39nJ#YhX1F#v`4BQ0V1Z)Ae09%2rz^%Zo zz(;_O0Jj0R0UrZC2HXzZ4txUm1h5U*2HXkU3EU0b4cr6V1KbPT3)~0X2Rr~g06Yjh z2s{Kl1biO&JTOeky2HOUlWJhi@64nISo;SvSqWVI7c;2^*8R;)Rsp4$nXCrN77KX* z*wD{HYT7@LR<{2}THXGU^g#RXCHXksTb5PV(-AhwZCbk4B-bH-qhKKq0-G+g5DB>c z3JZwzx`t=(*B9m&`u(2+Rc%urn}B0 z$B-_>b2O}8*(5ij%+@g$e3zlD`;nV0qzSlfyanG@DeHc0l7*}XZl7u)4*{Q;VZrxZ z%DUSe7P1kzb2jp~_m8Y=9~jxto)y{HetCrImo@#3yuLP#UvH9|QEvBb7VTewxfz_tRHs z&~3TObMDOldEUx$PjHG=oL^v_H?P1tE;k2%*3mcl{FSAiAiggeE~|7^;1gDUXSfU> z#tH^1Le64Wr886=2>63xo!-h4zq7<2bh>;#r$6Kj1pOsm zpC@!vC|G=x&sS4%BP1(4#bL%+OY(B_CzVVr${UwolIxjRGGWpLPm!xQ-<9v0G%nBW zD!O^xcz5nZkLr^%)}Kk4Sh63iM`;)-oUJ*;U@jub`xLP1j^gS+;Q|S+TNRE3|CB7#{a!M;K}%Dl?xDw}mDj#Ar=bn|^qWo3Abk^vme;PUz2C1J%QN~)Irjdi znRVL)q-j}t40O}Z!@w4m=fHeKlbZkVT5`5cS7loH9`!u#QOjHT1xn4$ZS7@{ps#|Y zCgtjdNNeSvJxi{cuOYo$_4-biw`D75^3{x)U}-zk?AO}Q{g7$4^Ai03D_`eGpS{&@ YTD_XA42eYL^B>_kUePkDcy{^!0fQ1-^#A|> From 3a4b0e8e9c0a455ca565323045bed57fb1dc4036 Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Wed, 6 May 2026 10:27:22 +0000 Subject: [PATCH 7/8] Route FP8 varlen MHA on gfx950 to CK; ASM group will land later Commit c437ccaf5 dropped fwd_hd128_fp8_group.co and fwd_hd128_fp8_causal_group.co from hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv. The gfx950 FP8 varlen ASM kernels will be re-added later; for now route those calls through CK by tightening can_impl_fmha_v3_fwd in flash_attn_varlen_fp8_pertensor_func's path. The condition is gated on get_gfx() == "gfx950" so gfx942 keeps using its own ASM group kernels. Without this change, dropping the gfx950 CSV rows alone would cause RuntimeError: invalid argument for fmha_v3_varlen_fwd: the dispatcher in module_fmha_v3_varlen_fwd is built with FAV3_ON=1 only (no FAV2_ON), so the in-module CK fallback in cpp_itfs/mha_fwd.cu is compiled out and a CSV miss propagates as -1 to the TORCH_CHECK in asm_mha_varlen_fwd. Verified with op_tests/test_mha_varlen_fp8.py for both causal and non-causal at b=2,n=32,nk=32,q=k=1024,d=128 on gfx950: no fp8_*_group.co LoadKernel attempts, output max_diff within tolerance. Fixed-shape FP8 (mha_fwd) is unaffected -- fwd_hd128_fp8.co and fwd_hd128_fp8_causal.co remain in the CSV and still picked by ASM. Co-authored-by: Cursor --- aiter/ops/mha.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aiter/ops/mha.py b/aiter/ops/mha.py index 434c0fdf18f..7c842b2b581 100644 --- a/aiter/ops/mha.py +++ b/aiter/ops/mha.py @@ -2100,6 +2100,10 @@ def can_impl_fmha_v3_fwd(): ret = ret and (not swa) ret = ret and (q.dtype == dtypes.bf16 or is_fmha_v3_fp8()) ret = ret and logits_soft_cap == 0.0 + # FP8 varlen (group-mode) ASM for gfx950 will be added later; for + # now route those calls through CK. gfx942 still has its own ASM + # group kernels and must keep using them. + ret = ret and not (is_fmha_v3_fp8() and get_gfx() == "gfx950") return ret q, k, v = [maybe_contiguous(x) for x in (q, k, v)] From 425b2aaca200bbebbd5fa4f72bc8096dd7a9cfb9 Mon Sep 17 00:00:00 2001 From: Sergey Solo Date: Wed, 6 May 2026 13:05:43 +0000 Subject: [PATCH 8/8] Reintroduce group mode kernels --- aiter/ops/mha.py | 14 ++++++++++---- hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv | 4 +++- hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co | Bin 39288 -> 39288 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal.co | Bin 39632 -> 39632 bytes .../fmha_v3_fwd/fwd_hd128_fp8_causal_group.co | Bin 0 -> 39880 bytes hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co | Bin 0 -> 39792 bytes 6 files changed, 13 insertions(+), 5 deletions(-) create mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal_group.co create mode 100755 hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co diff --git a/aiter/ops/mha.py b/aiter/ops/mha.py index 7c842b2b581..7e0379e2853 100644 --- a/aiter/ops/mha.py +++ b/aiter/ops/mha.py @@ -1319,6 +1319,11 @@ def can_impl_fmha_v3_fwd(): ret = ret and (not swa) ret = ret and (q.dtype == dtypes.bf16 or is_fmha_v3_fp8()) ret = ret and (cu_seqlens_q is None and cu_seqlens_kv is None) + # FP8 ASM kernels assemble the GQA-shift from a fixed log2 table + # (1,2,4,8,16); arbitrary divisor ratios route to CK. + if is_fmha_v3_fp8(): + gqa_ratio = nhead_q // nhead_k + ret = ret and ((gqa_ratio & (gqa_ratio - 1)) == 0) return ret q, k, v = [maybe_contiguous(x) for x in (q, k, v)] @@ -2100,10 +2105,11 @@ def can_impl_fmha_v3_fwd(): ret = ret and (not swa) ret = ret and (q.dtype == dtypes.bf16 or is_fmha_v3_fp8()) ret = ret and logits_soft_cap == 0.0 - # FP8 varlen (group-mode) ASM for gfx950 will be added later; for - # now route those calls through CK. gfx942 still has its own ASM - # group kernels and must keep using them. - ret = ret and not (is_fmha_v3_fp8() and get_gfx() == "gfx950") + # FP8 ASM kernels assemble the GQA-shift from a fixed log2 table + # (1,2,4,8,16); arbitrary divisor ratios route to CK. + if is_fmha_v3_fp8(): + gqa_ratio = nhead_q // nhead_k + ret = ret and ((gqa_ratio & (gqa_ratio - 1)) == 0) return ret q, k, v = [maybe_contiguous(x) for x in (q, k, v)] diff --git a/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv b/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv index cbd24fcd460..f418e10cd0f 100644 --- a/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv +++ b/hsa/gfx950/fmha_v3_fwd/fmha_fwd.csv @@ -8,4 +8,6 @@ bf16,192,128,2,0,0,128,128,_ZN5aiter32fmha_fwd_hd192_hd128_bf16_causalE,fwd_hd19 bf16,192,128,0,1,0,128,128,_ZN5aiter31fmha_fwd_hd192_hd128_bf16_groupE,fwd_hd192_hd128_bf16_group.co bf16,192,128,2,1,0,128,128,_ZN5aiter38fmha_fwd_hd192_hd128_bf16_causal_groupE,fwd_hd192_hd128_bf16_causal_group.co fp8bf16,128,128,0,0,0,256,128,_ZN5aiter24fmha_fwd_hd128_fp8_gfx950E,fwd_hd128_fp8.co -fp8bf16,128,128,2,0,0,256,128,_ZN5aiter31fmha_fwd_hd128_fp8_causal_gfx950E,fwd_hd128_fp8_causal.co \ No newline at end of file +fp8bf16,128,128,2,0,0,256,128,_ZN5aiter31fmha_fwd_hd128_fp8_causal_gfx950E,fwd_hd128_fp8_causal.co +fp8bf16,128,128,0,1,0,256,128,_ZN5aiter30fmha_fwd_hd128_fp8_group_gfx950E,fwd_hd128_fp8_group.co +fp8bf16,128,128,2,1,0,256,128,_ZN5aiter37fmha_fwd_hd128_fp8_causal_group_gfx950E,fwd_hd128_fp8_causal_group.co \ No newline at end of file diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8.co index e50fe8ee2eaccd90d3176bac5032ebb626ed4732..50cf62c3d25497261bc7b0a0fc37ff044ffd306d 100755 GIT binary patch delta 142 zcmeydiRs5CrVSki%nS?-o4XBeF*7l8ZkEqYV%>Z;#e#7&qxB3Xh{#N~%_c62ED+8@ x=FLC6SF$2;4l-{3>txQj*(O4l9m$9Q4hZKNFI?@4I!!J(=Z7~;V@^L07XVpXFT4N% delta 142 zcmeydiRs5CrVSki%>Vz_Z|*j@#mv;my;(jtiFNbY6br`9jMg)lAR;r_Hk-I8vOqWs xnK%FNUdf8YImo#Auai0BW}66Ib|fPLI3S#7yl}NE>NL6FoFCpWjXC{1TmX2sHOT+~ diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_causal.co index fa65f2c024d25e668f9558484261694362a686e3..1c7b59f8d572c2c74ab45c0088588eff1faa89d6 100755 GIT binary patch delta 111 zcmcbxmFdD(rVSm2%nS?-o4XDFF*7l8Zr*Q~#Jc&d)deW$9v4V(zNZ%_ga%6bMx<~c gskC6+?3*sZ4iTBjwyD$?%=WK2#S7O`+aJsg0Aa5r$p8QV delta 111 zcmcbxmFdD(rVSm2%>Vz_Z|*ky$IR5oy?MV~66@x-Ru`b0dt4yF`JP^!5E>}y8k>Y`w1#TX%h@DO%(14N$TAtIuiO|nUrY_gl(O$Zof zCm{(DQiy>nRVon>5D@VZ2sk%ctZl7At3j()Z=1foZSR%ar?uYh{hc{8VK%HhcJ$tQ z?`%H*eCPk4bN=(6GmmrT%$b?c%z3kn27@($u%C(a5raHUVU?%zwzG+aiPV-ulJWmb zWC%%undY^!+2=7VJ5!h}r=wEj6=gEvpXNBuYU(oOH?#u9& zlmuKs@?=K9z1Ed{Fr&ulsdhQa-Ic{pv*eD#>XH(dzlmkr?Q<3@nUZ5<+F!WFA-PaI z1=sjoyWJDU4-E4uDNJvrXU#>UtWp-37Gr@IjdFO5OrJ-TOb;i__YT$M^G6vW|0T?)k-^OtGNkFy;;v0c_y z+#7H#aa9GBBJjoGLR|zwjrnUe=C6w*|DJ$jX@3En65{ZnE`d{xS{<}#b%6TvviAcN zLf*MTXp6%`R3U8B>R`K82Rq_$fI_gHD};_XJVX`39<2`cYjtoa4hK&L9HmuG<%DqR zNE|MrO5p{q4!X2D5aV#LKj0{jIy0UM$KfTaAYRt$;1#V7PQ~E>6-2rE)bM&7KB9`@ zO|1^z(dyv+I2=gDpgKA9#o;5W7=EeM!3SC${5}o`I5~KI!TyuO**IK8mBOF2I`~Md zgHPgcfKpI3+D{uu96r>=aN3Zd*+a5s4@2Vca0q9Int|=;(}uJ-TtyeiFwGu@YxXcQ z4i8TU9989xGTJRs?5Npi7l#MzA~i7zBr^_I(FKyF*+Y(I z57XlCfC8yv1yWh9+C!{yxQZ^2T+JTxG<%pAhX=H^YNFa&rw#dWxQQ-|rJ6k~*X&_s z93Ifns8O}IP8$m1a1vb<#hN{oY4+fW!-G^5=h|GS4Zb*>L>EO+vxi#E9_r%ofJ4MP z;1Chwa1&h^jha2QX!g(=hX?Ez-l#*QEezM*~2@UJ-mN89^%__N0i;nft!iN?#j}0 zyIUVy`sMC!AH>M^sasG(&~$e@Kj^bDmVODl-A6I9ea1mQEtRyR*hOz`_1{rDZ6q-k zeesBm$uY8h9ub$HA8}fYMPEY1!((LoJR+_+KVnOaMPEY1V`F6d)QIJi(cyrycw9xP zYCdBoa~A(8$(W_tLyl$-)8gq%`Jfjv<~)N>!1RN9`r~ zn!{PD*~4%)GR-l>bAhQUlA5T!#HTr&pk@!XnmyFT z;UTI>)O(2#hqLItMDyjQu|;zXmvy;$Jfqa_t@fdNyR^bp8FZAmYhA?-dTad$#3+!X z(f~D#OzThJ^tekalRM9|(=4ZrZCx1^&RR!_#~F02_WH|Z=h8gFMx4Dn-mzjqwiCnI z{1YaXRFpX#C98`aWyQCT&vBIaavVj@>Hr46v7DtPwUe_mXTF%>^Sf)DL05km=i($U zWcXcG7$Fz6xUFau84L$A0;N8`qsUub87$bB5m-}E==Hptefa@olow;%P`@wS4tXYH zwX?=m;`gG^=(<~#5q&i-f57dnY*&UGvO$3{{?i0knlL(08DA-<$pVMc5s}LHOxaG; zD{?KSEy)6%lR#}B4_`Oz5t4SR3DZ>QkJ^i@HGeiu{kq8|-+S6?$;cgQ-*-Z>dVQEc_NnvRitSKpMp@xK{ za3lnMf(29`Yz))!uL5)d41;YTNj8Q%0O3o<@DVT!reVViNFRnyQjOu)0V~)C zhGEaA^Pv}jVbBL!VHZGx{B2YRJE)$R6eiew0+>cOV-n(|`jtuHBXs^%N#QUkfPGXS zmJ~hPk81J<#&iX~ z?rtQ$QH{Q;G3#Ng_XpCu0$bcovTY~vr?2-1Mn(LA)HDpT-9Uor&B5w1uaL+aU4h>v z(R2;I4P&Nv*xGlJmULUogYN0|wk@`ZtzEDXViRn_^m==<{XzR?yA4{~W9~>>r<*ho zQjfeFd{yc7zCb$lFJP7G5nIGpm1@BbdA#bF8)?#8pm!ZRCdEn9wju3t=$P)0i|Uc1 z$K3P?m6qRVYp2V#*|E&X>~`ueGJCc_mpgqu;)&SXt!c2?^8}hFV%!))$worq)*i0G zwt0bVfv`|m*pP3_7ZwSN8s^yM2y=zG4fAaCg!#h!hJC|~VT4OQkU8dRrn^8C6WJ^4 zzn1l@BUelNtdEF{kn~fsezQ>wr5n3L0@>dkAx~i+Ms(yrw?Ga- ze_GZLcb_CjV0*T^o;=?@BQn07rkhF*c26bGbklXYYw5RyyO+HvD7G2cE;Nnpx|ALR zX~J$%-!sza3ytzsnY&2h8(lL|Ccc1q>Vd8qU5PpT@x(0tTJji{@lyAchQU4cg1w2B z>6Av*bc<@bO_`n|aM`wY>lFJIdzQU@h;4|FDx@|Hu`_a$)Aha z#?mhyV{2FJOK!IH$FU^c*1y~c+1QOuB1Fh0&`1VH220xl%iC~0=0(V6dHxn@erhCD znveC_Fcm&+m>(hSa+$^|jmu+Da*mG<>!+h*w)YqW2c_V?B*H;(1>MjYRxeOpt% zAK2<1y&lJ3`#0FJ8Fg&Tl8%i}^Ha}`f1DeF>Gi?tQ5$gV|3E(W8+LqajFhk5){H+E z1C}Gju(=Ayu0Op2F(`SFf6VJj-i{~6OmDJnrstzBlJ!Pc6Cw1RM33z;zUtKD)yLg} z?*m@&{T^}_@+ZhgkWU~aB=`~_$&ewCG{`W>aL7oA1u_~k7BUW!3CV)wK&C;gkX%R} zWF90RvJ|o$vJz4NDGo)9f^<$0e6`T)a4f#uJ-soZ=fi|X96MWjzJ~LJZO2Y`!{hEz zk;ix9+(XFsO%I1&HEj>g66yq!c3zY3CtrW*ZZuJFoOU%e8Em&C^w@5(HQ6FFn}tXu^1kUrXg$iY(f8yS z-xaMrAB$C~cpN11u^32w;Vu&Ct=i}A4Ls$ZEyr|*n1tD?nBJ6iC8qxQa!hnS^^2MY z@{e~MJ^KeY_Jm$)?a3mC&kP%E zG)i$FkmHV=i<=mwJfkg)W0gE3NAxDp^&lK8TN=0ZN-6EQ0>(_5HW&U!!@so7IKR>D zK-WpINNI<4Fb-GJjvH}hqU3iG+a}a+I8#bHvX9Y1(=L_cmi*#56S_Smzc;dYI~b*S zY1)w^7ALCrmGF2LYagQ&Yq1D$GG$$&0suOqMQ_Ka^!XawaRg@A`@0L@)=;0jGddz-iz#a5^|0w1QU9 z2HL<}Fc+K!&I0qmJa7&;2b>4a0~de`zdW#BS!Ik+5L0j>a7f-AxM z!TZ4iumCIs3&CQr7%Txxz%sB5Tm`NIJ)j4y1S>%w=mY(rAEf(b7FWh+bF27?oQI#p zRq{EUkDtu>`6*nGpUTzn)3{px4sIcpKNq+qq^wmuul?avS+s zTq{4D+sx;2ZTy{FJ3oip#?R#*=I3$S`T5+V`~q$VzmR*J&*wV$McgiaF}H_b!tLdk za{KwaxC8t$?ht=BcbH$!9pUfcp5s?=FYxzrNBNap7k?jjg1?^=c?Z|c7jR+T$-TrE zaxe2m+;{n6?iJp}y~>wxr}$Ft2Yea#I`8Is`BmJTd^z_c-ow4aS8zY(E4lZ1FZUDP z$Mx}5+|PMG_e(y&{hAN2Uy~^9C-rPU5wV|$*iTNHgwVCMYs?~78>0JA2E+>CAU22{k_(v$nFW~*$%EVpnFE;%nFpB`WCi42$V$k4kozGHNCCtNDTEY3iXkpY38WNK2601H zLCPT>NCl)4;)VDiRS-WU01^M~gB0<1zfBSU{r4&2hyRfx{^!{g@sEE<5&!(B6!EWr zNfH0{QHuEYk5k0|`Xoh+{3As)7|f!{WEK+>&ElXzW^wRfvuHM(#ne=@c*PZFap+L9 zc-2*A@#?G1;x*Tp#StUS;#a<67O%U`EZ%T~SsXRWEZ%gJSsXLQET*TM#jk$VEZ%a9 zS-kaDvv}KWX7Tph&EkX!W-&Y4EKZta7AH?Oi&LkX#XIgWi!)|m+i_;mZa0fFXPU*? zv(4h2cbdhybIs!X`DSt9LbJGNky%`_#4O%*msz~~ZnJpLJ!bLVd(Gl~_nAe9!z?=!8vaGv$Cwh$}8+uLvgg#}pkxYa>?Ew(_^pKHE zf^J<8Lbq)&;tADVy}6q}=(DyM$z5PHcE zji_>W^_HChp|5(|NNmtO?}5-Oe`+Ll=)PZo(EYzL;(WTXhqetuue3^SC#nZ&{ps#a zv=FI|8f{_q+|3qN&yBUPdhTm94^2CAO>bfctLMgbuzD_|gVl4H9ju-kPt!KiI@En- zl9l-#W|iwtt6YCtmHJaoJ8lgwu;uzQEZ3i5x&92x^(XbaO^SDMQh^+AfgEpv9B+Xt zUYd5~n#DbdAP6RYROb+CFa zql49RnH{X2i*7T?^`}*?Kdo~8X;tb^b(=}9Kf`kU8J6qMuv~vewVC923*>kUCmd`&9Avx0zV~Qhz^FRGVo)pA&5}Ise{({tvnDu4A{mVORO6$gV0k zK@+vXSApr>|Ev9Y8>D``gRCE~yDGJlw&l75sh#M@`?$M0wUhRbb+1nCME}@ZRZqL$ z3OwU}tNK~@TdSXU=gIB3ACUwhPu0Kowye{8Ahhr69a+yD!t{5c2QRUIPieIxaGPqD*pj19`Umz`?3-txk zw3a(rS<$k)P+uTTn>nAQrR6GIs4o!v-i7)AW9^qjzNR4tsg0`l38L>(?-SJf1ob{a zy-!fxC#c+q`e*JNr1huXFSt?f6TDHX%QHW#KEVS+&TnH~PCN2KeTC{a=7stSrS_HF zS17fwF3?vfwXald)9?lQ3Z?dys@)oQUm?35`Aqr=>Aju=={_QD6YKX^>HTE={;Ga| zRlmRbf8+kD-j@2}++(GEZmcgxb$?avd(-<0qx+{W)>pU$eTBIgYxipEfc6v}U#E{# zlg8AsK0vAc`nil#%Wba0IM9Imc+>iIw4L=GcNxZrtMu|!=(knr6&|u)t$Qm;i(0Sp zkq6a!z)v=+_39wmq}Es0kT$ixrj|UU)*o0)+SU5HI1z`&#Lvu9wpDI^>6JU&#U#FkCQI7{zL~kuGV+&A}7@PlY7XEYJJ~c5?1R^ z?Wg0@-E_!j|2}aL2P14Ywe9IclFhERJ#$#H<*9AY9+7Ocd$-@;^Uq1PIci(y3zBWF z+IH-yWXo6Ej(15m3>`h^@5K|6ZL!++FQQ~yqPF#POSa`|+sUwGyGL#N_Dhm&h1&L= zmnGZ1YTNg|E7=Ovw(sM~4V0BrZF}ui$yTVgojxVmiqy6rV&pgSmZ@!TU_>`;Znf=9 zuVh=Lw!QVHWGh$O-u{tf^Qmp`z9ZSH)VBA2EZO{O+rPdq*#c_YPk$oWYSp%%^+~oh zYTGY)yK)|<7D-5vidmLFX%W~T*KX?kCT<2 z6{62UN!M(jVgL8QXN&Y}HvO7Szh(=#^=r0(Tfb&I^M?LxqyB88{%oWEY@_~c0$aI#0jO)8ApEzr#jIre@~{hpnE&rZK*r{A-S=boK@E%yb#mecRq z#dXi_D%`VEzY~g1cgyeC?#BDIyQ|ijx@QJM^xerq9ea233A{V`1m2x|Q?S~6!c-Fq zTAw`=tTqj=0c*fouohekt_ADBI*Pyhw60c-#p!A7tdYzAAv7H}iD5o`rp!Oh@i zunlYj+rf5l8@LU87fIGm)!Nq!z6EyOFE``G}FM z1qG9dJODNfGLf3j5BSxczvb6iW8Z2BAZlt~CgCNZ)8Sk#%76 z6(+)iEmxUH2;6v$i3nipS4^ZH+ooS&>omYmae?jrz$d=ck=q(1J75VneHIc7_ zdl#C>W^n%!6WIbDxZ6b9z(e<%$V1>^hl#X>M47Gf{1jPVCG*e!hpTaDxE_JvoOT3N}uixSHcpTnXa{%cAYq$oBp80YBC6MrnEw$yW+g zo_DKkA4VSzQ~h$J=|N-%zOSMFF}H)rsV23RkOh)Dh#c7(ZKq>BX`I3n(e`55-ZsGg z4z(MTiGUAvsDE_h$NEm$F4$vzr;KD&dV?;KQM{%SA6z3DrIpngWzIkuk^fVB&`&b_ z-ePCaNiu@2+Mtx$S>Y~%+2!$jm5=h0j3RFZ{V*M&A8!jf3&AqvW>bvB<1BP}9GMd) zWXY4abK{lC%t>61Y#l!{J5!#VV82~SaJy}?UAAUTwkwm_lWa=biP<^lCMRdc@j0|O zvd}N6Idbn=z|FsNz}}?ZGM6_^_h)=!svZzR#!7Dy2I2Z=ik~X3_lp=VRr&f8$u5 zI{FZ+`bQJ1e%Y9&dFeXPpK_cE+n7$DS*1Vae6ExqUU@G4rsF@gDf!RJ$N5<~f0|M; zu;k~hKYfKH4JlKeuRzKFjAD?fgs+gEF8j+)mA2y>Ov#rMrb4CXSS5aC|G6JFC3cpB g{XhFKqq6?WJY~6xPK!5I{(?tXik~S7We&{$-@$pht^fc4 literal 0 HcmV?d00001 diff --git a/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co b/hsa/gfx950/fmha_v3_fwd/fwd_hd128_fp8_group.co new file mode 100755 index 0000000000000000000000000000000000000000..90122e1840b3824b1206cc11244ebd30160e8274 GIT binary patch literal 39792 zcmeHQ3w#vSxj#FbgpeUMC|0nji-MsAV}t}Gcp0*2X1 z2mv8N3{t+PxyfQ}YZY1zTD5xH+&-+g_1<#(w6?c<|7Xrjn1$t$UA@10 z?`(cw{^$GVobP<|&6zphoO5O-IDO6x&SbJC5cV^Z9%53{6s&6c>|-o3Kat9^NizO_ zg$y7`5Yw_&w)p}Ksme4=R`XG@R;Mb80P85mYDm%;Ps_msLPj&5Oo_y-QpRhjFGp?7 z-p00PzZ8jH#*)`~HKEqWfxVJ)T6t~%7XB(wMECDX>(hL`0Oe?Xef`n86n|2K>F+%J zB%O&h)O_C$;&-vl0u<)v%*?x+kcPv~ilS1#Go#Gqt8|t7A8CF%!|5yWe@L2}4re$E z1J%xQhqJKI<@X=(xtv9gl|FaCg)K!zKFAz?ud~p#x1`)t;4FVSqtaR7dfFTCIXwF^ zJjKO+SAaa7;difgB_GVFah6xR9A)mxqGwrlM?rOQvCFrP72EA~7HNf&qZHa-u*#vR z&@=^Bd0o5RW5=9t<}-?!p33r7m$kA=t1vA}g_pH*aFjw{gq3BQm6j+KUe?NyQ3`z# zR@P`%W=5&-vQ}nCDRjkFUSY%BM;m6j-=!I9jZ)`j4b6>G=%R+IO?$7u&{^*EY4zqY z6$x5Hrl-W^ROru+#c`Hj8fW?CF_eD>R|;2^quf=gc_BXrEqX5$#aX^I&hoh0(FqSwuqR#x`{Pw`CPQSKBE0ZYyb3zvRUpTpV87o{7I9@f6^cPigd<*!SHWxXDmWE`0(1~%`b)!`G3bbJ z!`tyHcrRWBAI6|Saf9yS&=Z4>2sivPUIib=tKgFu6yW0E@dkP?4rgOf5#fbD#;f4d zcoqCT1_khfE@(e(A~ERDyWzAcA#M%HacdY5gN8%6I@FvWN1ryO#h@zEA%o)9FgR`v zLu1hJtlv>p<|w7Z5=HI`?L;|kvc#Y((j&v;)-W<|4WnbwfF@EC#UYt7sETw*R@@r0 z~^swXTI&V^9_8kes+Rk z@)$HA(5TVHTc=I=F(`?2MN!-uO5@g09)kwO73X5E( zPuv>z$F1Q|Uo^xv+p?byy;orb_*{&3gw@)&iJ{Y`jwr=0I(Qz3v1Jj)hN9GZ*{Ool zrq`kryQs11WC1QdxZR03?XN^Hw#8sM$`<9Z+joXG;ZYgyxDT`<&d2ySw zIBpF~W7lVcf zm*`uG6oazJRucE^CbuDO8~Sv+c|4=U=c)E$c)O&+RT*#;yK7xV4m!2|L&8bqsIou} zE7N*2ILqB7mC5ZF$Z3(&TuVnrg|pUCT<#1wR(gD8s&Zu;VJptwALUp!Z=4g$*?eO& ziz`Z{q@=iZ;<(J|FK2jt?iy#n)m#5L1IbGnK35f%#zoY# z715B%bTGqT;`KQSJ=K+g{CyezRTTxE^7k|PP;*9E5mpNID!~NEa~Uh0HLhZx2X3N! zYt`29)wq0qx2LjITVTkR0bcN*Zob-#)qvW1N;SuAz=3oHq_!SYmDBthUC(5xbgdlC~6)Zj1Q!VUI~4^1?e2)m)O1H0#t{123NYCTzONJb*M}r?hpR z7tj6W*kTHsn!;hacC@RLBcV<+IoU}}Ng>60E-9oegEc3GEL4z?6w0L7j|=4itiXKA z_vb=hfCOFwhJY4lF-eiBszS(XA*Ybor+|uqQ4TRLA?0RoidcD`5 zPW4OJrFz&F_Ex1@&>)Xj9djd3`UdbF$Brp>(!4FmdmKE{y>?MPeDs)`B$$X&f1|CH z)@!q)&hX4usxLfqrbO$Vx)%0??XA`{$n3ciEfeN82I1LoFx1>7G}t!Jv(1y{OY+98BjrgA&=_879x54Z}Kepi?4;z(1?+iL67jsP7ued4t2e zReT3Ye5+$ReB$-qyKNk0)k{*OMnu$19za8v1wDOZIitr;{3W=?%JciyP`8b@x2DnEcw8@7i@lbUC}k|!~osN{jo>%*{9IWEYf9itm;ub}PnU5Y$x zTc_Y~_ysoEu+MhnRoZ%!ZL@NYHQF~T?Y+6mjdMA@5$E@C@7B~G__w-;uf_S-`b~Cj zMw}Znlyl=^dg|Hvk8499y*^MqY(38XAFAhm!;Wu{P|DTYn()VLKs|~Lo2zi{`qCR< zgH{&hN4%+(ZF_3O)OEJa^m^1mvfk=gM+m(p(Q|u*w>tHB^>Mf4{aBQ|pMcJS{s{UM z^mh;mO5OxeGH3uO4KxTe7&H`Q0SyO@1dRq|g0eu_peZ0LCZeU#9Mb7xD}H*lS>?bzvVc+x#A{NzqtdkFb~`O)C( z=Iy~5Qk_K7?(cbnaMFx$okXvlls~2NG`6tnw+~`G-9CzI42}vd z;WLB!bDU!L0oCsCIlBp`lo@VeI8rM!bVzps-47x#zfrrbS1Wl(=QGTtd2^tDIP@#~ zjB6M5G2JK0qU0Ua#&EEfcl3~}6BWG+n7^ssP^OZ1Xb(dR&AV8&ThWVWNa(Rr^xn+u zZDXj|OY;sLvM^EK4nkv?tvw7CTZ>d%nXg}c{{#4tm=ydf;S?N9%nDu~e6_iY<{v$z zHBquCvZ2y21#gsYR?s5dqTtPmL{g9676n7*hTutZ=B=!8BSWsaLR)W zrv@2X*D|!NXPC2z;fyT|bGI^_^%%oBPcWRflVRR&h70#GTzrt>lIIvMeUag^V+@zS z%-pYZ_uRpMd~`)?hT%- zT6>+1y#D5Q*CIfI{3Ic7oiz0~(LV3`2D7EZ>)d<_{2#(FLM647(6g|REMX%5!RtY~BeZntAzwnV5TDvMyX(#opoyce>GTO;WvlP6( zb`>uRwL#i`YZQ&^*w5P#$OjXd!43Xffz+&=Sx+prxRDLCZk*ftG{r2R#6Cfbv03 zPywhAR0MK?ia{lyQji<80#pVn2UUP7K^~A7R0Z;Z{2=+?KTeT<`%PqV- zVgxUzr}OgHzs}2}M)C4(xAF4rxAXEHckuGqvAjHP94}9pz{?XS^77=#ynN@KygY3h zj-9~Eb~`UmpU%rOXY%r0ck%M<*}Ob=E-%lY&&vxI@baQXynOfFynN3+ynOGyynNq% zynO%tyzFrBveU`Sg@wHAa`AFW3HD_*+5*lQva36*j3ZgrUESk792o~b+YbakX(dO- zgP-ys5d732M<#%`t_6a(t>^Ic>8|daO+fH7ws2%3_}r~P@UtG{$RzM{o&bWMx06GB zzpFcMHxT^7eH?~3c6Bd42n4_6IgZ>3e(8%q@XL;IWGeXOF9X3p(8ZBy;Pbx&1Yhud zj#$ALy#WMY{38xs?ym0AGeGbw-sOl5eEA1J@RdK~h#kE57eMg7Uvs#gZtSA5LGZO^ zB`%_TfcBs6?nDbw`l#U+*3aE)Vg1}l3+v~;LCes*LsxYtwy}P0bQ|mEGTK-_m)XYp zxiK_vBke=oS0`DS-a%Hi|Fo+8r&a4e)x4uu;pSQGKSOH&8B+VtklKGzz1tLf7bfMa z_U5bh=BxJR>+Gd@hpt+f)T7$lquSe}+S{YEcaLIkYZ4JyJYr!uIQYHhF1l^>s@5dy z0v3PdGqgw~=NOHlx>BOVP*Gw(VrMalg<)h2rHc!#i&#vO&+x+dNsUpJILFECZDV+0 z{G`UOO8ivC?C)WCVf>_vHKjskYa7Fe_$lOP_VqBlFlG{zwls*?6#Oc&D>#_sC>R+t zIThJZsX)OSr9uTQQjvm2%w)t&M$BZyOh(LP#7zHuF%#?OhFe%acdLcD&b!*LqLoGY2sKeei)R9J|wE zbc%YS8{$D>j9@lxY_bf2|xKI(%@jq@q~QOEhH ze=ar7NAp_lV!oojyVN)z&6_!w<)wa=E;Y`Fc6F)oy=ZNYsP`+xq{N@bcpt_kjqyHX zyw4c#GsgS$<9*s3(?9dLAMHPlF~5z*c;C%RU!M7SjrScGaAAzqmpJlLhAuGmj% zjk!}}?)3l0+^G>ueR<|j>3A9&i_y)Us^e#66(Q*`Z| zu?9_9FUQ6JmH71wS%a3-RE71Me)sWpYuC^?>$~n!tnJqEWh*cytK%!m$yz<{sU#cp ze3h3xr04xUvQf`h2goKpzp{q3==oK(8*&1Aoxf24&xqvyA_k^_4Fo7>1i zJ^$FFCp2}wUOg`e)leNLeD?FhrF!k_w6Mi zJ^##px~|+!mvi>EiGw&9A+zaa&mK}_cD?Mm!-_0dFMIxoBBR5*z4~5!L6ObU%i3R3 zWV7|MV@DNPo?dpmLy=)w=sA5apHO5A^|F7F71<)atgBOzE!E3Th7{Sodf9hgQDn>X zvhTjC$nMk2zW+T%mamun08d=NS5Cd`jn@@ffnIj{lp-tC%YK9v+bCPAm%W9R*pRvP zvNPR^Y=vI-&fAKtOfP%)$BN9Wm%aat#RtsUoY@%l@rL zk*(6pe(`ffwpuUy)h`vJoKO)8&SYr*Wu?E&y18b~-HP*lyYhb^mYhZB?cdxMq zR(TGHKI5d^vwe=^-{(I!WZbhE_iV;Jo8N8Rv-#b|J=>YLjOP}O=N66U7LDf?jpr8s zA3nEe%)Q|`OFXxD-rDfX&Ako#E1P@!UDA1TZ^n9Yy2gHVwD&>$Z?6|O?(>ZMJmWsk zxX+{Wf5v^Dai3?rpT>AUjq!e(*xpZLtg-x(ud!6$_hP)CMqOihrSGS~++2aW7}w%Nh4_#=RVS_op#uXUy3db9Tm@T`Y5U#=YE^{9ewO zvx{lY?i$S5>E8)O)1B)3vpezb?9Qq+=FaJXAbl^gRL9vkxE;71_&D%!;11vp;FG{7fo;Gx;4a`U;2z)};9lTf z;C|qK-~r$P;3424;9=ll;1S>v;0wSPfB{n49r!Irs)04X<46s#_75Cc30(abj?@C{ z{>G72Krzgb)j-K?CJzD|`k6^h`^VzS_TPxB+s}#*w*Ov~PT;*|X>~mv0h83GrE5)6 z9r8EwX0is@bd{Nizzx@!Nf5a4Ix~@g&0jNx`DD_GvsS#zijWXlA45i(V-fkvM!0ltr_y$aA_u~`H zWCL)=WHWgP_~bM*zC%;m-DWqFO~73Ak{Bv`Wsn&Z5qG9 zBsHVlp4n#dFmUgDGuaH>zsO9s01w<_CN022_nFBfz{3tRX$2lBG?T5s7fO)7eQXs!JU1%3_bB z*yD3J%gY@ezr*YE6uZk^{@eY&!rRNsYbtJoWQD6RpfHx=tjz2Q#p4UI#$*>~y2cle zn=sB*;4I8`W;-X0$trRd+%aZsQRaA;>XUQUpG%pPWIvKM?VOCJ=&L^PVI@MAD2d*G z*D0xfq>(=0r~BQkr2CQN6*?i3^m|=Y G4qW536lIlkm-_t8n5=<@huZxt=Q}dT@ zjg-^1o>YJKQ<3r_Ri1mE{Bl@87X3>XM^*@?f_Zx>aElc--{I7R(|kfEXNrwql)L3{~tKDjq3ma literal 0 HcmV?d00001