From 085934f3935c833dc1541c90922447699850557e Mon Sep 17 00:00:00 2001
From: FanhaiLu1 <fanhai@google.com>
Date: Tue, 7 May 2024 01:42:02 +0000
Subject: [PATCH 1/2] Gemma sharding and test

---
 default_shardings/gemma.yaml |  10 +++++-----
 keys_original                | Bin 0 -> 33978 bytes
 original_scores              | Bin 0 -> 1476 bytes
 tests/test_llama_e2e.py      |  19 ++++++-------------
 4 files changed, 11 insertions(+), 18 deletions(-)
 create mode 100644 keys_original
 create mode 100644 original_scores

diff --git a/default_shardings/gemma.yaml b/default_shardings/gemma.yaml
index da57d36e..4beda7c4 100644
--- a/default_shardings/gemma.yaml
+++ b/default_shardings/gemma.yaml
@@ -3,7 +3,7 @@
 # "replicated" to signify "replicated".
 # Integer signify axis to shard: 0 <= shard axis < rank
 
-freqs_cis : null #  torch.complex64 (16384, 128)
+freqs_cis : -1 #  torch.complex64 (16384, 128)
 layers.*.self_attn.wo.weight : 1 # 1, -1] #  torch.float32 (2048, 2048)
 layers.*.self_attn.wq.weight : 0 # -1, 1] #  torch.float32 (2048, 2048)
 layers.*.self_attn.wk.weight : 0 # -1, 1] #  torch.float32 (256, 2048)
@@ -13,8 +13,8 @@ layers.*.mlp.gate_proj.bias : 0 # -1] #  torch.float32 (16384,)
 layers.*.mlp.up_proj.weight : 0 # -1, 1] #  torch.float32 (16384, 2048)
 layers.*.mlp.up_proj.bias : 0 # -1] #  torch.float32 (16384,)
 layers.*.mlp.down_proj.weight : 1 # 1, -1] #  torch.float32 (2048, 16384)
-layers.*.mlp.down_proj.bias : null #  torch.float32 (2048,)
-layers.*.input_layernorm.weight : null #  torch.float32 (2048,)
-layers.*.post_attention_layernorm.weight : null #  torch.float32 (2048,)
-norm.weight : null #  torch.float32 (2048,)
+layers.*.mlp.down_proj.bias : -1 #  torch.float32 (2048,)
+layers.*.input_layernorm.weight : -1 #  torch.float32 (2048,)
+layers.*.post_attention_layernorm.weight : -1 #  torch.float32 (2048,)
+norm.weight : -1 #  torch.float32 (2048,)
 embedder.weight : 1 # # 1, -1] #  torch.float32 (256000, 2048)
diff --git a/keys_original b/keys_original
new file mode 100644
index 0000000000000000000000000000000000000000..e2dae29f0e10deeaa7d2a7ee022f84d54b11c18e
GIT binary patch
literal 33978
zcmeI0d0ds%*2fPZgD8me2r5oEfG7wkXRl?7Wa5BBjwK<Wl8A6PYTUArL2+VoN>dZf
z%sF$GXRoEEX5E`pS93~DvvMlatjx-HaqITEdf)qg?&tk`KYkt$ob&8;_Wtg*e*3qG
zk8`VHvw3;h{_Uz~t8SZ+lAV<>$&orPH9aw{V{)Q1F(hL`T5Jz5+sHqz9M>f0BuCQt
zkc7$3)U+(`1V_r4$*F0{3C@)CtVxc9DV@DDZ9zf9JtcmYJefEyCDY~iA>KV=(<UW4
zJ4Fos{eACD*WoVmSZJo(a5s5P%9xy4rJt?4Gt+%woGmEMCC)3ZcATFx(<4aUjq{7k
ziF0vgdd^Bpnv|B7lH^RCl%C~1z>%EdNJ)-KO>%l?dbJ&tS=E{8Jt(tUNPL{9*Ms#_
zi`Gfx$~*o)F1)_2))NaKlkH5IR2ew7hwq;&{~bVDs?(X4QW?BJU2W(gfh+I-9K4P#
z>WSc$R6B;o_6UC>@;?dx->>fgTIb;*<gqK;U=z)URjMy9rx@tXUN8jGFbI6Ox$eio
z+?F4}1)8Q7>bd-d3WitE4F=<4wM|XscOh6`rffO~7ilNd!bEa`M(n8<@-6Vf7eJ!Z
zGyERe_*K|OzuFhlE*?d>RD&<jSZ;<xAQv`6Jhi|U>{{_0`q5kZc^O}ii=l$TXgGS}
zVYo^I$d5CjJyhouXuuWp3RlAt9nB5o^91Ff8PrTI;5Sun^c*apMKlh!!Fk=01F;P)
z#b8W;JNzxV>*3r>57XmeE_|u~YITt{c7kk(!j3qe*HdRMqYAhHPvLo*#XIeXD46HL
zN@|B&untA?6<r%wLJe$!+thbf2W(9_6o!L%Ggko{`pZm9^eAXa`P?5XU>d)tX35-#
zayUM;K7}>doWG|Bkj0PGWw3M&wHkue0{eCt&h4y0;7_TjbspF#7jvl(Oa}+~;(D_4
zE=u6>;DLXm&5%)Xgx*jN<-%@wi0k1}IIbpBO}-3kv<p~}fgA0+seqPKN0#T+Cwd3?
z$j>+EbNWQ3!2oQhbNN?zL+yaCqMJZt%z*XwXE~gGI2KxRL&$`QdatSn>){O@4~<{~
zuVe?h!_$~W*Qh40P-kH@?xY&%f$s^AE%+SdVMl(MGUzG3NgvtkVmADwnm~j)3T-7v
zK7eZMhxKr?>{U<Bu=2ngYvFuaqQAG6t3+JDOf5M^7_ZP*Ay0e4o8*Frs14e&75eiO
zj^Lwsn0)yleM8-$4}7Rb;{v>5HRtWni_fb-c$>Dthq@6xP4WCLeM*^nBG#oCew(Av
z!n1k}uUBm_6z@YbD8+U7G38n#?W5tZyjy*US=<qy;uW-jJMcERMyn+Ioc6Zt32T%q
zmvWiDO9Nq+tT`51;ZR0+l|G<@uqwK)tbBsxaAWMtM^$fVNGWs|Qg|_G@P?DTNO#i}
zsuR288lH|$Y{dicHI9TbtB_~oIJ8LxX7XE-d52*?+|b*xknJ!M8nTm}6!4Gie?+l3
zgeFS1E|T1;&F<=iDo~H0jK5S~xRySG>HMy$%S||mWmkYHgh$|6YYCMp8()K__zM(q
zB6h;rbXwiiJ#Zo<abs?di@7a^=%%nkmrC`fvV)Vb2?ep2&f_z53XpC32A$UB>J|(`
zr91QAh4Xjd<gejfh~rh{iEa2Cjgsog!G=-|`8<g}q|sQLS5g>f(OKvW`}GY`nq4Xg
zgZb|e4Lzxhr(#V`g8(X{c*<daT&9<(5A_Thg$33=?!p&!I|{Vd$1c%JtuFiq4Zs-i
zl<M)K5%3XJ<uA#JF-+79rovkgf;kvZiCC^1kSBerUgBD$xKvqGg@@u&I)g6((Y%VT
z2zZ7TN<RA2Y#O3+pbf^TavjVkph)Jt0hK+F+TwVO)>Wm>7f9^ZvL8DjmWy=|Ho%+U
zz<3Pc`EUuU(tceIJz+Dsa};cVURDq6FEzZ7U)I&JjAr6&o`-8_Fv$0C7pa<Z&Q)_@
zU$@@$8m*%4q9eO#sH%y>*quwri`^?Gz);aNe|}MiVicCa0^OJna1B17uTc*GTqpVR
z5!J#FyiHfs-$Z+VgoEl5RA2)dr1Gto^;WeMYtSy}%BLw1uhEw@26Ff=`SaFpb?KTO
zj$v3|DrycQe@;hfrb1|{`*0~8z*JpZvZX)Whiv*-KNP*$qc%Yk?n>Jzi2T%HbjKf+
z7Y|nVp`Pr5zvOHQMCq0|Ob0<z?!*MYQ9G#5`Osb8*0Dgmoi3}3ut(Sx=>;SNz!&e)
zC!%j1_+@O!S122!bgpH=67s@<)RhV2IFmNnv#2$9-}@fiqxPJxuc&gVi_O#q{sxmc
zMoy7k*anY)o9N?9`~!t!DQ=;+)F@m`#jph2F}LDpspX6M1zfHkz>oIJdX7C!lxTxK
ztKul9q8Q(^d~paSU@z(hzO+s~gywiu-GpF#WW~Vya+Vdq0~pP1wWoRq+TukkLlmYr
ze+5@$ZeM_=SdM@Za<0sjeAolEIbG#S?s@T2$+7PEn>^h?)y7@2Yp9nC7XoiyNcUi`
zJsZwbRdpOzNBi&?Y^sMq5YDBODn_zBgkPlka#Hw{JNRK^oIxvKkZz3qR7cFGJouP0
zgg`Jh<6%-2jW7u}Lkr%|lQB+a*<X~jJ1nJEa!R-3Q0@Q$+?@w%dvvjGLY?Tcy*{sm
zZ>R{G-~c(Df1thK%lkyD7pe1b7;DP-J1`uYKr2q7i*%2E1y|LcUPV7h<-Yb1IWxLp
z1P9V?U6Y%^LH#p~!rM@T$6~(Z9@Aeq6nk=oj)E{1&kYoEb?gQ2(n?*Gchjp>o$~b&
zYKgO~TZFQY&tOmZ#R|aHlKJzvDxRgU?MYZoPW+pc1{r!f&y}@yf&`3~O8ZoAq`~Zi
z8^8shk#i$lRByi80BhiVoydWlF175TTHtAz1*=7ks$&(YDR-W&qNzC+l2bDG75o*>
zz$JZ7jp9&hf?vZ-oDX4Kmo`v;$-_=O5<j4=l*a9~H_pRa9LCSejO%cSS`O`T9DEIk
z)A2{WhwfT$!fIGUTdjMNZ$q#h4~DbWUFgQu;d>Z})x=e#fQIwxH>soDbXVOI=g>>e
z`vMw>{rI%?vgqd(C|3JLaX*DmWe>+tTb@s?DU-ah7bma}2Z$%?qn60~<7f^gh^uHs
zMY|)h92UYgeNyzeLbafw_%XO*Q=BX{drSAj4x(Yt(ra7~-k@c?N&QNBqMOAyi~L|P
zOjjS+2kR}8dCT~9oJq;zeWt^6cv-F0;n<jqWZiFrm(=xj>m6DQ--s(HMGp!PU-SYV
zp<+?7DzaaZIMd#TmgyjF$hTEh%%SO!2iu{&oCGdX@rl&P+M;)hN|)0kRYI=f^tKBX
ze?Fon@(;>`Uf^tOD_$`lbHyo4$40UOe{2h9sD?hH$5JY{VIMjPv&BQz$CmQlKZwGA
zF7>jI7fCh#NKy2TJxF|KFe>N`Wf-ZGMQMHopKcp<1h1uIvaVclWlzHcI;RWe`)5KQ
zIalWM41LGGm&QUk@5HmXPrTrBc6V4zbFFFgAymaF94{(!k$U6by9M%C-a!4hhWeSt
z!*#d^;nW0baRBa8WqggMh}!q%B{&V*kuOf5nG{Ne(MxCn+Heq7<CF9jrt>shu48dE
zB&$%&<0;6nPBbo$2Vx#2<6ZH&=jke)fG$u*%dj<Wf|2wroWg8tEw^T0Zif|6$ot_q
zy-p#j4t+!=;twsgQn`=|cA=}P6!+32x+&Uu7do<yy=lAMRi6@_?*yB524AzLQgv<*
zQ+Q`Z9V~=h@UwQ8^E89{>I90hGU*n5uKYNi8S3c)m@FFP!g+KAYG45L(^X{cLv=ko
zCfSxNUi1{%sg`w!UC9ZhQj<m80DZYWyicC|K!tJ&-HMK-20TJseFc}}8Hl9g&`<V4
zRfLFF2<NSkk7+Ol_Ucto0N1Gnzs7^GzGOuz1xm+pRb0b8i7=0AaRINRLlnvjcpOCI
zMjFSX@hJ-A{kj*t%o~)Qr>JlAw=mV}D~{#{w7|D{wVI7NRvM0_o3vH0fL7qH!=V_X
zcpBH&^C^&K*;j~CK9n7}MD@5Q-&ZM;Z-ej$xB;J21$F0E9L{a29i4&i$xk%@AS@8y
z6(&wU7&fWPxSdM)TdCn0(2i$dH|a|fVUVyWryblx&Zu-*b2D5fnm(K2xR`gsY_&*N
z<Ez#|Sc%^3B0KPl?nQI)IIdNByj0GI5FQDkc!dUv3)pX!<0W_#s^RN&3BFb{xKu~r
zGW^Uw5~|AV6rZqf7I#@&yn+MPQ$yH8!Q2>5iu?Z_dSg0Ww;!fr{)tvgm$XCAq$_le
z)~izxrz0RnRvF8_Iteez8iFYa=h2Ij8F|p14wDZ%_#4TvNUnx{+*M8vU;5b^4R4EL
z*5WyETD!>!kRx-wRpBPS{FwNTJnW4Z=z!FyjaxvlxM44Km$q4FL{)ataos|f;AZ+9
z@};I%a48naDf|f>gD9Rsi)onDZz()dv!Sn!!BOHvF3<<!QX|w!%7Xi}%DxQ>RWwYb
zBGHDOG#b8CYt(cqwhCZ}+C=?v5uMP%@FNif;4vzr+3bN=U>dn{2qyC(8i9dwMxTLM
z_%;!x!d@zrb;ihVTfu_mvT7$+p~<jQDsru4`gp51N78lLu3v<k+DAOidQqQ2Fj1Ul
z9yaCi=p&lmSUQlV?2A2BP4ON@yc_P&S=vW?s2RDaR=iJo2NxQN&+`eK#~a|ZRl+&w
zfum5mRgU6^x+&k0UZXvBMu&`^1N|j;GI%Pmc<mFi8m)co-6b!#!(w*F>gd2td>77<
zySfKUtXw=!TVNx-MN9R3>>;@y1m|F`%%CccP{-gU`}?q*YhgDWMhkI<xV|V1z}EVX
zbOlPfiot3RlwhiAj1H*7=hQ=52VQucHdB#4PM=Xfs>9826kn9xdO=huK^>vv{IPWD
zm7c#$RLldJ?yF+y!OugXJ)Q4Uk|=Wr*3$QWp<7cq#ByEfX*R$V>9<oT*gg?&(r3y?
zD&?52gQYrJdYT#J#v|n9yDqM5IiH{iE}%8AkQaiFtokN-!9LM|!$N9|n!{=K{+KGx
z#03NSD6EBi?v2GVi|5rweMEQ01JZ*yrGMNfZuqFw_+i|M2awc4*rE?wzd#3)jviY|
zS5iXXk!C+po7WMb7Ivj{IE(MGl06uJAE>H)Q#zifXcm@9)vZ)+q5+GzSh8>$M~h-S
z$D80cIaBS*8w+3>EVf#Ur!Chfq{n}n3!#nVdzti!rz(4G*@rIplzjy|U_ZT2J0M(K
z%WZoheMJ|<%bx~2Y=i^k&6jAY{AB=nh4eXR^l|Cn3+=hQO_cbkK5o4z4z9YmtQ5Ye
zh&HM3MCIDqfon>S79%cjFAsoNT&cejEj>(bP!A*NKGnuMGRJ--IZMB&x-?OuKgW3x
zs604<9lEb*4w3sl7oKImuIEyrDCu`l2x0glSGpP}9fJ)t2P?>5Y5}>cRSjcA+Y_WC
zI4>DjN6wvc+3mHWF1u)(^iuiou5=d7F~NR8#(qX+(1IM`$t6@>;!ao1q(1y5UJ0C1
z#Z(~P=8|pzx5eAY7;ZtE^*eA~^$~A9K?Txfy+~W4{%7br=uL;9xenlLD7Kf2XK}TT
zsmpYLZmXZ6kk;$59EUfoFV$k|gYw5Va~T4LfFWQA7y^cXAz%m?0)~JgU<eojhJYbp
z2p9r}fFWQA7y^cXAz%m?0)~JgU<eojhJYbp2p9r}fFWQA7y^cXAz%m?0)~JgU<eoj
zhJYbp2p9r}fFWQA7y^cXAz%m?0)~JgU<eojhJYbp2p9r}fFWQA7y^cXAz%m?0)~Jg
zU<eojhJYbp2p9r}fFWQA7y^cXAz%m?0)~JgU<eojhJYbp2p9r}fFWQA7y^cXAz%m?
z0)~Jg@Lv&#kMs0evc7t_Ek_=gJ3h{>ip}Qb_3wl0+LUcVN_JMlBuDDF)bzx(j#E+`
zS*erKV|#S9jr{*!VcwN9II&{fv##=$mG^(1K}%bQCuR_moajvKn3dv4O-xJuOQKU|
zlaQJm+au_|o=|9)$cU&eQBhqY!=k!$?HbWJGOBZ^T%E!q!Xmqdc8!Xx{HRk@<t&f9
zHa8$hW>|U0$GKEa_D@m|)=w>3C%;$T^(sGj?0TGA<)lM0CZs(Xqd;A4=po}O@A{98
ze;~g-9%D?lGi8z^ImPj0kbm9GI*$bz>hffef6xnkGQz)ZM%~9E9H0JVgg;W?$q@fa
zs^w!L2IW2(;(y=6_&5)b4u6aca0~u3hWj5s{qyw&Sybiqlf<pOWSnQ^>+t-_|82I)
TkK^OqE0NmV<zMCVzd!pw$k;CO

literal 0
HcmV?d00001

diff --git a/original_scores b/original_scores
new file mode 100644
index 0000000000000000000000000000000000000000..dd8981024aa3412d92dd38af8af9586c61e3cb1b
GIT binary patch
literal 1476
zcmaKsT}TvB6vt<Gb$1)7FhSddUIN=9ZC%|D!8r>{EyO&u8HGu(th0knPLA`TQ1oC?
zK}A6i(H3P;LG>;e={*}{*-M!OeLNdQ5fo)=FV?;7D*NGjhQqxx_uSvTcmDUBxag?h
zIJcWCv5g$hX@=6H^d;4VDQSjm?o1`EWK(~y8jCu)uKzY<mn_YYjx;3(EJZc>gduki
zC~7KU$$h3~BnJJw&Uw9^F3LA)WU@!rZP_V&G^T1vt2wwo`<>VAoi-Zg(;b}-+DiQc
zdWD#_^vZoA=M`<Dn|h6<JH29Btf7z1(p~2zNmEr>vJ|b)<o6mW*^pDa6v^Us_qGmw
zjivJ)dR0?gbh+Q4ZR9@XGLM%%wujqLa_sKomaH*!EV?dt?(FHix>UuoRGHz=3Ns!j
zMKdptujPUzc=lI!`eIRk5qoKk)!KTr@I7*oDYQo(;3b$9I>}och6$*{weS$uX69fN
z@4)x=A9x+7g+1_x45FP}2M;<(Bi@5~xJI6l-{Tg(!$nG(7<-8##K|W#@$mQ}G~=!D
z&v+F*coZ+<1Q`_$;RSGz5X=Z3SfAO9t?&dEz$7lbhXy_(PvMqO4b_pG&<o*AJ=D^b
zJY$n^4l3{`Y!SQ>5!>MmR!4>)MnvHOtjBeD1y7+XBjFz5_Sk6{Bp>06P#@_gn*d-E
zZnVFKUoe6r!X40X4$?4(7f8QwK&XLX2$C6+#1BH6q{pw|-}YHyJ6y&m&`%Db9Untg
z<`qtnt*{%JQYW8(z3Zbljd^*c>bR;Br3Pigq!%p~buCHN!ZYIZaL)ld)tz~HVgt9m
zgxK_7d{Z`*q^g`qvTK}BQn6^m(gLfAPse<LP%s<_hXbLOaG<R%=nsYcK5ESYIufRx
zP_U)7Ipk-%9Q}6eg_rJ-d0e!y%`RttesXMN0|i#D4ir=JbHGzTn5S>Wl;yN4#^eiB
zTYx!8B`d~c)2En`FG*bigQLZzn2{5<7?S_T8VVpCD?nD(GA=ruJ92Y3I~tcMmAT$h
jdz95aY74CPQKR!%(d4>w0c<!fR<iQKRnmvWXJh{XWG0yM

literal 0
HcmV?d00001

diff --git a/tests/test_llama_e2e.py b/tests/test_llama_e2e.py
index c4eb32b4..0b27b28b 100644
--- a/tests/test_llama_e2e.py
+++ b/tests/test_llama_e2e.py
@@ -22,13 +22,13 @@
 import torch
 import torch_xla2
 from torch.utils import _pytree as pytree
-from . import helpers
 
 
 from jetstream_pt.engine import PyTorchEngine
 from jetstream_pt.third_party.llama import model_exportable, model_args
 from jetstream_pt.third_party.llama.generation_original import LlamaOriginal
 from jetstream_pt import environment
+from tests import helpers
 
 
 class LlamaE2ETest(unittest.TestCase):
@@ -93,9 +93,8 @@ def test_jetstream_llama2_seed(self):
     jax.config.update("jax_platform_name", "cpu")
     print(f"---------> {jax.devices()}")
 
-    torch.set_default_dtype(torch.bfloat16)
     # pylint: disable-next=all
-    env, model_arg = helpers.make_env_tiny()
+    env, model_arg = helpers.make_env_tiny(bf16_enable=True)
     # pylint: disable-next=all
     tokens = np.arange(10, dtype=np.int32)
     true_length = tokens.shape[-1]
@@ -221,7 +220,6 @@ def test_llama_e2e_float32(self):
     print(f"---------> {jax.devices()}")
 
     env, model_arg = helpers.make_env_tiny(bf16_enable=False)
-    torch.set_default_dtype(torch.float32)
     out_tokens, expected_output_tokens = self._llama_e2e(env, model_arg)
     self.assertEqual(out_tokens, expected_output_tokens)
 
@@ -232,7 +230,6 @@ def test_llama_e2e_bfloat16(self):
     print(f"---------> {jax.devices()}")
 
     env, model_arg = helpers.make_env_tiny(bf16_enable=True)
-    torch.set_default_dtype(torch.bfloat16)
     out_tokens, expected_output_tokens = self._llama_e2e(env, model_arg)
     self.assertNotEqual(out_tokens, expected_output_tokens)
 
@@ -242,9 +239,8 @@ def test_llama_e2e_two_addtional_tokens(self):
     jax.config.update("jax_platform_name", "cpu")
     print(f"---------> {jax.devices()}")
 
-    torch.set_default_dtype(torch.bfloat16)
     # pylint: disable-next=all
-    env, model_arg = helpers.make_env_tiny()
+    env, model_arg = helpers.make_env_tiny(bf16_enable=True)
     # pylint: disable-next=all
     tokens = np.arange(10, dtype=np.int32)
     tokens = np.append(tokens, [15050, 3503], axis=-1)
@@ -315,9 +311,8 @@ def test_llama_e2e_four_addtional_tokens(self):
     jax.config.update("jax_platform_name", "cpu")
     print(f"---------> {jax.devices()}")
 
-    torch.set_default_dtype(torch.bfloat16)
     # pylint: disable-next=all
-    env, model_arg = helpers.make_env_tiny()
+    env, model_arg = helpers.make_env_tiny(bf16_enable=True)
     # pylint: disable-next=all
     tokens = np.arange(10, dtype=np.int32)
     tokens = np.append(tokens, [15050, 3503, 11833, 28551], axis=-1)
@@ -387,7 +382,6 @@ def test_llama_with_original_prefill_decode_32(self):
     print(f"---------> {jax.devices()}")
 
     env, model_arg = helpers.make_env_tiny(bf16_enable=False)
-    torch.set_default_dtype(torch.float32)
     # pylint: disable-next=all
     tokens = np.arange(10, dtype=np.int32)
     true_length = tokens.shape[-1]
@@ -458,12 +452,11 @@ def test_llama_with_original_prefill_decode_32(self):
 
   # pylint: disable-next=all
   def test_llama_with_original_prefill_decode(self):
-    """test jetstream llama by comparing original prefill and decode steps with float32"""
+    """test jetstream llama by comparing original prefill and decode steps with bf16"""
     jax.config.update("jax_platform_name", "cpu")
     print(f"---------> {jax.devices()}")
 
-    torch.set_default_dtype(torch.bfloat16)
-    env, model_arg = helpers.make_env_tiny()
+    env, model_arg = helpers.make_env_tiny(bf16_enable=True)
     # pylint: disable-next=all
     tokens = np.arange(10, dtype=np.int32)
     true_length = tokens.shape[-1]

From 42a93c478ddca26c2941c5391fc0ee35e3ac24a9 Mon Sep 17 00:00:00 2001
From: FanhaiLu1 <fanhai@google.com>
Date: Tue, 7 May 2024 01:43:44 +0000
Subject: [PATCH 2/2] rm oringal

---
 keys_original   | Bin 33978 -> 0 bytes
 original_scores | Bin 1476 -> 0 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 keys_original
 delete mode 100644 original_scores

diff --git a/keys_original b/keys_original
deleted file mode 100644
index e2dae29f0e10deeaa7d2a7ee022f84d54b11c18e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 33978
zcmeI0d0ds%*2fPZgD8me2r5oEfG7wkXRl?7Wa5BBjwK<Wl8A6PYTUArL2+VoN>dZf
z%sF$GXRoEEX5E`pS93~DvvMlatjx-HaqITEdf)qg?&tk`KYkt$ob&8;_Wtg*e*3qG
zk8`VHvw3;h{_Uz~t8SZ+lAV<>$&orPH9aw{V{)Q1F(hL`T5Jz5+sHqz9M>f0BuCQt
zkc7$3)U+(`1V_r4$*F0{3C@)CtVxc9DV@DDZ9zf9JtcmYJefEyCDY~iA>KV=(<UW4
zJ4Fos{eACD*WoVmSZJo(a5s5P%9xy4rJt?4Gt+%woGmEMCC)3ZcATFx(<4aUjq{7k
ziF0vgdd^Bpnv|B7lH^RCl%C~1z>%EdNJ)-KO>%l?dbJ&tS=E{8Jt(tUNPL{9*Ms#_
zi`Gfx$~*o)F1)_2))NaKlkH5IR2ew7hwq;&{~bVDs?(X4QW?BJU2W(gfh+I-9K4P#
z>WSc$R6B;o_6UC>@;?dx->>fgTIb;*<gqK;U=z)URjMy9rx@tXUN8jGFbI6Ox$eio
z+?F4}1)8Q7>bd-d3WitE4F=<4wM|XscOh6`rffO~7ilNd!bEa`M(n8<@-6Vf7eJ!Z
zGyERe_*K|OzuFhlE*?d>RD&<jSZ;<xAQv`6Jhi|U>{{_0`q5kZc^O}ii=l$TXgGS}
zVYo^I$d5CjJyhouXuuWp3RlAt9nB5o^91Ff8PrTI;5Sun^c*apMKlh!!Fk=01F;P)
z#b8W;JNzxV>*3r>57XmeE_|u~YITt{c7kk(!j3qe*HdRMqYAhHPvLo*#XIeXD46HL
zN@|B&untA?6<r%wLJe$!+thbf2W(9_6o!L%Ggko{`pZm9^eAXa`P?5XU>d)tX35-#
zayUM;K7}>doWG|Bkj0PGWw3M&wHkue0{eCt&h4y0;7_TjbspF#7jvl(Oa}+~;(D_4
zE=u6>;DLXm&5%)Xgx*jN<-%@wi0k1}IIbpBO}-3kv<p~}fgA0+seqPKN0#T+Cwd3?
z$j>+EbNWQ3!2oQhbNN?zL+yaCqMJZt%z*XwXE~gGI2KxRL&$`QdatSn>){O@4~<{~
zuVe?h!_$~W*Qh40P-kH@?xY&%f$s^AE%+SdVMl(MGUzG3NgvtkVmADwnm~j)3T-7v
zK7eZMhxKr?>{U<Bu=2ngYvFuaqQAG6t3+JDOf5M^7_ZP*Ay0e4o8*Frs14e&75eiO
zj^Lwsn0)yleM8-$4}7Rb;{v>5HRtWni_fb-c$>Dthq@6xP4WCLeM*^nBG#oCew(Av
z!n1k}uUBm_6z@YbD8+U7G38n#?W5tZyjy*US=<qy;uW-jJMcERMyn+Ioc6Zt32T%q
zmvWiDO9Nq+tT`51;ZR0+l|G<@uqwK)tbBsxaAWMtM^$fVNGWs|Qg|_G@P?DTNO#i}
zsuR288lH|$Y{dicHI9TbtB_~oIJ8LxX7XE-d52*?+|b*xknJ!M8nTm}6!4Gie?+l3
zgeFS1E|T1;&F<=iDo~H0jK5S~xRySG>HMy$%S||mWmkYHgh$|6YYCMp8()K__zM(q
zB6h;rbXwiiJ#Zo<abs?di@7a^=%%nkmrC`fvV)Vb2?ep2&f_z53XpC32A$UB>J|(`
zr91QAh4Xjd<gejfh~rh{iEa2Cjgsog!G=-|`8<g}q|sQLS5g>f(OKvW`}GY`nq4Xg
zgZb|e4Lzxhr(#V`g8(X{c*<daT&9<(5A_Thg$33=?!p&!I|{Vd$1c%JtuFiq4Zs-i
zl<M)K5%3XJ<uA#JF-+79rovkgf;kvZiCC^1kSBerUgBD$xKvqGg@@u&I)g6((Y%VT
z2zZ7TN<RA2Y#O3+pbf^TavjVkph)Jt0hK+F+TwVO)>Wm>7f9^ZvL8DjmWy=|Ho%+U
zz<3Pc`EUuU(tceIJz+Dsa};cVURDq6FEzZ7U)I&JjAr6&o`-8_Fv$0C7pa<Z&Q)_@
zU$@@$8m*%4q9eO#sH%y>*quwri`^?Gz);aNe|}MiVicCa0^OJna1B17uTc*GTqpVR
z5!J#FyiHfs-$Z+VgoEl5RA2)dr1Gto^;WeMYtSy}%BLw1uhEw@26Ff=`SaFpb?KTO
zj$v3|DrycQe@;hfrb1|{`*0~8z*JpZvZX)Whiv*-KNP*$qc%Yk?n>Jzi2T%HbjKf+
z7Y|nVp`Pr5zvOHQMCq0|Ob0<z?!*MYQ9G#5`Osb8*0Dgmoi3}3ut(Sx=>;SNz!&e)
zC!%j1_+@O!S122!bgpH=67s@<)RhV2IFmNnv#2$9-}@fiqxPJxuc&gVi_O#q{sxmc
zMoy7k*anY)o9N?9`~!t!DQ=;+)F@m`#jph2F}LDpspX6M1zfHkz>oIJdX7C!lxTxK
ztKul9q8Q(^d~paSU@z(hzO+s~gywiu-GpF#WW~Vya+Vdq0~pP1wWoRq+TukkLlmYr
ze+5@$ZeM_=SdM@Za<0sjeAolEIbG#S?s@T2$+7PEn>^h?)y7@2Yp9nC7XoiyNcUi`
zJsZwbRdpOzNBi&?Y^sMq5YDBODn_zBgkPlka#Hw{JNRK^oIxvKkZz3qR7cFGJouP0
zgg`Jh<6%-2jW7u}Lkr%|lQB+a*<X~jJ1nJEa!R-3Q0@Q$+?@w%dvvjGLY?Tcy*{sm
zZ>R{G-~c(Df1thK%lkyD7pe1b7;DP-J1`uYKr2q7i*%2E1y|LcUPV7h<-Yb1IWxLp
z1P9V?U6Y%^LH#p~!rM@T$6~(Z9@Aeq6nk=oj)E{1&kYoEb?gQ2(n?*Gchjp>o$~b&
zYKgO~TZFQY&tOmZ#R|aHlKJzvDxRgU?MYZoPW+pc1{r!f&y}@yf&`3~O8ZoAq`~Zi
z8^8shk#i$lRByi80BhiVoydWlF175TTHtAz1*=7ks$&(YDR-W&qNzC+l2bDG75o*>
zz$JZ7jp9&hf?vZ-oDX4Kmo`v;$-_=O5<j4=l*a9~H_pRa9LCSejO%cSS`O`T9DEIk
z)A2{WhwfT$!fIGUTdjMNZ$q#h4~DbWUFgQu;d>Z})x=e#fQIwxH>soDbXVOI=g>>e
z`vMw>{rI%?vgqd(C|3JLaX*DmWe>+tTb@s?DU-ah7bma}2Z$%?qn60~<7f^gh^uHs
zMY|)h92UYgeNyzeLbafw_%XO*Q=BX{drSAj4x(Yt(ra7~-k@c?N&QNBqMOAyi~L|P
zOjjS+2kR}8dCT~9oJq;zeWt^6cv-F0;n<jqWZiFrm(=xj>m6DQ--s(HMGp!PU-SYV
zp<+?7DzaaZIMd#TmgyjF$hTEh%%SO!2iu{&oCGdX@rl&P+M;)hN|)0kRYI=f^tKBX
ze?Fon@(;>`Uf^tOD_$`lbHyo4$40UOe{2h9sD?hH$5JY{VIMjPv&BQz$CmQlKZwGA
zF7>jI7fCh#NKy2TJxF|KFe>N`Wf-ZGMQMHopKcp<1h1uIvaVclWlzHcI;RWe`)5KQ
zIalWM41LGGm&QUk@5HmXPrTrBc6V4zbFFFgAymaF94{(!k$U6by9M%C-a!4hhWeSt
z!*#d^;nW0baRBa8WqggMh}!q%B{&V*kuOf5nG{Ne(MxCn+Heq7<CF9jrt>shu48dE
zB&$%&<0;6nPBbo$2Vx#2<6ZH&=jke)fG$u*%dj<Wf|2wroWg8tEw^T0Zif|6$ot_q
zy-p#j4t+!=;twsgQn`=|cA=}P6!+32x+&Uu7do<yy=lAMRi6@_?*yB524AzLQgv<*
zQ+Q`Z9V~=h@UwQ8^E89{>I90hGU*n5uKYNi8S3c)m@FFP!g+KAYG45L(^X{cLv=ko
zCfSxNUi1{%sg`w!UC9ZhQj<m80DZYWyicC|K!tJ&-HMK-20TJseFc}}8Hl9g&`<V4
zRfLFF2<NSkk7+Ol_Ucto0N1Gnzs7^GzGOuz1xm+pRb0b8i7=0AaRINRLlnvjcpOCI
zMjFSX@hJ-A{kj*t%o~)Qr>JlAw=mV}D~{#{w7|D{wVI7NRvM0_o3vH0fL7qH!=V_X
zcpBH&^C^&K*;j~CK9n7}MD@5Q-&ZM;Z-ej$xB;J21$F0E9L{a29i4&i$xk%@AS@8y
z6(&wU7&fWPxSdM)TdCn0(2i$dH|a|fVUVyWryblx&Zu-*b2D5fnm(K2xR`gsY_&*N
z<Ez#|Sc%^3B0KPl?nQI)IIdNByj0GI5FQDkc!dUv3)pX!<0W_#s^RN&3BFb{xKu~r
zGW^Uw5~|AV6rZqf7I#@&yn+MPQ$yH8!Q2>5iu?Z_dSg0Ww;!fr{)tvgm$XCAq$_le
z)~izxrz0RnRvF8_Iteez8iFYa=h2Ij8F|p14wDZ%_#4TvNUnx{+*M8vU;5b^4R4EL
z*5WyETD!>!kRx-wRpBPS{FwNTJnW4Z=z!FyjaxvlxM44Km$q4FL{)ataos|f;AZ+9
z@};I%a48naDf|f>gD9Rsi)onDZz()dv!Sn!!BOHvF3<<!QX|w!%7Xi}%DxQ>RWwYb
zBGHDOG#b8CYt(cqwhCZ}+C=?v5uMP%@FNif;4vzr+3bN=U>dn{2qyC(8i9dwMxTLM
z_%;!x!d@zrb;ihVTfu_mvT7$+p~<jQDsru4`gp51N78lLu3v<k+DAOidQqQ2Fj1Ul
z9yaCi=p&lmSUQlV?2A2BP4ON@yc_P&S=vW?s2RDaR=iJo2NxQN&+`eK#~a|ZRl+&w
zfum5mRgU6^x+&k0UZXvBMu&`^1N|j;GI%Pmc<mFi8m)co-6b!#!(w*F>gd2td>77<
zySfKUtXw=!TVNx-MN9R3>>;@y1m|F`%%CccP{-gU`}?q*YhgDWMhkI<xV|V1z}EVX
zbOlPfiot3RlwhiAj1H*7=hQ=52VQucHdB#4PM=Xfs>9826kn9xdO=huK^>vv{IPWD
zm7c#$RLldJ?yF+y!OugXJ)Q4Uk|=Wr*3$QWp<7cq#ByEfX*R$V>9<oT*gg?&(r3y?
zD&?52gQYrJdYT#J#v|n9yDqM5IiH{iE}%8AkQaiFtokN-!9LM|!$N9|n!{=K{+KGx
z#03NSD6EBi?v2GVi|5rweMEQ01JZ*yrGMNfZuqFw_+i|M2awc4*rE?wzd#3)jviY|
zS5iXXk!C+po7WMb7Ivj{IE(MGl06uJAE>H)Q#zifXcm@9)vZ)+q5+GzSh8>$M~h-S
z$D80cIaBS*8w+3>EVf#Ur!Chfq{n}n3!#nVdzti!rz(4G*@rIplzjy|U_ZT2J0M(K
z%WZoheMJ|<%bx~2Y=i^k&6jAY{AB=nh4eXR^l|Cn3+=hQO_cbkK5o4z4z9YmtQ5Ye
zh&HM3MCIDqfon>S79%cjFAsoNT&cejEj>(bP!A*NKGnuMGRJ--IZMB&x-?OuKgW3x
zs604<9lEb*4w3sl7oKImuIEyrDCu`l2x0glSGpP}9fJ)t2P?>5Y5}>cRSjcA+Y_WC
zI4>DjN6wvc+3mHWF1u)(^iuiou5=d7F~NR8#(qX+(1IM`$t6@>;!ao1q(1y5UJ0C1
z#Z(~P=8|pzx5eAY7;ZtE^*eA~^$~A9K?Txfy+~W4{%7br=uL;9xenlLD7Kf2XK}TT
zsmpYLZmXZ6kk;$59EUfoFV$k|gYw5Va~T4LfFWQA7y^cXAz%m?0)~JgU<eojhJYbp
z2p9r}fFWQA7y^cXAz%m?0)~JgU<eojhJYbp2p9r}fFWQA7y^cXAz%m?0)~JgU<eoj
zhJYbp2p9r}fFWQA7y^cXAz%m?0)~JgU<eojhJYbp2p9r}fFWQA7y^cXAz%m?0)~Jg
zU<eojhJYbp2p9r}fFWQA7y^cXAz%m?0)~JgU<eojhJYbp2p9r}fFWQA7y^cXAz%m?
z0)~Jg@Lv&#kMs0evc7t_Ek_=gJ3h{>ip}Qb_3wl0+LUcVN_JMlBuDDF)bzx(j#E+`
zS*erKV|#S9jr{*!VcwN9II&{fv##=$mG^(1K}%bQCuR_moajvKn3dv4O-xJuOQKU|
zlaQJm+au_|o=|9)$cU&eQBhqY!=k!$?HbWJGOBZ^T%E!q!Xmqdc8!Xx{HRk@<t&f9
zHa8$hW>|U0$GKEa_D@m|)=w>3C%;$T^(sGj?0TGA<)lM0CZs(Xqd;A4=po}O@A{98
ze;~g-9%D?lGi8z^ImPj0kbm9GI*$bz>hffef6xnkGQz)ZM%~9E9H0JVgg;W?$q@fa
zs^w!L2IW2(;(y=6_&5)b4u6aca0~u3hWj5s{qyw&Sybiqlf<pOWSnQ^>+t-_|82I)
TkK^OqE0NmV<zMCVzd!pw$k;CO

diff --git a/original_scores b/original_scores
deleted file mode 100644
index dd8981024aa3412d92dd38af8af9586c61e3cb1b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1476
zcmaKsT}TvB6vt<Gb$1)7FhSddUIN=9ZC%|D!8r>{EyO&u8HGu(th0knPLA`TQ1oC?
zK}A6i(H3P;LG>;e={*}{*-M!OeLNdQ5fo)=FV?;7D*NGjhQqxx_uSvTcmDUBxag?h
zIJcWCv5g$hX@=6H^d;4VDQSjm?o1`EWK(~y8jCu)uKzY<mn_YYjx;3(EJZc>gduki
zC~7KU$$h3~BnJJw&Uw9^F3LA)WU@!rZP_V&G^T1vt2wwo`<>VAoi-Zg(;b}-+DiQc
zdWD#_^vZoA=M`<Dn|h6<JH29Btf7z1(p~2zNmEr>vJ|b)<o6mW*^pDa6v^Us_qGmw
zjivJ)dR0?gbh+Q4ZR9@XGLM%%wujqLa_sKomaH*!EV?dt?(FHix>UuoRGHz=3Ns!j
zMKdptujPUzc=lI!`eIRk5qoKk)!KTr@I7*oDYQo(;3b$9I>}och6$*{weS$uX69fN
z@4)x=A9x+7g+1_x45FP}2M;<(Bi@5~xJI6l-{Tg(!$nG(7<-8##K|W#@$mQ}G~=!D
z&v+F*coZ+<1Q`_$;RSGz5X=Z3SfAO9t?&dEz$7lbhXy_(PvMqO4b_pG&<o*AJ=D^b
zJY$n^4l3{`Y!SQ>5!>MmR!4>)MnvHOtjBeD1y7+XBjFz5_Sk6{Bp>06P#@_gn*d-E
zZnVFKUoe6r!X40X4$?4(7f8QwK&XLX2$C6+#1BH6q{pw|-}YHyJ6y&m&`%Db9Untg
z<`qtnt*{%JQYW8(z3Zbljd^*c>bR;Br3Pigq!%p~buCHN!ZYIZaL)ld)tz~HVgt9m
zgxK_7d{Z`*q^g`qvTK}BQn6^m(gLfAPse<LP%s<_hXbLOaG<R%=nsYcK5ESYIufRx
zP_U)7Ipk-%9Q}6eg_rJ-d0e!y%`RttesXMN0|i#D4ir=JbHGzTn5S>Wl;yN4#^eiB
zTYx!8B`d~c)2En`FG*bigQLZzn2{5<7?S_T8VVpCD?nD(GA=ruJ92Y3I~tcMmAT$h
jdz95aY74CPQKR!%(d4>w0c<!fR<iQKRnmvWXJh{XWG0yM