cJojGW9{h+s3)@*Q*4{wCkf}oM!eDtg
znVOT^0;(^5A;RCUAir^AWuOVEWdnl
z)CT#dQoqB5HS1nbc26`9c}8f%>u36LjE=#?{)(H3idslrzrCDmWXOhQz>&JDtmj`4
z(*2L(yZxDU{a?Iq8Prx+7Opr;SL;dLsZQwh)f&uuy}=ir8&A`Gi!HRdClyV7Ki(rE
zgt!DBDki0{%&RcH?UWQTR>-+O|==iap{f_8{9oPXp{eBUvYuii};J88UpmA1_I+
z6e^e+XdkA;_2E`Fo!Z`fN)xoop;e&Ptop8>j9)$3*P(*3JCwhKcORNTlZMzA@b{?v
zeTO118V(2llBUDH9POG{Cx}N;<+Y*^n>+`^i(5CUBUdNO73??Q4+4#<oJg!q+ZY9-!
z`{i@mAw}*S`#@FjT
zA3r_i_kJ?0x!OF)oOj}!QdMUa)5?lau5CG$>F-!ed1>H_KBa{SAuuQ-NyZ)K-j(?qf{mIMi%pK|pE*X|l{rTr?V_c44qhZfFGDcF_G
zO>+e&e(JUpg06+hhrIlFmi|EC;Nc~Fmb1_@a;mq62;p=lFji(ih-3r9g)j
z=CT{9LUA-H`faX*+{6*i7>lO{ZUV3r`L2Gda>@
zw2Y$csc_U2R%S&Z**9lIc8{1|AiI&t|bGxsGX4XyjRePmY&{w@2&bHO*lvwK$R@=RK3*55){R6Nq
zeY;9}Xu(K2uJZm9Yn{qpJGeMjmE-S%lSCS8#w+vtvBcM^MILn9J+%j17TN#Fv~p`o
z1N@!X)OS;v2D#FtL0!_AzF+XA%%>}!HBRV}3ZHJD2j6A|&NZeNZa&dEdFR@E7Y&4R
ze+qn>!jjNAgCaW?NWPfjnw`A4bQ#@d*krW8bOj7egz;av)dT!(XGCM3YO8z}BQ(Vp
zNz=Cu;zKk>l@$yKFC64IC4YCF6tYYd2mZSr9wB>A;W`P;}^RjR^^;9dRSL0$bVU;=Op
z6#Zh7l9|xL!&Td8;xX=%LIR7fxu#<}hOam1Xbkn4^~moQ5-Q=wVv09GuGOuO=WpgJ
z*5%HJAaWQ~Wq;h6MuYdoYlD}Wg#7xGLgDp`HMmIM$8BMy<}ct3udGIIRPj{x{9Tk0
zm&XvDhtXFz*y7EJ_Zt@x5`;x*pfG>yfub~&5q;x5fe9y*>1%Hx^+0^Cc~t-0pwcx;
zxZRQ~keIei>R7T4G}Z=rKK79mxq0d=V#4HW#$|F9uYgiT4?=46xdGRwd*ACt>05IG
z_VF`zQvYqH%`U#M=FLwD;lK>bJpdKd2D=k2rgL%Wb$O^tQ!^&htEJ~yY0Y)l*M5VX
z(`q9ZxB{Xe?SEOF2Qp9GBvpyrkVDl{60NlsU0?R(cg3zPr5E&`%pI7oA$DTNr+0wF<=U>RjBT#@!gdlb;l5Y+oJtzdlB_l3fD&b
zo`-1v&hBdaUy
z<%$(Z#v||DsS?Kd>p|qO%KpVRs4e-HyIS_UMR
z_>SmM_@zbkZ$MAsq@o7W_B+H~AAd`QXQB?a7Z(B_)sGvgS5)S;t9IJ8_aE+Nbz529
zKmFxrP~a}YoVGeBw}tTU4A>w9WYaPKPYbNXHxn#j{{|k1)X6NYb1rmpvA)BH=DE1s
zirhi+C3hqF3a-CE5-`FfV=h|fP
z{};*)AmyCzDff+%aq1r1_(H_KODuu)_Z$ITvrp7oObdG!&$7=(>TWY|1Te_VH))63
z7d^ioW|r83&0{TOA%|C5TffP+UGQcCYC1?qHOYm^$P^7_is;S?j-JgD(;r%w(1QBl
zN9Oe94MhVFtsKhBfFoGz+j89585CZ73!FEciW`v-ozb`c^cy~Srf*&SfnH!z&$aa;
zuK<>*i?DH*NED*Hj5)&QIC0u*K>Dj9^uzpQ-P;bbNm6cE=mh1XQyLetyXBX;AFoQU
zi7dIg&=Um!l61rjOCfXHt&P{=h2G|0$rcw3V1*Db7Hj>6i?fUi1>9C6_c
zvQLy8z5^f(e*H;9pKoSvDmd6Nu&mYziNZZL61nzj)eS_Sz)
ztXJg66ZftF(DLPK0Y*B(=5n3d$d*JbUYXctbpY=EHUyaXk2jRPuV=!6YV80C|Io!<9dq=7y49wR3zH>qs?=oSC6u(3(Ga$
zdzTjV^*ERl&&(mXvv`Y37@Hmg5@mTLD&w_V#Z&
z1ztM&BA-$p1Rf5$NvU#z#wr8Eo)w9RUN6J+-B`8S%DR*=I>o%kJdRB)s@{257@h?Q
z_Wci8g(^!9ch`Q@eL#h}L`nWPNEkCoJ`!;y%}jGZ$PA`lrmJMQ%Hu;E!6~CEI`qzR
zSjXr%i}(*)-dzUUxxX9aDr&6#m;G=uJx*x&dzoxd8><4;qjb{H(bb{q0y;@W6=^1g
zrIyeoR8zlLor?1;9i7=w$6rm|H6lmn?e^#9;NZZ--G;6
z79FYXG;<0C6xDa-mlcUTG@xdOasU&j*4@%;Z*5RyU6TVl&HHO<1fSw~fk*P#r0rcp
zeVzWFehjk$uk0URqu5o2E>ulIL;
zD<0-eIqtMsAKA5^^?n+x48g3qoA&LC$7l%P2-7%HKxyOajGcCqHf*CG*7TMh
zt$=;A^`?bLU;NDNLlrZG0(EE=#_Y$;<6$>56hwRelUGXP8Pd3u?mPk{2U*D`xu-Be
zE7NzrHVoNPEuZAmyJ2bv-4oN_*fNppqT^NklHIbkl_j@U<7)9Qi>tOiekWk2HzqLQ
zZNfD^zjaK2O4_Q*$Z>y*^lQ@)w?A(Mt9Prad~Jw@uq*Jzp9gPw-mszj
zhb(>Y&gGM0U!Lr#Bs*RgouoEz1N^rLp`Srm_BP_5Rum6>TSLdQ!*~s0cXyO=#U=mD
zpOWbodyIa3h&?VUwsAO-UevNC?`7~B_m@Ay<76vfKe3yjmN
z`oQ7nfEc%Ox!&^w3@Oc~y|(*wjDMnXeW1ABDR{1&oGHLx+aHpFP%bobQs@?9-{``A
zlp#bVUJ%qd$xHb~B$VN`3I>bRKMTNaf9txAmck^+l8e3xIA=3>AnfcYJ{$c$3VO<&-&kLgU_E+U^G-4!!=pCcaCk?Te?<7rPdc1FjbuSm|
zS$CATzWjZo$r0RST+EC
zD&Gz3tQxk^arJ-6XQ=sum2G#+Z1MvlSjLy`f+R35!{eM#2|~%*-gc8;=BsqEHtkJa
zO>sdxHd(!D%Suj}7uO!qmYB>h?0r*aBvVgnkos0oVss>ow>EP}+&=UEm*n1zizDX9
z;0OLm4~P`@Gve4H-k^}C$5WFB3-~Z62TLr4FmfNw@8clSph~(sf_P~7>22aQo(B4$0;ki0pA_HignJNG%yp_9^rkWs+40D<)CCfr6$10OC9)S;m^n3x
zhtYge!=DUCebh$<0`%wwirHRzhVK40U{GWeNPNJr=w@IuMC2Lev&l6{C>Ylu{HQlX
z;f2b0&wBF#bN-$iVc
z*q42(I@9D8>(GCdkg0SV=HFfY$OkVP579(NqzSn-UEKC0^nNm{`2u~yY>>W>zF0@!
zD@*vzWdTsj$@jm9Iu}Y7uPc2Mv~b}hAt4@Te!tEm_nXt-9ejuK5%u#HGtxvx7btlz
zt!fAaoGl}36)vs-;dul`D@$(4ENO>Uc?&s4yUXY}y>)@dPTF}_v%gExws7>+1~$S+
zi5eq4`)7Qk|Mu!RqFDuekBb<7Mhi%U&QZv1!Yw9}`;saXC1%wFlo-5b4u5jg%UcHl
zb1IdfxLgMUgP0d5+yl`(D>LSV6AaMy(_j;MJ1EoV6zu=SbSrL_Si3DJE-Q)gh*b3J
zY$4P0C$_v)Oq28vd8tcIYq+wGsU!8A_$+GaxY}UY!vd3?yT-2HuSajTtSpcG5?@S;
V2mAH|{&U2+GwJ`wllyA={{W!mlRy9f
literal 0
HcmV?d00001
diff --git a/examples/mnist_gan.py b/examples/mnist_gan.py
index 438ed3cb42fe..67e4b7ba7f6f 100644
--- a/examples/mnist_gan.py
+++ b/examples/mnist_gan.py
@@ -9,7 +9,7 @@
from tinygrad.tensor import Tensor, Function, register
from extra.utils import get_parameters
import tinygrad.optim as optim
-from test_mnist import X_train, Y_train
+from test_mnist import X_train
from torchvision.utils import make_grid, save_image
import torch
GPU = os.getenv("GPU") is not None
@@ -52,9 +52,9 @@ def forward(self, x, train=True):
if __name__ == "__main__":
generator = LinearGen()
discriminator = LinearDisc()
- batch_size = 128
+ batch_size = 512
k = 1
- epochs = 100
+ epochs = 300
generator_params = get_parameters(generator)
discriminator_params = get_parameters(discriminator)
gen_loss = []
@@ -62,13 +62,13 @@ def forward(self, x, train=True):
output_folder = "outputs"
os.makedirs(output_folder, exist_ok=True)
train_data_size = len(X_train)
- ds_noise = Tensor(np.random.uniform(size=(64,128)).astype(np.float32), gpu=GPU, requires_grad=False)
+ ds_noise = Tensor(np.random.randn(64,128).astype(np.float32), gpu=GPU, requires_grad=False)
n_steps = int(train_data_size/batch_size)
if GPU:
[x.cuda_() for x in generator_params+discriminator_params]
# optimizers
- optim_g = optim.Adam(generator_params, lr=0.001)
- optim_d = optim.Adam(discriminator_params, lr=0.001)
+ optim_g = optim.Adam(generator_params,lr=0.0002, b1=0.5) # 0.0002 for equilibrium!
+ optim_d = optim.Adam(discriminator_params,lr=0.0002, b1=0.5)
def regularization_l2(model, a=1e-4):
#TODO: l2 reg loss
@@ -88,7 +88,7 @@ def real_label(bs):
def fake_label(bs):
y = np.zeros((bs,2), np.float32)
- y[range(bs), [0]*bs] = -2.0
+ y[range(bs), [0]*bs] = -2.0 # Can we do label smoothin? i.e -2.0 changed to -1.98789.
fake_labels = Tensor(y, gpu=GPU)
return fake_labels
@@ -124,18 +124,18 @@ def train_generator(optimizer, data_fake):
print(f"Epoch {epoch} of {epochs}")
for i in tqdm(range(n_steps)):
image = generator_batch()
- for step in range(k):
- noise = Tensor(np.random.uniform(size=(batch_size,128)), gpu=GPU)
+ for step in range(k): # Try with k = 5 or 7.
+ noise = Tensor(np.random.randn(batch_size,128), gpu=GPU)
data_fake = generator.forward(noise).detach()
data_real = image
loss_d_step = train_discriminator(optim_d, data_real, data_fake)
loss_d += loss_d_step
- noise = Tensor(np.random.uniform(size=(batch_size,128)), gpu=GPU)
+ noise = Tensor(np.random.randn(batch_size,128), gpu=GPU)
data_fake = generator.forward(noise)
loss_g_step = train_generator(optim_g, data_fake)
loss_g += loss_g_step
fake_images = generator.forward(ds_noise).detach().cpu().data
- fake_images = (fake_images.reshape(-1,1,28,28)+ 1)/2
+ fake_images = (fake_images.reshape(-1, 1, 28, 28)+ 1) / 2 # 0 - 1 range.
fake_images = make_grid(torch.tensor(fake_images))
save_image(fake_images, os.path.join(output_folder,f"image_{epoch}.jpg"))
epoch_loss_g = loss_g / n_steps