Pytorch - autograd.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Pytorch - autograd.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"F39znuDsybP_","colab_type":"code","colab":{}},"source":["import torch\n","\n","dtype = torch.float\n","device = torch.device(\"cpu\")\n","# device = torch.device(\"cuda:0\") # Uncomment this to run on GPU"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Oh7hHLN5Fy6H","colab_type":"code","colab":{}},"source":["# N is batch size; D_in is input dimension;\n","# H is hidden dimension; D_out is output dimension.\n","N, D_in, H, D_out = 64, 1000, 100, 10"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"1PjvUTZqrXPg","colab_type":"code","colab":{}},"source":["# Create random Tensors to hold input and outputs.\n","# Setting requires_grad=False indicates that we do not need to compute gradients\n","# with respect to these Tensors during the backward pass.\n","x = torch.randn(N, D_in, device=device, dtype=dtype)\n","y = torch.randn(N, D_out, device=device, dtype=dtype)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"iofB3htnUat0","colab_type":"code","colab":{}},"source":["# Create random Tensors for weights.\n","# Setting requires_grad=True indicates that we want to compute gradients with\n","# respect to these Tensors during the backward pass.\n","w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)\n","w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Vfo2RmwJUk-t","colab_type":"code","outputId":"9bab1f41-3e31-4be9-8e81-70f6784488d4","executionInfo":{"status":"ok","timestamp":1563861640751,"user_tz":-480,"elapsed":6794,"user":{"displayName":"Tuvshinbayar Tuvshinzul","photoUrl":"","userId":"01278297265491069175"}},"colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["learning_rate = 1e-6\n","for t in range(500):\n","    # Forward pass: compute predicted y using operations on Tensors; these\n","    # are exactly the same operations we used to compute the forward pass using\n","    # Tensors, but we do not need to keep references to intermediate values since\n","    # we are not implementing the backward pass by hand.\n","    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n","\n","    # Compute and print loss using operations on Tensors.\n","    # Now loss is a Tensor of shape (1,)\n","    # loss.item() gets the a scalar value held in the loss.\n","    loss = (y_pred - y).pow(2).sum()\n","    print(t, loss.item())\n","\n","    # Use autograd to compute the backward pass. This call will compute the\n","    # gradient of loss with respect to all Tensors with requires_grad=True.\n","    # After this call w1.grad and w2.grad will be Tensors holding the gradient\n","    # of the loss with respect to w1 and w2 respectively.\n","    loss.backward()\n","\n","    # Manually update weights using gradient descent. Wrap in torch.no_grad()\n","    # because weights have requires_grad=True, but we don't need to track this\n","    # in autograd.\n","    # An alternative way is to operate on weight.data and weight.grad.data.\n","    # Recall that tensor.data gives a tensor that shares the storage with\n","    # tensor, but doesn't track history.\n","    # You can also use torch.optim.SGD to achieve this.\n","    with torch.no_grad():\n","        w1 -= learning_rate * w1.grad\n","        w2 -= learning_rate * w2.grad\n","        # Manually zero the gradients after updating weights\n","        w1.grad.zero_()\n","        w2.grad.zero_()"],"execution_count":5,"outputs":[{"output_type":"stream","text":["0 29982280.0\n","1 29005064.0\n","2 33916552.0\n","3 38849860.0\n","4 37770340.0\n","5 27887158.0\n","6 15635558.0\n","7 7203771.5\n","8 3325100.5\n","9 1787720.75\n","10 1172491.25\n","11 884169.1875\n","12 717651.5\n","13 602948.25\n","14 515084.0\n","15 444246.8125\n","16 385582.28125\n","17 336362.40625\n","18 294699.46875\n","19 259166.90625\n","20 228702.5\n","21 202486.90625\n","22 179859.875\n","23 160219.0\n","24 143104.953125\n","25 128134.53125\n","26 114991.09375\n","27 103418.46875\n","28 93192.9140625\n","29 84138.65625\n","30 76101.2265625\n","31 68947.75\n","32 62566.70703125\n","33 56862.63671875\n","34 51753.953125\n","35 47169.6796875\n","36 43048.3671875\n","37 39336.7421875\n","38 35989.59765625\n","39 32966.8125\n","40 30231.337890625\n","41 27755.0\n","42 25508.70703125\n","43 23467.92578125\n","44 21611.05078125\n","45 19919.1328125\n","46 18376.390625\n","47 16968.03515625\n","48 15680.025390625\n","49 14501.1201171875\n","50 13421.32421875\n","51 12430.9931640625\n","52 11522.4794921875\n","53 10688.0927734375\n","54 9920.876953125\n","55 9214.69921875\n","56 8564.580078125\n","57 7965.54150390625\n","58 7412.76611328125\n","59 6902.54736328125\n","60 6430.8701171875\n","61 5994.884765625\n","62 5591.64794921875\n","63 5218.224609375\n","64 4872.337890625\n","65 4552.07666015625\n","66 4255.0380859375\n","67 3979.26513671875\n","68 3723.124267578125\n","69 3485.017333984375\n","70 3263.70751953125\n","71 3057.736083984375\n","72 2865.99951171875\n","73 2687.453125\n","74 2521.03271484375\n","75 2365.88818359375\n","76 2221.166259765625\n","77 2086.080810546875\n","78 1959.988525390625\n","79 1842.238525390625\n","80 1732.177001953125\n","81 1629.257080078125\n","82 1533.0323486328125\n","83 1442.976318359375\n","84 1358.662353515625\n","85 1279.69873046875\n","86 1205.690673828125\n","87 1136.2850341796875\n","88 1071.219482421875\n","89 1010.1729736328125\n","90 952.8662109375\n","91 899.0614013671875\n","92 848.5516357421875\n","93 801.0734252929688\n","94 756.4507446289062\n","95 714.4818725585938\n","96 675.029296875\n","97 637.8980712890625\n","98 602.9478149414062\n","99 570.0777587890625\n","100 539.1126708984375\n","101 509.9311218261719\n","102 482.4452819824219\n","103 456.54052734375\n","104 432.1116027832031\n","105 409.07916259765625\n","106 387.3515930175781\n","107 366.85430908203125\n","108 347.5076904296875\n","109 329.2510070800781\n","110 311.9998779296875\n","111 295.7127685546875\n","112 280.3251953125\n","113 265.7820739746094\n","114 252.03643798828125\n","115 239.05079650878906\n","116 226.7645263671875\n","117 215.15228271484375\n","118 204.16073608398438\n","119 193.7593231201172\n","120 183.92152404785156\n","121 174.60433959960938\n","122 165.78285217285156\n","123 157.435302734375\n","124 149.52874755859375\n","125 142.04165649414062\n","126 134.9418182373047\n","127 128.21469116210938\n","128 121.840576171875\n","129 115.79732513427734\n","130 110.06706237792969\n","131 104.63426971435547\n","132 99.48480987548828\n","133 94.59617614746094\n","134 89.95928955078125\n","135 85.5617904663086\n","136 81.38603973388672\n","137 77.4232406616211\n","138 73.66195678710938\n","139 70.09020233154297\n","140 66.6975326538086\n","141 63.47745132446289\n","142 60.418914794921875\n","143 57.513427734375\n","144 54.752872467041016\n","145 52.1298942565918\n","146 49.63518142700195\n","147 47.26597595214844\n","148 45.01449203491211\n","149 42.87279510498047\n","150 40.837249755859375\n","151 38.90419387817383\n","152 37.063472747802734\n","153 35.31340026855469\n","154 33.647926330566406\n","155 32.06343078613281\n","156 30.556894302368164\n","157 29.12278938293457\n","158 27.758651733398438\n","159 26.46048927307129\n","160 25.224287033081055\n","161 24.04825210571289\n","162 22.928606033325195\n","163 21.86137580871582\n","164 20.84651756286621\n","165 19.879581451416016\n","166 18.959009170532227\n","167 18.08249282836914\n","168 17.2475643157959\n","169 16.451807022094727\n","170 15.693886756896973\n","171 14.972196578979492\n","172 14.284004211425781\n","173 13.628701210021973\n","174 13.003722190856934\n","175 12.407806396484375\n","176 11.840825080871582\n","177 11.299882888793945\n","178 10.784403800964355\n","179 10.292645454406738\n","180 9.824007034301758\n","181 9.378138542175293\n","182 8.952022552490234\n","183 8.545862197875977\n","184 8.158492088317871\n","185 7.789312839508057\n","186 7.43667459487915\n","187 7.100548267364502\n","188 6.779966354370117\n","189 6.474254608154297\n","190 6.182461738586426\n","191 5.904661178588867\n","192 5.6391520500183105\n","193 5.385613918304443\n","194 5.144269943237305\n","195 4.9135355949401855\n","196 4.69342565536499\n","197 4.483362674713135\n","198 4.282756805419922\n","199 4.091649532318115\n","200 3.908834218978882\n","201 3.734527826309204\n","202 3.56801176071167\n","203 3.409372568130493\n","204 3.2575769424438477\n","205 3.1127262115478516\n","206 2.9744369983673096\n","207 2.842426061630249\n","208 2.7163805961608887\n","209 2.596045732498169\n","210 2.480999708175659\n","211 2.371135711669922\n","212 2.2664847373962402\n","213 2.166365146636963\n","214 2.0707194805145264\n","215 1.9793181419372559\n","216 1.8922866582870483\n","217 1.8087748289108276\n","218 1.729138970375061\n","219 1.6531976461410522\n","220 1.5804344415664673\n","221 1.5109477043151855\n","222 1.444584846496582\n","223 1.381103754043579\n","224 1.3206056356430054\n","225 1.2628023624420166\n","226 1.2074663639068604\n","227 1.154552698135376\n","228 1.1041685342788696\n","229 1.0558298826217651\n","230 1.0096389055252075\n","231 0.9656803011894226\n","232 0.9235422611236572\n","233 0.883255124092102\n","234 0.8447402715682983\n","235 0.8080141544342041\n","236 0.7726383805274963\n","237 0.7391276359558105\n","238 0.7070517539978027\n","239 0.6763253211975098\n","240 0.6469202041625977\n","241 0.6187861561775208\n","242 0.5919473767280579\n","243 0.5663126707077026\n","244 0.5417301058769226\n","245 0.5182839632034302\n","246 0.49583446979522705\n","247 0.4744163453578949\n","248 0.45389723777770996\n","249 0.4342791736125946\n","250 0.4155098497867584\n","251 0.3976253569126129\n","252 0.3803897500038147\n","253 0.36399170756340027\n","254 0.34828317165374756\n","255 0.33320775628089905\n","256 0.3189050257205963\n","257 0.3051706850528717\n","258 0.29200735688209534\n","259 0.2794755697250366\n","260 0.26745277643203735\n","261 0.2559502422809601\n","262 0.24498412013053894\n","263 0.23436814546585083\n","264 0.2243644744157791\n","265 0.21471481025218964\n","266 0.20551921427249908\n","267 0.19672563672065735\n","268 0.18826879560947418\n","269 0.1802302449941635\n","270 0.17253759503364563\n","271 0.1651214212179184\n","272 0.15808063745498657\n","273 0.15130071341991425\n","274 0.14483536779880524\n","275 0.13866271078586578\n","276 0.13275255262851715\n","277 0.12707577645778656\n","278 0.12162193655967712\n","279 0.11644738167524338\n","280 0.11147476732730865\n","281 0.10674048960208893\n","282 0.10214743763208389\n","283 0.09781557321548462\n","284 0.09363682568073273\n","285 0.08970026671886444\n","286 0.08588168025016785\n","287 0.08222309499979019\n","288 0.07872186601161957\n","289 0.07536078244447708\n","290 0.07217703014612198\n","291 0.06912628561258316\n","292 0.06616819649934769\n","293 0.06337381154298782\n","294 0.06068961322307587\n","295 0.05813471972942352\n","296 0.055654365569353104\n","297 0.053286291658878326\n","298 0.051050130277872086\n","299 0.04888896644115448\n","300 0.046843335032463074\n","301 0.0448552705347538\n","302 0.04294070973992348\n","303 0.04113292321562767\n","304 0.03941679745912552\n","305 0.03774039447307587\n","306 0.03615690395236015\n","307 0.0346299409866333\n","308 0.03317772224545479\n","309 0.03178500011563301\n","310 0.030460001900792122\n","311 0.029177721589803696\n","312 0.027949133887887\n","313 0.026782605797052383\n","314 0.025648269802331924\n","315 0.024571914225816727\n","316 0.023563409224152565\n","317 0.022572506219148636\n","318 0.02163667604327202\n","319 0.0207465048879385\n","320 0.01988750696182251\n","321 0.019055478274822235\n","322 0.018258484080433846\n","323 0.017503296956419945\n","324 0.01677822507917881\n","325 0.016086537390947342\n","326 0.015423336997628212\n","327 0.014780424535274506\n","328 0.014174015261232853\n","329 0.013586149550974369\n","330 0.01302958745509386\n","331 0.01249145157635212\n","332 0.011975903995335102\n","333 0.01148656103760004\n","334 0.01101754605770111\n","335 0.010561277158558369\n","336 0.010131088085472584\n","337 0.009720184840261936\n","338 0.009327564388513565\n","339 0.008947118185460567\n","340 0.00858074240386486\n","341 0.0082389609888196\n","342 0.007906045764684677\n","343 0.007591036148369312\n","344 0.007290140260010958\n","345 0.006997601594775915\n","346 0.006719572469592094\n","347 0.006450452376157045\n","348 0.006191431079059839\n","349 0.0059457700699567795\n","350 0.005708279088139534\n","351 0.005488603375852108\n","352 0.005272379610687494\n","353 0.005066945683211088\n","354 0.004868091084063053\n","355 0.004675330128520727\n","356 0.0044974032789468765\n","357 0.004322992172092199\n","358 0.004157974384725094\n","359 0.004000678192824125\n","360 0.003846355015411973\n","361 0.003699430264532566\n","362 0.0035638303961604834\n","363 0.0034258700907230377\n","364 0.003296981565654278\n","365 0.00317469728179276\n","366 0.003057644469663501\n","367 0.0029439316131174564\n","368 0.002835270017385483\n","369 0.0027320501394569874\n","370 0.002631496638059616\n","371 0.0025385746266692877\n","372 0.0024453753139823675\n","373 0.0023570891935378313\n","374 0.002273492980748415\n","375 0.002190931933000684\n","376 0.002112568588927388\n","377 0.00203891028650105\n","378 0.001967614283785224\n","379 0.0018983352929353714\n","380 0.00183411396574229\n","381 0.0017684543272480369\n","382 0.0017063122941181064\n","383 0.0016501310747116804\n","384 0.0015949226217344403\n","385 0.0015407565515488386\n","386 0.0014881053939461708\n","387 0.0014364065136760473\n","388 0.0013879518955945969\n","389 0.0013420196482911706\n","390 0.0012980588944628835\n","391 0.0012552628759294748\n","392 0.0012147064553573728\n","393 0.0011766848620027304\n","394 0.0011397666530683637\n","395 0.0011029805755242705\n","396 0.0010671732015907764\n","397 0.0010342643363401294\n","398 0.0010033316211774945\n","399 0.0009717961074784398\n","400 0.0009409382473677397\n","401 0.000911536393687129\n","402 0.0008838438079692423\n","403 0.0008566551841795444\n","404 0.0008313035941682756\n","405 0.0008069210452958941\n","406 0.0007830705726519227\n","407 0.0007602998521178961\n","408 0.0007371755782514811\n","409 0.00071621936513111\n","410 0.0006960426107980311\n","411 0.0006745756836608052\n","412 0.000656110409181565\n","413 0.0006374438526108861\n","414 0.000618890393525362\n","415 0.0006027420167811215\n","416 0.000585744681302458\n","417 0.0005691451369784772\n","418 0.0005535691161639988\n","419 0.0005383184761740267\n","420 0.0005238940357230604\n","421 0.0005101036513224244\n","422 0.0004965361440554261\n","423 0.00048366919509135187\n","424 0.00047095323679968715\n","425 0.0004581452812999487\n","426 0.0004467568069230765\n","427 0.0004349001683294773\n","428 0.00042390028829686344\n","429 0.0004125982231926173\n","430 0.000403369078412652\n","431 0.00039269495755434036\n","432 0.0003829888883046806\n","433 0.000373457238310948\n","434 0.000365729967597872\n","435 0.00035610617487691343\n","436 0.0003478454891592264\n","437 0.0003394202794879675\n","438 0.00033142371103167534\n","439 0.0003229599678888917\n","440 0.00031534029403701425\n","441 0.00030866661109030247\n","442 0.0003012513625435531\n","443 0.0002933612558990717\n","444 0.00028732576174661517\n","445 0.0002811868616845459\n","446 0.00027491123182699084\n","447 0.0002684291685000062\n","448 0.0002629145164974034\n","449 0.00025693149655126035\n","450 0.0002513907093089074\n","451 0.0002460792602505535\n","452 0.00024101916642393917\n","453 0.00023613522353116423\n","454 0.00023081933613866568\n","455 0.00022596886265091598\n","456 0.0002209968661190942\n","457 0.00021688442211598158\n","458 0.0002123660669894889\n","459 0.0002078908437397331\n","460 0.00020359497284516692\n","461 0.00020003519603051245\n","462 0.00019633739429991692\n","463 0.00019169731240253896\n","464 0.00018778308003675193\n","465 0.00018471777730155736\n","466 0.00018130100215785205\n","467 0.00017722406482789665\n","468 0.00017390798893757164\n","469 0.00017094332724809647\n","470 0.00016728435002733022\n","471 0.00016430353571195155\n","472 0.00016124456305988133\n","473 0.00015793894999660552\n","474 0.00015575111319776624\n","475 0.00015240134962368757\n","476 0.0001494111929787323\n","477 0.00014706028741784394\n","478 0.0001449049450457096\n","479 0.00014231419481802732\n","480 0.0001395776926074177\n","481 0.0001371749531244859\n","482 0.00013492506695911288\n","483 0.00013248127652332187\n","484 0.0001300826552323997\n","485 0.00012812578643206507\n","486 0.00012603755749296397\n","487 0.00012380543921608478\n","488 0.00012168799003120512\n","489 0.00011994400847470388\n","490 0.00011773573351092637\n","491 0.00011620496661635116\n","492 0.00011427961726440117\n","493 0.00011205831833649427\n","494 0.00011049464228563011\n","495 0.00010885603842325509\n","496 0.00010712756920838729\n","497 0.00010532654414419085\n","498 0.00010388618829892948\n","499 0.0001019661794998683\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XMPZ-rZOxMXE","colab_type":"code","colab":{}},"source":["test_x = torch.randn(64, 1000)\n","text_y = torch.randn(64, 10)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"_VpfG7MLZibz","colab_type":"code","outputId":"b82fb1f6-534e-45a2-a3a0-6bcb3e217c1c","executionInfo":{"status":"ok","timestamp":1563861689995,"user_tz":-480,"elapsed":1051,"user":{"displayName":"Tuvshinbayar Tuvshinzul","photoUrl":"","userId":"01278297265491069175"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["\n","y_pred = test_x.mm(w1).clamp(min=0).mm(w2)\n","loss = (y_pred - y).pow(2).sum()\n","print(loss.item())"],"execution_count":9,"outputs":[{"output_type":"stream","text":["14135488.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"832hMenLZ1Zw","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"u61bZKJ2eXj-","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}