Sample Code/Day_17_SampleCode.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"範例.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"13QP9FNpt54x"},"source":["範例目標:<br>\r\n","1. 實做效能調校\r\n","2. 大型資料集處理\r\n"]},{"cell_type":"markdown","metadata":{"id":"iKDNxBy3t6QM"},"source":["範例重點:<br>\r\n","1. 在這裡介紹3種加速方法針對 Pandas ，在 Python 中還有很多方式可以提升效能\r\n","2. 欄位的型態降級有助於減少記憶體佔用空間"]},{"cell_type":"code","metadata":{"id":"tY22_2dwt6b8"},"source":["import pandas as pd\r\n","import numpy as np \r\n","import time"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":514},"id":"HYUC-ohZAWLt","executionInfo":{"status":"ok","timestamp":1608482170490,"user_tz":-480,"elapsed":597,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"5b2986cb-71f3-49c4-cc86-501f60b0d7b0"},"source":["score_df = pd.DataFrame([[1,50,80,70,'boy',1], \r\n","              [2,60,45,50,'boy',2],\r\n","              [3,98,43,55,'boy',1],\r\n","              [4,70,69,89,'boy',2],\r\n","              [5,56,79,60,'girl',1],\r\n","              [6,60,68,55,'girl',2],\r\n","              [7,45,70,77,'girl',1],\r\n","              [8,55,77,76,'girl',2],\r\n","              [9,25,57,60,'girl',1],\r\n","              [10,88,40,43,'girl',3],\r\n","              [11,25,60,45,'boy',3],\r\n","              [12,80,60,23,'boy',3],\r\n","              [13,20,90,66,'girl',3],\r\n","              [14,50,50,50,'girl',3],\r\n","              [15,89,67,77,'girl',3]],columns=['student_id','math_score','english_score','chinese_score','sex','class'])\r\n","score_df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>student_id</th>\n","      <th>math_score</th>\n","      <th>english_score</th>\n","      <th>chinese_score</th>\n","      <th>sex</th>\n","      <th>class</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>50</td>\n","      <td>80</td>\n","      <td>70</td>\n","      <td>boy</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>60</td>\n","      <td>45</td>\n","      <td>50</td>\n","      <td>boy</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>98</td>\n","      <td>43</td>\n","      <td>55</td>\n","      <td>boy</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>70</td>\n","      <td>69</td>\n","      <td>89</td>\n","      <td>boy</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>56</td>\n","      <td>79</td>\n","      <td>60</td>\n","      <td>girl</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>6</td>\n","      <td>60</td>\n","      <td>68</td>\n","      <td>55</td>\n","      <td>girl</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>7</td>\n","      <td>45</td>\n","      <td>70</td>\n","      <td>77</td>\n","      <td>girl</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>8</td>\n","      <td>55</td>\n","      <td>77</td>\n","      <td>76</td>\n","      <td>girl</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>9</td>\n","      <td>25</td>\n","      <td>57</td>\n","      <td>60</td>\n","      <td>girl</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>10</td>\n","      <td>88</td>\n","      <td>40</td>\n","      <td>43</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>11</td>\n","      <td>25</td>\n","      <td>60</td>\n","      <td>45</td>\n","      <td>boy</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>12</td>\n","      <td>80</td>\n","      <td>60</td>\n","      <td>23</td>\n","      <td>boy</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>13</td>\n","      <td>20</td>\n","      <td>90</td>\n","      <td>66</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>14</td>\n","      <td>50</td>\n","      <td>50</td>\n","      <td>50</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>15</td>\n","      <td>89</td>\n","      <td>67</td>\n","      <td>77</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["    student_id  math_score  english_score  chinese_score   sex  class\n","0            1          50             80             70   boy      1\n","1            2          60             45             50   boy      2\n","2            3          98             43             55   boy      1\n","3            4          70             69             89   boy      2\n","4            5          56             79             60  girl      1\n","5            6          60             68             55  girl      2\n","6            7          45             70             77  girl      1\n","7            8          55             77             76  girl      2\n","8            9          25             57             60  girl      1\n","9           10          88             40             43  girl      3\n","10          11          25             60             45   boy      3\n","11          12          80             60             23   boy      3\n","12          13          20             90             66  girl      3\n","13          14          50             50             50  girl      3\n","14          15          89             67             77  girl      3"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eml9UysOAiH9","executionInfo":{"status":"ok","timestamp":1608482367128,"user_tz":-480,"elapsed":818,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"1a440a50-a13b-4d1a-b2d1-4b5a26d37491"},"source":["#agg使用Python的內建函式\r\n","star_time = time.time()\r\n","score_df.groupby('class').agg('mean')\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.0032105445861816406"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3uHEbBcPAnzJ","executionInfo":{"status":"ok","timestamp":1608482368646,"user_tz":-480,"elapsed":886,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"a367ace2-96e2-407f-b54b-511960611959"},"source":["##agg使用自定義函式\r\n","star_time = time.time()\r\n","score_df.groupby('class').agg(lambda x: x.mean())\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.01680469512939453"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"u5Y595E2Bhhj","executionInfo":{"status":"ok","timestamp":1608482441432,"user_tz":-480,"elapsed":885,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"a2b652fd-71c4-4562-d46a-5247676acb26"},"source":["#transform使用Python的內建函式\r\n","star_time = time.time()\r\n","score_df.groupby('class').transform('mean')\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.019934415817260742"]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"rdwjbA-DBMg9","executionInfo":{"status":"ok","timestamp":1608482424820,"user_tz":-480,"elapsed":903,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"9d5f3fcf-689b-42fc-8763-782cd3fa8fa0"},"source":["##transform使用自定義函式\r\n","star_time = time.time()\r\n","score_df.groupby('class').transform(lambda x: x.mean())\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.027081966400146484"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"markdown","metadata":{"id":"kUtIQZk_WoDq"},"source":["做到這邊可以看出在使用agg和transform進行操作時，儘量使用Python的內建函式，能夠提高執行效率"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":514},"id":"CW6AJNSBBgRE","executionInfo":{"status":"ok","timestamp":1608482902314,"user_tz":-480,"elapsed":834,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"8da69215-c810-4903-c480-d0e38bf32465"},"source":["score_df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>student_id</th>\n","      <th>math_score</th>\n","      <th>english_score</th>\n","      <th>chinese_score</th>\n","      <th>sex</th>\n","      <th>class</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>50</td>\n","      <td>80</td>\n","      <td>70</td>\n","      <td>boy</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>60</td>\n","      <td>45</td>\n","      <td>50</td>\n","      <td>boy</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>98</td>\n","      <td>43</td>\n","      <td>55</td>\n","      <td>boy</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>70</td>\n","      <td>69</td>\n","      <td>89</td>\n","      <td>boy</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>56</td>\n","      <td>79</td>\n","      <td>60</td>\n","      <td>girl</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>6</td>\n","      <td>60</td>\n","      <td>68</td>\n","      <td>55</td>\n","      <td>girl</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>7</td>\n","      <td>45</td>\n","      <td>70</td>\n","      <td>77</td>\n","      <td>girl</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>8</td>\n","      <td>55</td>\n","      <td>77</td>\n","      <td>76</td>\n","      <td>girl</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>9</td>\n","      <td>25</td>\n","      <td>57</td>\n","      <td>60</td>\n","      <td>girl</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>10</td>\n","      <td>88</td>\n","      <td>40</td>\n","      <td>43</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>11</td>\n","      <td>25</td>\n","      <td>60</td>\n","      <td>45</td>\n","      <td>boy</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>12</td>\n","      <td>80</td>\n","      <td>60</td>\n","      <td>23</td>\n","      <td>boy</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>13</td>\n","      <td>20</td>\n","      <td>90</td>\n","      <td>66</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>14</td>\n","      <td>50</td>\n","      <td>50</td>\n","      <td>50</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>15</td>\n","      <td>89</td>\n","      <td>67</td>\n","      <td>77</td>\n","      <td>girl</td>\n","      <td>3</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["    student_id  math_score  english_score  chinese_score   sex  class\n","0            1          50             80             70   boy      1\n","1            2          60             45             50   boy      2\n","2            3          98             43             55   boy      1\n","3            4          70             69             89   boy      2\n","4            5          56             79             60  girl      1\n","5            6          60             68             55  girl      2\n","6            7          45             70             77  girl      1\n","7            8          55             77             76  girl      2\n","8            9          25             57             60  girl      1\n","9           10          88             40             43  girl      3\n","10          11          25             60             45   boy      3\n","11          12          80             60             23   boy      3\n","12          13          20             90             66  girl      3\n","13          14          50             50             50  girl      3\n","14          15          89             67             77  girl      3"]},"metadata":{"tags":[]},"execution_count":15}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Ywu42JWzDU2X","executionInfo":{"status":"ok","timestamp":1608483046040,"user_tz":-480,"elapsed":892,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"8f1af192-4e6b-4bf3-e9e8-c4bbe39bc2ed"},"source":["#篩選出對應資料\r\n","#用list方式搜索\r\n","score_df1 = score_df.copy()\r\n","star_time = time.time()\r\n","score_df1['Pass_math'] = [i>=60 for i in score_df1.math_score]\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.0021719932556152344"]},"metadata":{"tags":[]},"execution_count":16}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"k-kyAsjaEK-r","executionInfo":{"status":"ok","timestamp":1608483137998,"user_tz":-480,"elapsed":761,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"7a90911f-9124-4e85-e29a-a7083523a248"},"source":["#用DataFrame column方式搜索\r\n","score_df1 = score_df.copy()\r\n","star_time = time.time()\r\n","score_df1['Pass_math'] = score_df1.math_score>=60\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.02496051788330078"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fh1OLu4PEBFV","executionInfo":{"status":"ok","timestamp":1608483211273,"user_tz":-480,"elapsed":868,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"ad4488f4-ec33-4829-edb0-7c9dccf42104"},"source":["#用自定義式搜索\r\n","score_df2 = score_df.copy()\r\n","star_time = time.time()\r\n","score_df2['Pass_math'] = score_df2.math_score.apply(lambda x : x>=60)\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.0016407966613769531"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PLNhpIzsD37u","executionInfo":{"status":"ok","timestamp":1608483261538,"user_tz":-480,"elapsed":768,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"8aee9e23-05f0-4078-c34d-2c1283f51d24"},"source":["#用isin()\r\n","score_df3 = score_df.copy()\r\n","star_time = time.time()\r\n","score_df3['Pass_math'] = score_df3.math_score.isin(range(60, 100))\r\n","end_time = time.time()\r\n","end_time - star_time"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.0014753341674804688"]},"metadata":{"tags":[]},"execution_count":19}]},{"cell_type":"markdown","metadata":{"id":"1pnoWdwIXb14"},"source":["以上實驗看出採用isin() 篩選出對應資料室最快的，速度快是因為它採用了向量化的資料處理方式（這裡的isin() 是其中一種方式，還有其他方式，大家可以嘗試一下）"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bPr6Y3iMTkSk","executionInfo":{"status":"ok","timestamp":1608487186281,"user_tz":-480,"elapsed":464,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"63b8fd33-a1f7-4ca6-efea-44be615d1126"},"source":["np.random.randint(3,9,10)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([3, 7, 5, 5, 8, 4, 5, 8, 6, 3])"]},"metadata":{"tags":[]},"execution_count":48}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PVN_qrb4SJaB","executionInfo":{"status":"ok","timestamp":1608488258514,"user_tz":-480,"elapsed":1483,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"44fd6932-acd9-4769-ebc6-ab9a2f30dbc3"},"source":["#遇到大資料集時，常有記憶體不足的問題\r\n","#首先先生成大資料，因為改善部分不同所以分成浮點數float與整數int的資料集，可以看到不管浮點數還是整數都佔了800128bytes\r\n","float_data = pd.DataFrame(np.random.uniform(0,5,100000).reshape(1000,100))\r\n","int_data = pd.DataFrame(np.random.randint(0,1000,100000).reshape(1000,100))\r\n","int_data.memory_usage(deep=True).sum(), float_data.memory_usage(deep=True).sum()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(800128, 800128)"]},"metadata":{"tags":[]},"execution_count":75}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"U5_Y9siSUYPY","executionInfo":{"status":"ok","timestamp":1608487827843,"user_tz":-480,"elapsed":816,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"653e5c09-bc82-462b-e381-b666d9cc2034"},"source":["#整數型態int改成uint減少記憶體正用空間，使用前800128bytes，使用後剩下200128bytes\r\n","downcast_int = int_data.apply(pd.to_numeric,downcast='unsigned')\r\n","int_data.memory_usage(deep=True).sum(),downcast_int.memory_usage(deep=True).sum()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(800128, 200128)"]},"metadata":{"tags":[]},"execution_count":72}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":111},"id":"coA4a_BDVBaO","executionInfo":{"status":"ok","timestamp":1608487803424,"user_tz":-480,"elapsed":878,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"6fc57e44-0213-49fc-bd59-b009d3092d9c"},"source":["#原本有100個欄位是int64，經過downcast變成了100個欄位的uint16\r\n","compare_int = pd.concat([int_data.dtypes,downcast_int.dtypes],axis=1)\r\n","compare_int.columns = ['before','after']\r\n","compare_int.apply(pd.value_counts)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>before</th>\n","      <th>after</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>uint16</th>\n","      <td>NaN</td>\n","      <td>100.0</td>\n","    </tr>\n","    <tr>\n","      <th>int64</th>\n","      <td>100.0</td>\n","      <td>NaN</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        before  after\n","uint16     NaN  100.0\n","int64    100.0    NaN"]},"metadata":{"tags":[]},"execution_count":71}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"x7C0PsR-UPy-","executionInfo":{"status":"ok","timestamp":1608488736543,"user_tz":-480,"elapsed":821,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"afcf3b97-3e04-4737-88fb-f55a25396ccb"},"source":["#浮點數型態float64改成float32減少記憶體正用空間，使用前800128bytes，使用後剩下400128bytes\r\n","downcast_float = float_data.apply(pd.to_numeric,downcast='float')\r\n","float_data.memory_usage(deep=True).sum(),downcast_float.memory_usage(deep=True).sum()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(800128, 400128)"]},"metadata":{"tags":[]},"execution_count":76}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":111},"id":"cT4H5tT5UXLB","executionInfo":{"status":"ok","timestamp":1608488736990,"user_tz":-480,"elapsed":1087,"user":{"displayName":"献竤黃","photoUrl":"","userId":"07529243043474362942"}},"outputId":"1d252215-08c6-44a5-da59-f0602df10f3f"},"source":["#原本有100個欄位是float64，經過downcast變成了100個欄位的float32\r\n","compare_int = pd.concat([float_data.dtypes,downcast_float.dtypes],axis=1)\r\n","compare_int.columns = ['before','after']\r\n","compare_int.apply(pd.value_counts)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>before</th>\n","      <th>after</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>float32</th>\n","      <td>NaN</td>\n","      <td>100.0</td>\n","    </tr>\n","    <tr>\n","      <th>float64</th>\n","      <td>100.0</td>\n","      <td>NaN</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["         before  after\n","float32     NaN  100.0\n","float64   100.0    NaN"]},"metadata":{"tags":[]},"execution_count":77}]}]}