MNIST_TF2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib widget\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# check gpu\n",
    "tf.config.list_physical_devices('GPU') \n",
    "tf.test.is_built_with_cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 觀察數據集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, (60000, 28, 28), dtype('uint8'))"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(train_data, train_label), (_, _) = tf.keras.datasets.mnist.load_data()\n",
    "train_data.ndim, train_data.shape, train_data.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7941168beb6f4a529987a6faecacb179",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(5, dtype('uint8'))"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plt.imshow(train_data[0])\n",
    "plt.show()\n",
    "train_label[0], train_label.dtype"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 撰寫讀取資料class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MNISTLoader():\n",
    "    def __init__(self):\n",
    "        \n",
    "        (self.train_data, self.train_label), (self.test_data, self.test_label) = tf.keras.datasets.mnist.load_data()\n",
    "        # prep data\n",
    "        self.train_data = self.train_data.astype(np.float32) / 255.0\n",
    "        self.test_data = self.test_data.astype(np.float32) / 255.0\n",
    "        # add color channel\n",
    "        self.train_data = np.expand_dims(self.train_data, axis=-1)      # shape:[60000, 28, 28] -> [60000, 28, 28, 1]\n",
    "        self.test_data = np.expand_dims(self.test_data, axis=-1)        # [10000, 28, 28] -> [10000, 28, 28, 1]\n",
    "        self.train_label = self.train_label.astype(np.int32)    # uint8 -> int32\n",
    "        self.test_label = self.test_label.astype(np.int32)      # uint8 -> int32\n",
    "        \n",
    "        self.num_train_data, self.num_test_data = self.train_data.shape[0], self.test_data.shape[0]\n",
    "\n",
    "    def get_batch(self, batch_size):\n",
    "        # random batch size\n",
    "        index = np.random.randint(0, self.num_train_data, batch_size)\n",
    "        return self.train_data[index, :], self.train_label[index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 方法一"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 開發MLP Model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(tf.keras.Model):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.flatten = tf.keras.layers.Flatten()\n",
    "        self.dense1 = tf.keras.layers.Dense(units=100, activation=tf.nn.relu)\n",
    "        self.dense2 = tf.keras.layers.Dense(units=20, activation=tf.nn.leaky_relu)\n",
    "        self.dense3 = tf.keras.layers.Dense(units=10)\n",
    "\n",
    "    @tf.function\n",
    "    def call(self, inputs):  # [batch_size, 28, 28, 1]\n",
    "        flat1 = self.flatten(inputs)  # [batch_size, 784]\n",
    "        dens1 = self.dense1(flat1)  # [batch_size, 100]\n",
    "        dens2 = self.dense2(dens1)  # [batch_size, 20]\n",
    "        dens3 = self.dense3(dens2)  # [batch_size, 10]\n",
    "        output = tf.nn.softmax(dens3)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_epochs = 5\n",
    "batch_size = 2000\n",
    "learning_rate = 0.0015\n",
    "\n",
    "model = MLP()\n",
    "data_loader = MNISTLoader()\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "model_name = \"mnist_model1\"\n",
    "log_dir = 'tensorboard'\n",
    "summary_writer = tf.summary.create_file_writer(log_dir)\n",
    "tf.summary.trace_on(profiler=True)\n",
    "\n",
    "num_batches = int(data_loader.num_train_data // batch_size * num_epochs)\n",
    "num_batches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 訓練"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 loss 2.3598012924194336 [2.35980129]\n",
      "1 loss 2.2228996753692627 [2.22289968]\n",
      "2 loss 2.110011339187622 [2.11001134]\n",
      "3 loss 2.021075487136841 [2.02107549]\n",
      "4 loss 1.917817234992981 [1.91781723]\n",
      "5 loss 1.8207504749298096 [1.82075047]\n",
      "6 loss 1.7157223224639893 [1.71572232]\n",
      "7 loss 1.6493419408798218 [1.64934194]\n",
      "8 loss 1.5577243566513062 [1.55772436]\n",
      "9 loss 1.4631267786026 [1.46312678]\n",
      "10 loss 1.4052798748016357 [1.40527987]\n",
      "11 loss 1.300802230834961 [1.30080223]\n",
      "12 loss 1.2274671792984009 [1.22746718]\n",
      "13 loss 1.1776010990142822 [1.1776011]\n",
      "14 loss 1.11951744556427 [1.11951745]\n",
      "15 loss 1.0440082550048828 [1.04400826]\n",
      "16 loss 1.0039774179458618 [1.00397742]\n",
      "17 loss 0.9484937191009521 [0.948493719]\n",
      "18 loss 0.9291660189628601 [0.929166]\n",
      "19 loss 0.8487364053726196 [0.848736405]\n",
      "20 loss 0.8097838163375854 [0.809783816]\n",
      "21 loss 0.7894232273101807 [0.789423227]\n",
      "22 loss 0.7549102902412415 [0.75491029]\n",
      "23 loss 0.7469093203544617 [0.74690932]\n",
      "24 loss 0.7053620219230652 [0.705362]\n",
      "25 loss 0.6722714900970459 [0.67227149]\n",
      "26 loss 0.5972467660903931 [0.597246766]\n",
      "27 loss 0.6541031002998352 [0.6541031]\n",
      "28 loss 0.626574695110321 [0.626574695]\n",
      "29 loss 0.5659906268119812 [0.565990627]\n",
      "30 loss 0.543681800365448 [0.5436818]\n",
      "31 loss 0.5469489097595215 [0.54694891]\n",
      "32 loss 0.4940531849861145 [0.494053185]\n",
      "33 loss 0.5358554124832153 [0.535855412]\n",
      "34 loss 0.4881407916545868 [0.488140792]\n",
      "35 loss 0.47165676951408386 [0.47165677]\n",
      "36 loss 0.46460798382759094 [0.464607984]\n",
      "37 loss 0.4977644979953766 [0.497764498]\n",
      "38 loss 0.44924044609069824 [0.449240446]\n",
      "39 loss 0.4594934284687042 [0.459493428]\n",
      "40 loss 0.42680269479751587 [0.426802695]\n",
      "41 loss 0.431165874004364 [0.431165874]\n",
      "42 loss 0.4292805790901184 [0.429280579]\n",
      "43 loss 0.4216812252998352 [0.421681225]\n",
      "44 loss 0.3928995430469513 [0.392899543]\n",
      "45 loss 0.4322985112667084 [0.432298511]\n",
      "46 loss 0.42526257038116455 [0.42526257]\n",
      "47 loss 0.39507725834846497 [0.395077258]\n",
      "48 loss 0.3686330318450928 [0.368633032]\n",
      "49 loss 0.4001661539077759 [0.400166154]\n",
      "50 loss 0.3749440610408783 [0.374944061]\n",
      "51 loss 0.3729262053966522 [0.372926205]\n",
      "52 loss 0.3758467435836792 [0.375846744]\n",
      "53 loss 0.336041659116745 [0.336041659]\n",
      "54 loss 0.36281871795654297 [0.362818718]\n",
      "55 loss 0.31197938323020935 [0.311979383]\n",
      "56 loss 0.3481505513191223 [0.348150551]\n",
      "57 loss 0.3541701138019562 [0.354170114]\n",
      "58 loss 0.3174252510070801 [0.317425251]\n",
      "59 loss 0.3252098858356476 [0.325209886]\n",
      "60 loss 0.339431494474411 [0.339431494]\n",
      "61 loss 0.3272565007209778 [0.327256501]\n",
      "62 loss 0.3449207544326782 [0.344920754]\n",
      "63 loss 0.28837987780570984 [0.288379878]\n",
      "64 loss 0.32328900694847107 [0.323289]\n",
      "65 loss 0.3106970191001892 [0.310697019]\n",
      "66 loss 0.3212421238422394 [0.321242124]\n",
      "67 loss 0.3066505789756775 [0.306650579]\n",
      "68 loss 0.2999393939971924 [0.299939394]\n",
      "69 loss 0.3445238769054413 [0.344523877]\n",
      "70 loss 0.3205205202102661 [0.32052052]\n",
      "71 loss 0.30599820613861084 [0.305998206]\n",
      "72 loss 0.2715417146682739 [0.271541715]\n",
      "73 loss 0.31773442029953003 [0.31773442]\n",
      "74 loss 0.27088621258735657 [0.270886213]\n",
      "75 loss 0.29146069288253784 [0.291460693]\n",
      "76 loss 0.298077791929245 [0.298077792]\n",
      "77 loss 0.2754930555820465 [0.275493056]\n",
      "78 loss 0.2808499336242676 [0.280849934]\n",
      "79 loss 0.26797905564308167 [0.267979056]\n",
      "80 loss 0.30715352296829224 [0.307153523]\n",
      "81 loss 0.2796371579170227 [0.279637158]\n",
      "82 loss 0.2737596333026886 [0.273759633]\n",
      "83 loss 0.2615628242492676 [0.261562824]\n",
      "84 loss 0.2637423872947693 [0.263742387]\n",
      "85 loss 0.3146991431713104 [0.314699143]\n",
      "86 loss 0.2700166702270508 [0.27001667]\n",
      "87 loss 0.2716779410839081 [0.271677941]\n",
      "88 loss 0.2571932077407837 [0.257193208]\n",
      "89 loss 0.2688863277435303 [0.268886328]\n",
      "90 loss 0.28394678235054016 [0.283946782]\n",
      "91 loss 0.25702032446861267 [0.257020324]\n",
      "92 loss 0.24408459663391113 [0.244084597]\n",
      "93 loss 0.23555350303649902 [0.235553503]\n",
      "94 loss 0.24565264582633972 [0.245652646]\n",
      "95 loss 0.25573480129241943 [0.255734801]\n",
      "96 loss 0.2593725323677063 [0.259372532]\n",
      "97 loss 0.24096274375915527 [0.240962744]\n",
      "98 loss 0.25026389956474304 [0.2502639]\n",
      "99 loss 0.24832040071487427 [0.248320401]\n",
      "100 loss 0.2448604851961136 [0.244860485]\n",
      "101 loss 0.2680056095123291 [0.26800561]\n",
      "102 loss 0.25599977374076843 [0.255999774]\n",
      "103 loss 0.22818876802921295 [0.228188768]\n",
      "104 loss 0.23746639490127563 [0.237466395]\n",
      "105 loss 0.2297959327697754 [0.229795933]\n",
      "106 loss 0.24167431890964508 [0.241674319]\n",
      "107 loss 0.2223922610282898 [0.222392261]\n",
      "108 loss 0.24385757744312286 [0.243857577]\n",
      "109 loss 0.24036678671836853 [0.240366787]\n",
      "110 loss 0.23856760561466217 [0.238567606]\n",
      "111 loss 0.23136770725250244 [0.231367707]\n",
      "112 loss 0.2189778834581375 [0.218977883]\n",
      "113 loss 0.25204020738601685 [0.252040207]\n",
      "114 loss 0.22810311615467072 [0.228103116]\n",
      "115 loss 0.247786745429039 [0.247786745]\n",
      "116 loss 0.21231113374233246 [0.212311134]\n",
      "117 loss 0.232871875166893 [0.232871875]\n",
      "118 loss 0.2033296376466751 [0.203329638]\n",
      "119 loss 0.21877476572990417 [0.218774766]\n",
      "120 loss 0.20880688726902008 [0.208806887]\n",
      "121 loss 0.236892431974411 [0.236892432]\n",
      "122 loss 0.22448422014713287 [0.22448422]\n",
      "123 loss 0.20363262295722961 [0.203632623]\n",
      "124 loss 0.19836466014385223 [0.19836466]\n",
      "125 loss 0.19013231992721558 [0.19013232]\n",
      "126 loss 0.20713095366954803 [0.207130954]\n",
      "127 loss 0.19615541398525238 [0.196155414]\n",
      "128 loss 0.2121695876121521 [0.212169588]\n",
      "129 loss 0.21463432908058167 [0.214634329]\n",
      "130 loss 0.1980777382850647 [0.198077738]\n",
      "131 loss 0.19644422829151154 [0.196444228]\n",
      "132 loss 0.1960998922586441 [0.196099892]\n",
      "133 loss 0.1995798945426941 [0.199579895]\n",
      "134 loss 0.19728653132915497 [0.197286531]\n",
      "135 loss 0.22681327164173126 [0.226813272]\n",
      "136 loss 0.21027164161205292 [0.210271642]\n",
      "137 loss 0.16225895285606384 [0.162258953]\n",
      "138 loss 0.23379231989383698 [0.23379232]\n",
      "139 loss 0.21906347572803497 [0.219063476]\n",
      "140 loss 0.2088097482919693 [0.208809748]\n",
      "141 loss 0.2083718627691269 [0.208371863]\n",
      "142 loss 0.20971286296844482 [0.209712863]\n",
      "143 loss 0.2139372080564499 [0.213937208]\n",
      "144 loss 0.20630072057247162 [0.206300721]\n",
      "145 loss 0.17455869913101196 [0.174558699]\n",
      "146 loss 0.1796392947435379 [0.179639295]\n",
      "147 loss 0.20265743136405945 [0.202657431]\n",
      "148 loss 0.21189509332180023 [0.211895093]\n",
      "149 loss 0.19413284957408905 [0.19413285]\n",
      "0 loss 0.20635342597961426 [0.206353426]\n",
      "1 loss 0.21735043823719025 [0.217350438]\n",
      "2 loss 0.16427890956401825 [0.16427891]\n",
      "3 loss 0.18170097470283508 [0.181700975]\n",
      "4 loss 0.20049482583999634 [0.200494826]\n",
      "5 loss 0.18778100609779358 [0.187781]\n",
      "6 loss 0.18418213725090027 [0.184182137]\n",
      "7 loss 0.20482008159160614 [0.204820082]\n",
      "8 loss 0.18303059041500092 [0.18303059]\n",
      "9 loss 0.20289427042007446 [0.20289427]\n",
      "10 loss 0.17734386026859283 [0.17734386]\n",
      "11 loss 0.18060052394866943 [0.180600524]\n",
      "12 loss 0.1874982863664627 [0.187498286]\n",
      "13 loss 0.17909587919712067 [0.179095879]\n",
      "14 loss 0.18254974484443665 [0.182549745]\n",
      "15 loss 0.1884116232395172 [0.188411623]\n",
      "16 loss 0.2079501748085022 [0.207950175]\n",
      "17 loss 0.19562648236751556 [0.195626482]\n",
      "18 loss 0.1795085072517395 [0.179508507]\n",
      "19 loss 0.18180358409881592 [0.181803584]\n",
      "20 loss 0.19034168124198914 [0.190341681]\n",
      "21 loss 0.18030980229377747 [0.180309802]\n",
      "22 loss 0.19265510141849518 [0.192655101]\n",
      "23 loss 0.17214305698871613 [0.172143057]\n",
      "24 loss 0.16920946538448334 [0.169209465]\n",
      "25 loss 0.16901127994060516 [0.16901128]\n",
      "26 loss 0.1570795178413391 [0.157079518]\n",
      "27 loss 0.16643497347831726 [0.166434973]\n",
      "28 loss 0.1795707792043686 [0.179570779]\n",
      "29 loss 0.1685391366481781 [0.168539137]\n",
      "30 loss 0.1729629635810852 [0.172962964]\n",
      "31 loss 0.14996930956840515 [0.14996931]\n",
      "32 loss 0.17458172142505646 [0.174581721]\n",
      "33 loss 0.15742425620555878 [0.157424256]\n",
      "34 loss 0.15144816040992737 [0.15144816]\n",
      "35 loss 0.1854924112558365 [0.185492411]\n",
      "36 loss 0.16356387734413147 [0.163563877]\n",
      "37 loss 0.15271581709384918 [0.152715817]\n",
      "38 loss 0.15456831455230713 [0.154568315]\n",
      "39 loss 0.18739421665668488 [0.187394217]\n",
      "40 loss 0.2063666433095932 [0.206366643]\n",
      "41 loss 0.1946686953306198 [0.194668695]\n",
      "42 loss 0.19363956153392792 [0.193639562]\n",
      "43 loss 0.1658337414264679 [0.165833741]\n",
      "44 loss 0.1761888712644577 [0.176188871]\n",
      "45 loss 0.1680830717086792 [0.168083072]\n",
      "46 loss 0.14575667679309845 [0.145756677]\n",
      "47 loss 0.16332878172397614 [0.163328782]\n",
      "48 loss 0.14577023684978485 [0.145770237]\n",
      "49 loss 0.17171235382556915 [0.171712354]\n",
      "50 loss 0.1794860064983368 [0.179486]\n",
      "51 loss 0.1521042287349701 [0.152104229]\n",
      "52 loss 0.13982297480106354 [0.139822975]\n",
      "53 loss 0.15750336647033691 [0.157503366]\n",
      "54 loss 0.16363516449928284 [0.163635164]\n",
      "55 loss 0.1525917947292328 [0.152591795]\n",
      "56 loss 0.17547252774238586 [0.175472528]\n",
      "57 loss 0.16936790943145752 [0.169367909]\n",
      "58 loss 0.15970154106616974 [0.159701541]\n",
      "59 loss 0.14187577366828918 [0.141875774]\n",
      "60 loss 0.16209088265895844 [0.162090883]\n",
      "61 loss 0.14668835699558258 [0.146688357]\n",
      "62 loss 0.167514368891716 [0.167514369]\n",
      "63 loss 0.18431147933006287 [0.184311479]\n",
      "64 loss 0.13163544237613678 [0.131635442]\n",
      "65 loss 0.1258796751499176 [0.125879675]\n",
      "66 loss 0.15578091144561768 [0.155780911]\n",
      "67 loss 0.15134689211845398 [0.151346892]\n",
      "68 loss 0.1519334614276886 [0.151933461]\n",
      "69 loss 0.1626163125038147 [0.162616313]\n",
      "70 loss 0.15352067351341248 [0.153520674]\n",
      "71 loss 0.14436715841293335 [0.144367158]\n",
      "72 loss 0.12450525164604187 [0.124505252]\n",
      "73 loss 0.16470375657081604 [0.164703757]\n",
      "74 loss 0.1443338841199875 [0.144333884]\n",
      "75 loss 0.13902856409549713 [0.139028564]\n",
      "76 loss 0.14891541004180908 [0.14891541]\n",
      "77 loss 0.14201706647872925 [0.142017066]\n",
      "78 loss 0.14588849246501923 [0.145888492]\n",
      "79 loss 0.1391964703798294 [0.13919647]\n",
      "80 loss 0.15671324729919434 [0.156713247]\n",
      "81 loss 0.15471328794956207 [0.154713288]\n",
      "82 loss 0.14129550755023956 [0.141295508]\n",
      "83 loss 0.15726134181022644 [0.157261342]\n",
      "84 loss 0.14422191679477692 [0.144221917]\n",
      "85 loss 0.1457657366991043 [0.145765737]\n",
      "86 loss 0.12914445996284485 [0.12914446]\n",
      "87 loss 0.1405804604291916 [0.14058046]\n",
      "88 loss 0.131499245762825 [0.131499246]\n",
      "89 loss 0.15060628950595856 [0.15060629]\n",
      "90 loss 0.15326377749443054 [0.153263777]\n",
      "91 loss 0.13226409256458282 [0.132264093]\n",
      "92 loss 0.14781586825847626 [0.147815868]\n",
      "93 loss 0.14327074587345123 [0.143270746]\n",
      "94 loss 0.13645151257514954 [0.136451513]\n",
      "95 loss 0.14681340754032135 [0.146813408]\n",
      "96 loss 0.13004642724990845 [0.130046427]\n",
      "97 loss 0.1590152531862259 [0.159015253]\n",
      "98 loss 0.14776454865932465 [0.147764549]\n",
      "99 loss 0.1449533998966217 [0.1449534]\n",
      "100 loss 0.14445754885673523 [0.144457549]\n",
      "101 loss 0.1451951563358307 [0.145195156]\n",
      "102 loss 0.1312708556652069 [0.131270856]\n",
      "103 loss 0.12835490703582764 [0.128354907]\n",
      "104 loss 0.13228444755077362 [0.132284448]\n",
      "105 loss 0.1303313970565796 [0.130331397]\n",
      "106 loss 0.14416494965553284 [0.14416495]\n",
      "107 loss 0.13513633608818054 [0.135136336]\n",
      "108 loss 0.13112375140190125 [0.131123751]\n",
      "109 loss 0.13238996267318726 [0.132389963]\n",
      "110 loss 0.1451849639415741 [0.145184964]\n",
      "111 loss 0.11093195527791977 [0.110931955]\n",
      "112 loss 0.13780641555786133 [0.137806416]\n",
      "113 loss 0.10972420871257782 [0.109724209]\n",
      "114 loss 0.11907026916742325 [0.119070269]\n",
      "115 loss 0.14314699172973633 [0.143146992]\n",
      "116 loss 0.13809332251548767 [0.138093323]\n",
      "117 loss 0.13474023342132568 [0.134740233]\n",
      "118 loss 0.12407355010509491 [0.12407355]\n",
      "119 loss 0.13695618510246277 [0.136956185]\n",
      "120 loss 0.12356411665678024 [0.123564117]\n",
      "121 loss 0.1441318839788437 [0.144131884]\n",
      "122 loss 0.11843445897102356 [0.118434459]\n",
      "123 loss 0.12885090708732605 [0.128850907]\n",
      "124 loss 0.1272561103105545 [0.12725611]\n",
      "125 loss 0.13625489175319672 [0.136254892]\n",
      "126 loss 0.12711341679096222 [0.127113417]\n",
      "127 loss 0.12811535596847534 [0.128115356]\n",
      "128 loss 0.14106732606887817 [0.141067326]\n",
      "129 loss 0.13694946467876434 [0.136949465]\n",
      "130 loss 0.12014288455247879 [0.120142885]\n",
      "131 loss 0.14548993110656738 [0.145489931]\n",
      "132 loss 0.12350401282310486 [0.123504013]\n",
      "133 loss 0.12592528760433197 [0.125925288]\n",
      "134 loss 0.11821769922971725 [0.118217699]\n",
      "135 loss 0.11518881469964981 [0.115188815]\n",
      "136 loss 0.1071394681930542 [0.107139468]\n",
      "137 loss 0.1055530235171318 [0.105553024]\n",
      "138 loss 0.12706778943538666 [0.127067789]\n",
      "139 loss 0.11788280308246613 [0.117882803]\n",
      "140 loss 0.08883032202720642 [0.088830322]\n",
      "141 loss 0.12124995142221451 [0.121249951]\n",
      "142 loss 0.11458376049995422 [0.11458376]\n",
      "143 loss 0.12084594368934631 [0.120845944]\n",
      "144 loss 0.12640875577926636 [0.126408756]\n",
      "145 loss 0.12084290385246277 [0.120842904]\n",
      "146 loss 0.11505314707756042 [0.115053147]\n",
      "147 loss 0.11554047465324402 [0.115540475]\n",
      "148 loss 0.10410040616989136 [0.104100406]\n",
      "149 loss 0.11323952674865723 [0.113239527]\n",
      "0 loss 0.10037104785442352 [0.100371048]\n",
      "1 loss 0.12047987431287766 [0.120479874]\n",
      "2 loss 0.11502222716808319 [0.115022227]\n",
      "3 loss 0.12787604331970215 [0.127876043]\n",
      "4 loss 0.12280944734811783 [0.122809447]\n",
      "5 loss 0.13949431478977203 [0.139494315]\n",
      "6 loss 0.14546151459217072 [0.145461515]\n",
      "7 loss 0.12275388836860657 [0.122753888]\n",
      "8 loss 0.1533626914024353 [0.153362691]\n",
      "9 loss 0.1379731148481369 [0.137973115]\n",
      "10 loss 0.11349740624427795 [0.113497406]\n",
      "11 loss 0.11815593391656876 [0.118155934]\n",
      "12 loss 0.10956504940986633 [0.109565049]\n",
      "13 loss 0.10722903907299042 [0.107229039]\n",
      "14 loss 0.10710202157497406 [0.107102022]\n",
      "15 loss 0.1174822673201561 [0.117482267]\n",
      "16 loss 0.11654084175825119 [0.116540842]\n",
      "17 loss 0.11617378145456314 [0.116173781]\n",
      "18 loss 0.12699849903583527 [0.126998499]\n",
      "19 loss 0.10469955205917358 [0.104699552]\n",
      "20 loss 0.12379268556833267 [0.123792686]\n",
      "21 loss 0.08663567155599594 [0.0866356716]\n",
      "22 loss 0.11897508800029755 [0.118975088]\n",
      "23 loss 0.11795882880687714 [0.117958829]\n",
      "24 loss 0.11270839720964432 [0.112708397]\n",
      "25 loss 0.11341604590415955 [0.113416046]\n",
      "26 loss 0.1023550033569336 [0.102355]\n",
      "27 loss 0.12781061232089996 [0.127810612]\n",
      "28 loss 0.10655547678470612 [0.106555477]\n",
      "29 loss 0.11438172310590744 [0.114381723]\n",
      "30 loss 0.11958122998476028 [0.11958123]\n",
      "31 loss 0.1173774003982544 [0.1173774]\n",
      "32 loss 0.1242377981543541 [0.124237798]\n",
      "33 loss 0.09755507856607437 [0.0975550786]\n",
      "34 loss 0.09717199206352234 [0.0971719921]\n",
      "35 loss 0.11458619683980942 [0.114586197]\n",
      "36 loss 0.09410392493009567 [0.0941039249]\n",
      "37 loss 0.09736263751983643 [0.0973626375]\n",
      "38 loss 0.1126224473118782 [0.112622447]\n",
      "39 loss 0.1022014170885086 [0.102201417]\n",
      "40 loss 0.11390326917171478 [0.113903269]\n",
      "41 loss 0.0913248360157013 [0.091324836]\n",
      "42 loss 0.11051018536090851 [0.110510185]\n",
      "43 loss 0.1096741333603859 [0.109674133]\n",
      "44 loss 0.12205726653337479 [0.122057267]\n",
      "45 loss 0.09427937120199203 [0.0942793712]\n",
      "46 loss 0.09484139829874039 [0.0948414]\n",
      "47 loss 0.12388550490140915 [0.123885505]\n",
      "48 loss 0.09511931240558624 [0.0951193124]\n",
      "49 loss 0.1049078032374382 [0.104907803]\n",
      "50 loss 0.10741306841373444 [0.107413068]\n",
      "51 loss 0.1171170324087143 [0.117117032]\n",
      "52 loss 0.1143941804766655 [0.11439418]\n",
      "53 loss 0.1055932343006134 [0.105593234]\n",
      "54 loss 0.11677810549736023 [0.116778105]\n",
      "55 loss 0.10778087377548218 [0.107780874]\n",
      "56 loss 0.09377679973840714 [0.0937768]\n",
      "57 loss 0.10320742428302765 [0.103207424]\n",
      "58 loss 0.11488459259271622 [0.114884593]\n",
      "59 loss 0.09503593295812607 [0.095035933]\n",
      "60 loss 0.10593364387750626 [0.105933644]\n",
      "61 loss 0.11007166653871536 [0.110071667]\n",
      "62 loss 0.11726801842451096 [0.117268018]\n",
      "63 loss 0.09140412509441376 [0.0914041251]\n",
      "64 loss 0.09067743271589279 [0.0906774327]\n",
      "65 loss 0.09163186699151993 [0.091631867]\n",
      "66 loss 0.08108541369438171 [0.0810854137]\n",
      "67 loss 0.11288771033287048 [0.11288771]\n",
      "68 loss 0.10358276963233948 [0.10358277]\n",
      "69 loss 0.09933863580226898 [0.0993386358]\n",
      "70 loss 0.10621097683906555 [0.106210977]\n",
      "71 loss 0.08972407877445221 [0.0897240788]\n",
      "72 loss 0.09923097491264343 [0.0992309749]\n",
      "73 loss 0.1013118252158165 [0.101311825]\n",
      "74 loss 0.11703319847583771 [0.117033198]\n",
      "75 loss 0.09511125087738037 [0.0951112509]\n",
      "76 loss 0.10065941512584686 [0.100659415]\n",
      "77 loss 0.10285477340221405 [0.102854773]\n",
      "78 loss 0.10632088780403137 [0.106320888]\n",
      "79 loss 0.10270439088344574 [0.102704391]\n",
      "80 loss 0.08454764634370804 [0.0845476463]\n",
      "81 loss 0.1014741063117981 [0.101474106]\n",
      "82 loss 0.09557249397039413 [0.095572494]\n",
      "83 loss 0.0933552160859108 [0.0933552161]\n",
      "84 loss 0.10257162153720856 [0.102571622]\n",
      "85 loss 0.08381124585866928 [0.0838112459]\n",
      "86 loss 0.09986381232738495 [0.0998638123]\n",
      "87 loss 0.0998145341873169 [0.0998145342]\n",
      "88 loss 0.11698468774557114 [0.116984688]\n",
      "89 loss 0.08676842600107193 [0.086768426]\n",
      "90 loss 0.10934069752693176 [0.109340698]\n",
      "91 loss 0.08098500967025757 [0.0809850097]\n",
      "92 loss 0.09138689935207367 [0.0913869]\n",
      "93 loss 0.08014639467000961 [0.0801463947]\n",
      "94 loss 0.09411309659481049 [0.0941131]\n",
      "95 loss 0.10737626254558563 [0.107376263]\n",
      "96 loss 0.07943989336490631 [0.0794398934]\n",
      "97 loss 0.10370419919490814 [0.103704199]\n",
      "98 loss 0.08716027438640594 [0.0871602744]\n",
      "99 loss 0.09209057688713074 [0.0920905769]\n",
      "100 loss 0.08514154702425003 [0.085141547]\n",
      "101 loss 0.07073696702718735 [0.070736967]\n",
      "102 loss 0.07462058961391449 [0.0746205896]\n",
      "103 loss 0.08886582404375076 [0.088865824]\n",
      "104 loss 0.07739487290382385 [0.0773948729]\n",
      "105 loss 0.09878063201904297 [0.098780632]\n",
      "106 loss 0.071625255048275 [0.071625255]\n",
      "107 loss 0.0921967551112175 [0.0921967551]\n",
      "108 loss 0.08718911558389664 [0.0871891156]\n",
      "109 loss 0.0906623974442482 [0.0906624]\n",
      "110 loss 0.08616892248392105 [0.0861689225]\n",
      "111 loss 0.09655582904815674 [0.096555829]\n",
      "112 loss 0.0887528508901596 [0.0887528509]\n",
      "113 loss 0.08113207668066025 [0.0811320767]\n",
      "114 loss 0.09064361453056335 [0.0906436145]\n",
      "115 loss 0.08864308893680573 [0.0886430889]\n",
      "116 loss 0.10530557483434677 [0.105305575]\n",
      "117 loss 0.13727563619613647 [0.137275636]\n",
      "118 loss 0.09380371123552322 [0.0938037112]\n",
      "119 loss 0.08121351152658463 [0.0812135115]\n",
      "120 loss 0.08245779573917389 [0.0824577957]\n",
      "121 loss 0.07809720188379288 [0.0780972]\n",
      "122 loss 0.09693272411823273 [0.0969327241]\n",
      "123 loss 0.07331518828868866 [0.0733151883]\n",
      "124 loss 0.0728948786854744 [0.0728948787]\n",
      "125 loss 0.07076403498649597 [0.070764035]\n",
      "126 loss 0.09025007486343384 [0.0902500749]\n",
      "127 loss 0.10777479410171509 [0.107774794]\n",
      "128 loss 0.09199289232492447 [0.0919928923]\n",
      "129 loss 0.09026788175106049 [0.0902678818]\n",
      "130 loss 0.07040241360664368 [0.0704024136]\n",
      "131 loss 0.11092168837785721 [0.110921688]\n",
      "132 loss 0.09257319569587708 [0.0925731957]\n",
      "133 loss 0.08761824667453766 [0.0876182467]\n",
      "134 loss 0.07920701801776886 [0.079207018]\n",
      "135 loss 0.08410760760307312 [0.0841076076]\n",
      "136 loss 0.08712167292833328 [0.0871216729]\n",
      "137 loss 0.09025823324918747 [0.0902582332]\n",
      "138 loss 0.08477921783924103 [0.0847792178]\n",
      "139 loss 0.07917729765176773 [0.0791773]\n",
      "140 loss 0.08919806033372879 [0.0891980603]\n",
      "141 loss 0.0969749242067337 [0.0969749242]\n",
      "142 loss 0.0953747034072876 [0.0953747]\n",
      "143 loss 0.09261911362409592 [0.0926191136]\n",
      "144 loss 0.09961915761232376 [0.0996191576]\n",
      "145 loss 0.09010638296604156 [0.090106383]\n",
      "146 loss 0.09416650235652924 [0.0941665]\n",
      "147 loss 0.07891183346509933 [0.0789118335]\n",
      "148 loss 0.09351420402526855 [0.093514204]\n",
      "149 loss 0.08088038861751556 [0.0808803886]\n",
      "0 loss 0.07016908377408981 [0.0701690838]\n",
      "1 loss 0.09062594175338745 [0.0906259418]\n",
      "2 loss 0.09364086389541626 [0.0936408639]\n",
      "3 loss 0.0782877579331398 [0.0782877579]\n",
      "4 loss 0.07780805230140686 [0.0778080523]\n",
      "5 loss 0.08013711124658585 [0.0801371112]\n",
      "6 loss 0.08834807574748993 [0.0883480757]\n",
      "7 loss 0.10370860248804092 [0.103708602]\n",
      "8 loss 0.08974546939134598 [0.0897454694]\n",
      "9 loss 0.07988560199737549 [0.0798856]\n",
      "10 loss 0.09311271458864212 [0.0931127146]\n",
      "11 loss 0.09539783746004105 [0.0953978375]\n",
      "12 loss 0.0752435177564621 [0.0752435178]\n",
      "13 loss 0.08133121579885483 [0.0813312158]\n",
      "14 loss 0.0779246911406517 [0.0779246911]\n",
      "15 loss 0.07161282002925873 [0.07161282]\n",
      "16 loss 0.07929255813360214 [0.0792925581]\n",
      "17 loss 0.07450200617313385 [0.0745020062]\n",
      "18 loss 0.07720360159873962 [0.0772036]\n",
      "19 loss 0.08448018878698349 [0.0844801888]\n",
      "20 loss 0.07261750102043152 [0.0726175]\n",
      "21 loss 0.07144378870725632 [0.0714437887]\n",
      "22 loss 0.08407997339963913 [0.0840799734]\n",
      "23 loss 0.09189112484455109 [0.0918911248]\n",
      "24 loss 0.07908474653959274 [0.0790847465]\n",
      "25 loss 0.09842179715633392 [0.0984218]\n",
      "26 loss 0.07831943780183792 [0.0783194378]\n",
      "27 loss 0.07980059832334518 [0.0798006]\n",
      "28 loss 0.08168189972639084 [0.0816819]\n",
      "29 loss 0.07109218835830688 [0.0710921884]\n",
      "30 loss 0.07015170902013779 [0.070151709]\n",
      "31 loss 0.07139459252357483 [0.0713945925]\n",
      "32 loss 0.08399741351604462 [0.0839974135]\n",
      "33 loss 0.06813967227935791 [0.0681396723]\n",
      "34 loss 0.09349478781223297 [0.0934947878]\n",
      "35 loss 0.08308853209018707 [0.0830885321]\n",
      "36 loss 0.06716527044773102 [0.0671652704]\n",
      "37 loss 0.061838939785957336 [0.0618389398]\n",
      "38 loss 0.08611898124217987 [0.0861189812]\n",
      "39 loss 0.08919679373502731 [0.0891967937]\n",
      "40 loss 0.08815351873636246 [0.0881535187]\n",
      "41 loss 0.08693239837884903 [0.0869324]\n",
      "42 loss 0.06478288769721985 [0.0647828877]\n",
      "43 loss 0.07907222211360931 [0.0790722221]\n",
      "44 loss 0.07194527238607407 [0.0719452724]\n",
      "45 loss 0.07296997308731079 [0.0729699731]\n",
      "46 loss 0.06812232732772827 [0.0681223273]\n",
      "47 loss 0.07336709648370743 [0.0733671]\n",
      "48 loss 0.08299960196018219 [0.0829996]\n",
      "49 loss 0.07580091059207916 [0.0758009106]\n",
      "50 loss 0.07154726982116699 [0.0715472698]\n",
      "51 loss 0.06890969723463058 [0.0689097]\n",
      "52 loss 0.07662427425384521 [0.0766242743]\n",
      "53 loss 0.06249306723475456 [0.0624930672]\n",
      "54 loss 0.07453331351280212 [0.0745333135]\n",
      "55 loss 0.07205292582511902 [0.0720529258]\n",
      "56 loss 0.07590083032846451 [0.0759008303]\n",
      "57 loss 0.06581928580999374 [0.0658192858]\n",
      "58 loss 0.07392635196447372 [0.073926352]\n",
      "59 loss 0.06828681379556656 [0.0682868138]\n",
      "60 loss 0.06818016618490219 [0.0681801662]\n",
      "61 loss 0.0695853978395462 [0.0695854]\n",
      "62 loss 0.07215072214603424 [0.0721507221]\n",
      "63 loss 0.062482718378305435 [0.0624827184]\n",
      "64 loss 0.06561151146888733 [0.0656115115]\n",
      "65 loss 0.06915315985679626 [0.0691531599]\n",
      "66 loss 0.06174341216683388 [0.0617434122]\n",
      "67 loss 0.06620556861162186 [0.0662055686]\n",
      "68 loss 0.06292273104190826 [0.062922731]\n",
      "69 loss 0.06482132524251938 [0.0648213252]\n",
      "70 loss 0.08247701078653336 [0.0824770108]\n",
      "71 loss 0.07481584697961807 [0.074815847]\n",
      "72 loss 0.06751716136932373 [0.0675171614]\n",
      "73 loss 0.08951564133167267 [0.0895156413]\n",
      "74 loss 0.0635739117860794 [0.0635739118]\n",
      "75 loss 0.06734655052423477 [0.0673465505]\n",
      "76 loss 0.06877122074365616 [0.0687712207]\n",
      "77 loss 0.0557999387383461 [0.0557999387]\n",
      "78 loss 0.06013937294483185 [0.0601393729]\n",
      "79 loss 0.06782063841819763 [0.0678206384]\n",
      "80 loss 0.07013028115034103 [0.0701302812]\n",
      "81 loss 0.07355662435293198 [0.0735566244]\n",
      "82 loss 0.0811828076839447 [0.0811828077]\n",
      "83 loss 0.07333840429782867 [0.0733384043]\n",
      "84 loss 0.07319820672273636 [0.0731982067]\n",
      "85 loss 0.07943212985992432 [0.0794321299]\n",
      "86 loss 0.07328122109174728 [0.0732812211]\n",
      "87 loss 0.07585427910089493 [0.0758542791]\n",
      "88 loss 0.07029221951961517 [0.0702922195]\n",
      "89 loss 0.0572013296186924 [0.0572013296]\n",
      "90 loss 0.07560980319976807 [0.0756098]\n",
      "91 loss 0.06543432176113129 [0.0654343218]\n",
      "92 loss 0.06714344024658203 [0.0671434402]\n",
      "93 loss 0.06933483481407166 [0.0693348348]\n",
      "94 loss 0.06655371189117432 [0.0665537119]\n",
      "95 loss 0.07081378996372223 [0.07081379]\n",
      "96 loss 0.06712112575769424 [0.0671211258]\n",
      "97 loss 0.06356742233037949 [0.0635674223]\n",
      "98 loss 0.06510752439498901 [0.0651075244]\n",
      "99 loss 0.06885465979576111 [0.0688546598]\n",
      "100 loss 0.06851447373628616 [0.0685144737]\n",
      "101 loss 0.05640663206577301 [0.0564066321]\n",
      "102 loss 0.07304911315441132 [0.0730491132]\n",
      "103 loss 0.057398855686187744 [0.0573988557]\n",
      "104 loss 0.07332193106412888 [0.0733219311]\n",
      "105 loss 0.06981996446847916 [0.0698199645]\n",
      "106 loss 0.07872635871171951 [0.0787263587]\n",
      "107 loss 0.07608325034379959 [0.0760832503]\n",
      "108 loss 0.08262122422456741 [0.0826212242]\n",
      "109 loss 0.06281595677137375 [0.0628159568]\n",
      "110 loss 0.0792723223567009 [0.0792723224]\n",
      "111 loss 0.06773791462182999 [0.0677379146]\n",
      "112 loss 0.06343528628349304 [0.0634352863]\n",
      "113 loss 0.06438571959733963 [0.0643857196]\n",
      "114 loss 0.06604088842868805 [0.0660408884]\n",
      "115 loss 0.06669751554727554 [0.0666975155]\n",
      "116 loss 0.07778377830982208 [0.0777837783]\n",
      "117 loss 0.05941528081893921 [0.0594152808]\n",
      "118 loss 0.054755643010139465 [0.054755643]\n",
      "119 loss 0.06576709449291229 [0.0657670945]\n",
      "120 loss 0.06546925753355026 [0.0654692575]\n",
      "121 loss 0.08382751792669296 [0.0838275179]\n",
      "122 loss 0.07254558801651001 [0.072545588]\n",
      "123 loss 0.0567612498998642 [0.0567612499]\n",
      "124 loss 0.06984356045722961 [0.0698435605]\n",
      "125 loss 0.05342596396803856 [0.053425964]\n",
      "126 loss 0.06176735460758209 [0.0617673546]\n",
      "127 loss 0.05541181564331055 [0.0554118156]\n",
      "128 loss 0.07604217529296875 [0.0760421753]\n",
      "129 loss 0.06518933922052383 [0.0651893392]\n",
      "130 loss 0.06545975804328918 [0.065459758]\n",
      "131 loss 0.0733092725276947 [0.0733092725]\n",
      "132 loss 0.06071037799119949 [0.060710378]\n",
      "133 loss 0.05993606150150299 [0.0599360615]\n",
      "134 loss 0.06168128922581673 [0.0616812892]\n"
     ]
    }
   ],
   "source": [
    "# @tf.function\n",
    "def one_batch_step(X, y, **kwargs):\n",
    "    with tf.GradientTape() as tape:\n",
    "        y_pred = model(X)\n",
    "        loss = tf.keras.losses.sparse_categorical_crossentropy(y_true=y, y_pred=y_pred)\n",
    "        loss = tf.reduce_mean(loss)\n",
    "        tf.print(f\"{batch_index} loss {loss}\", [loss])\n",
    "        with summary_writer.as_default():\n",
    "            tf.summary.scalar(\"loss\", loss, step=batch_index)\n",
    "    grads = tape.gradient(loss, model.variables)\n",
    "    optimizer.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "    \n",
    "for epoch_index in range(num_epochs):\n",
    "    for batch_index in range(num_batches):\n",
    "        X, y = data_loader.get_batch(batch_size)\n",
    "        one_batch_step(X, y, batch_index=batch_index)\n",
    "with summary_writer.as_default():\n",
    "    tf.summary.trace_export(name=\"model_trace\", step=0, profiler_outdir=log_dir)\n",
    "    \n",
    "tf.saved_model.save(model, f\"saved/{model_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.saved_model.load(f\"saved/{model_name}\")\n",
    "\n",
    "def test_model(model):\n",
    "    sparse_categorical_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n",
    "    num_batches = int(data_loader.num_test_data // batch_size)\n",
    "\n",
    "    for batch_index in range(num_batches):\n",
    "        start_index, end_index = batch_index * batch_size, (batch_index + 1) * batch_size\n",
    "        y_pred = model(data_loader.test_data[start_index: end_index])\n",
    "        sparse_categorical_accuracy.update_state(y_true=data_loader.test_label[start_index: end_index], y_pred=y_pred)\n",
    "    print(\"test accuracy: %f\" % sparse_categorical_accuracy.result())\n",
    "    \n",
    "test_model(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 方法二"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras Pipleline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Flatten(),\n",
    "    tf.keras.layers.Dense(100, activation=tf.nn.relu),\n",
    "    tf.keras.layers.Dense(units=20, activation=tf.nn.leaky_relu),\n",
    "    tf.keras.layers.Dense(10),\n",
    "    tf.keras.layers.Softmax()\n",
    "])\n",
    "\n",
    "\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
    "    loss=tf.keras.losses.sparse_categorical_crossentropy,\n",
    "    metrics=[tf.keras.metrics.sparse_categorical_accuracy]\n",
    ")\n",
    "\n",
    "model.fit(data_loader.train_data, data_loader.train_label, epochs=num_epochs, batch_size=batch_size)\n",
    "\n",
    "print(model.evaluate(data_loader.test_data, data_loader.test_label))\n",
    "\n",
    "model_name = \"mnist_model2\"\n",
    "tf.saved_model.save(model, f\"saved/{model_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.saved_model.load(f\"saved/{model_name}\")\n",
    "test_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf2",
   "language": "python",
   "name": "tf2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}