paddle/pretrained.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import paddle.fluid as fluid\n",
    "import paddle\n",
    "import numpy as np\n",
    "import os\n",
    "import math\n",
    "from paddle.fluid.debugger import draw_block_graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预训练模型保存路径\n",
    "# 来自：https://github.com/PaddlePaddle/models/tree/develop/fluid/image_classification#supported-models-and-performances\n",
    "pretrained_model_path = \"se_resnext_50/129\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 根据program加载参数\n",
    "def load_pretrained_params(exe, program):\n",
    "\n",
    "    # 只加载实际存在的\n",
    "    def if_exist(var):\n",
    "        path = os.path.join(pretrained_model_path, var.name)\n",
    "        exist = os.path.exists(path)\n",
    "        if exist:\n",
    "                print(\"Load\", path)\n",
    "        return exist\n",
    "    \n",
    "    fluid.io.load_vars(exe, pretrained_model_path, predicate=if_exist, main_program=program)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预训练模型定义，已去掉不需要的层\n",
    "# 来自：\n",
    "class SE_ResNeXt50():\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.variables = []\n",
    "    \n",
    "    def net(self, input, class_dim=1000):\n",
    "        cardinality = 32\n",
    "        reduction_ratio = 16\n",
    "        depth = [3, 4, 6] #, 3]\n",
    "        num_filters = [128, 256, 512, 1024]\n",
    "        \n",
    "        conv = self.conv_bn_layer(\n",
    "            input=input,\n",
    "            num_filters=64,\n",
    "            filter_size=7,\n",
    "            stride=2,\n",
    "            act='relu')\n",
    "        conv = fluid.layers.pool2d(\n",
    "            input=conv,\n",
    "            pool_size=3,\n",
    "            pool_stride=2,\n",
    "            pool_padding=1,\n",
    "            pool_type='max')\n",
    "        \n",
    "        for block in range(len(depth)):\n",
    "            for i in range(depth[block]):\n",
    "                conv = self.bottleneck_block(\n",
    "                    input=conv,\n",
    "                    num_filters=num_filters[block],\n",
    "                    stride=2 if i == 0 and block != 0 else 1,\n",
    "                    cardinality=cardinality,\n",
    "                    reduction_ratio=reduction_ratio)\n",
    "        pool = fluid.layers.pool2d( input=conv, pool_size=7, pool_type='avg', global_pooling=True)            \n",
    "        return pool\n",
    "\n",
    "    def shortcut(self, input, ch_out, stride):\n",
    "        ch_in = input.shape[1]\n",
    "        if ch_in != ch_out or stride != 1:\n",
    "            filter_size = 1\n",
    "            return self.conv_bn_layer(input, ch_out, filter_size, stride)\n",
    "        else:\n",
    "            return input\n",
    "\n",
    "    def bottleneck_block(self, input, num_filters, stride, cardinality,\n",
    "                         reduction_ratio):\n",
    "        conv0 = self.conv_bn_layer(\n",
    "            input=input, num_filters=num_filters, filter_size=1, act='relu')\n",
    "        conv1 = self.conv_bn_layer(\n",
    "            input=conv0,\n",
    "            num_filters=num_filters,\n",
    "            filter_size=3,\n",
    "            stride=stride,\n",
    "            groups=cardinality,\n",
    "            act='relu')\n",
    "        conv2 = self.conv_bn_layer(\n",
    "            input=conv1, num_filters=num_filters * 2, filter_size=1, act=None)\n",
    "        scale = self.squeeze_excitation(\n",
    "            input=conv2,\n",
    "            num_channels=num_filters * 2,\n",
    "            reduction_ratio=reduction_ratio)\n",
    "\n",
    "        short = self.shortcut(input, num_filters * 2, stride)\n",
    "\n",
    "        return fluid.layers.elementwise_add(x=short, y=scale, act='relu')\n",
    "\n",
    "    def conv_bn_layer(self,\n",
    "                      input,\n",
    "                      num_filters,\n",
    "                      filter_size,\n",
    "                      stride=1,\n",
    "                      groups=1,\n",
    "                      act=None):\n",
    "        conv = fluid.layers.conv2d(\n",
    "            input=input,\n",
    "            num_filters=num_filters,\n",
    "            filter_size=filter_size,\n",
    "            stride=stride,\n",
    "            padding=(filter_size - 1) / 2,\n",
    "            groups=groups,\n",
    "            act=None,\n",
    "            bias_attr=False)\n",
    "        self.variables.append(conv)\n",
    "        bn = fluid.layers.batch_norm(input=conv, act=act)\n",
    "        self.variables.append(bn)\n",
    "        return bn\n",
    "\n",
    "    def squeeze_excitation(self, input, num_channels, reduction_ratio):\n",
    "        pool = fluid.layers.pool2d(\n",
    "            input=input, pool_size=0, pool_type='avg', global_pooling=True)\n",
    "        stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0)\n",
    "        squeeze = fluid.layers.fc(input=pool,\n",
    "                                  size=num_channels / reduction_ratio,\n",
    "                                  act='relu',\n",
    "                                  param_attr=fluid.param_attr.ParamAttr(\n",
    "                                      initializer=fluid.initializer.Uniform(\n",
    "                                          -stdv, stdv)))\n",
    "        self.variables.append(squeeze)\n",
    "        stdv = 1.0 / math.sqrt(squeeze.shape[1] * 1.0)\n",
    "        excitation = fluid.layers.fc(input=squeeze,\n",
    "                                     size=num_channels,\n",
    "                                     act='sigmoid',\n",
    "                                     param_attr=fluid.param_attr.ParamAttr(\n",
    "                                         initializer=fluid.initializer.Uniform(\n",
    "                                             -stdv, stdv)))\n",
    "        self.variables.append(excitation)\n",
    "        scale = fluid.layers.elementwise_mul(x=input, y=excitation, axis=0)\n",
    "        return scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 自定义模型\n",
    "def network(image, train_base_model=False):\n",
    "  \n",
    "    # 预训练模型\n",
    "    base_model = SE_ResNeXt50().net(image)\n",
    "    # 控制是否训练base_model\n",
    "    base_model.stop_gradient = not train_base_model \n",
    "    \n",
    "    # 复制一个只包含base_model的program，放便只加载需要的参数\n",
    "    base_model_program = fluid.default_main_program().clone() \n",
    "    \n",
    "    # 添加新的层\n",
    "    y = base_model\n",
    "    y = fluid.layers.fc(base_model, size=10, act='softmax')\n",
    "    \n",
    "    return y, base_model_program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = fluid.core.is_compiled_with_cuda()\n",
    "place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Load', 'se_resnext_50/129/batch_norm_10.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_33.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_33.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_33.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_17.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_32.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_31.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_28.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_25.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_31.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_31.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_26.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_31.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_31.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_17.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_16.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_16.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_8.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_42.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_30.w_1')\n",
      "('Load', 'se_resnext_50/129/conv2d_1.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_30.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_35.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_36.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_29.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_24.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_29.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_29.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_24.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_28.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_27.w_1')\n",
      "('Load', 'se_resnext_50/129/fc_14.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_27.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_5.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_27.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_19.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_37.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_27.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_15.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_12.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_34.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_18.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_26.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_1.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_25.w_1')\n",
      "('Load', 'se_resnext_50/129/conv2d_17.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_38.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_15.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_26.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_24.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_28.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_13.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_36.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_32.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_12.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_23.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_23.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_23.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_24.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_21.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_17.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_22.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_10.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_30.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_37.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_22.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_9.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_39.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_1.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_20.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_42.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_20.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_2.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_6.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_19.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_5.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_8.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_9.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_8.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_11.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_5.w_2')\n",
      "('Load', 'se_resnext_50/129/conv2d_10.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_37.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_10.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_34.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_15.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_8.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_18.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_5.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_9.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_37.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_38.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_0.w_1')\n",
      "('Load', 'se_resnext_50/129/fc_1.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_2.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_35.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_15.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_24.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_12.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_16.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_32.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_34.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_12.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_2.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_18.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_6.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_6.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_12.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_19.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_13.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_38.w_1')\n",
      "('Load', 'se_resnext_50/129/fc_7.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_26.w_2')\n",
      "('Load', 'se_resnext_50/129/conv2d_28.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_40.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_41.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_13.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_1.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_32.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_10.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_38.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_18.w_1')\n",
      "('Load', 'se_resnext_50/129/conv2d_11.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_0.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_30.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_13.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_8.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_23.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_7.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_12.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_11.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_15.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_10.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_42.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_42.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_39.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_10.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_39.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_20.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_14.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_25.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_20.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_3.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_21.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_4.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_33.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_1.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_25.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_21.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_38.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_40.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_29.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_41.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_4.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_40.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_40.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_35.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_39.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_22.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_41.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_0.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_36.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_0.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_23.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_40.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_13.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_8.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_36.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_5.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_39.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_6.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_13.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_35.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_6.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_4.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_6.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_20.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_14.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_23.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_12.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_32.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_20.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_17.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_15.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_16.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_1.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_0.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_16.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_16.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_7.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_15.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_11.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_5.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_7.w_2')\n",
      "('Load', 'se_resnext_50/129/conv2d_42.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_3.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_2.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_34.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_21.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_13.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_37.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_16.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_20.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_3.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_33.w_1')\n",
      "('Load', 'se_resnext_50/129/fc_11.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_14.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_25.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_9.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_11.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_4.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_3.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_7.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_4.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_5.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_4.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_17.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_21.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_3.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_8.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_14.w_2')\n",
      "('Load', 'se_resnext_50/129/conv2d_3.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_22.w_0')\n",
      "('Load', 'se_resnext_50/129/fc_9.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_9.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_0.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_6.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_7.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_2.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_3.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_7.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_29.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_34.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_36.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_14.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_25.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_0.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_2.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_35.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_17.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_2.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_25.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_23.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_24.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_27.w_2')\n",
      "('Load', 'se_resnext_50/129/fc_9.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_30.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_18.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_18.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_18.w_2')\n",
      "('Load', 'se_resnext_50/129/batch_norm_21.w_2')\n",
      "('Load', 'se_resnext_50/129/conv2d_19.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_28.w_1')\n",
      "('Load', 'se_resnext_50/129/fc_19.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_19.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_19.w_1')\n",
      "('Load', 'se_resnext_50/129/fc_22.b_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_4.w_0')\n",
      "('Load', 'se_resnext_50/129/conv2d_14.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_26.b_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_41.b_0')\n",
      "('Load', 'se_resnext_50/129/fc_11.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_21.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_24.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_22.w_0')\n",
      "('Load', 'se_resnext_50/129/batch_norm_41.w_1')\n",
      "('Load', 'se_resnext_50/129/batch_norm_22.b_0')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Batch 0 Loss 2.311845 Acc 0.054688\n",
      "Epoch 0 Batch 100 Loss 2.132499 Acc 0.453125\n",
      "Epoch 0 Batch 200 Loss 1.953558 Acc 0.539062\n",
      "Epoch 0 Batch 300 Loss 1.840348 Acc 0.546875\n",
      "Epoch 1 Batch 0 Loss 1.777899 Acc 0.515625\n",
      "Epoch 1 Batch 100 Loss 1.705209 Acc 0.609375\n",
      "Epoch 1 Batch 200 Loss 1.572993 Acc 0.570312\n",
      "Epoch 1 Batch 300 Loss 1.525481 Acc 0.578125\n",
      "Epoch 2 Batch 0 Loss 1.530093 Acc 0.539062\n",
      "Epoch 2 Batch 100 Loss 1.480704 Acc 0.632812\n",
      "Epoch 2 Batch 200 Loss 1.379631 Acc 0.593750\n",
      "Epoch 2 Batch 300 Loss 1.353684 Acc 0.593750\n",
      "Epoch 3 Batch 0 Loss 1.393383 Acc 0.546875\n",
      "Epoch 3 Batch 100 Loss 1.345412 Acc 0.625000\n",
      "Epoch 3 Batch 200 Loss 1.266868 Acc 0.593750\n",
      "Epoch 3 Batch 300 Loss 1.247394 Acc 0.625000\n",
      "Epoch 4 Batch 0 Loss 1.308322 Acc 0.570312\n",
      "Epoch 4 Batch 100 Loss 1.255022 Acc 0.640625\n",
      "Epoch 4 Batch 200 Loss 1.193787 Acc 0.601562\n",
      "Epoch 4 Batch 300 Loss 1.174929 Acc 0.640625\n",
      "Epoch 5 Batch 0 Loss 1.250353 Acc 0.593750\n",
      "Epoch 5 Batch 100 Loss 1.189938 Acc 0.656250\n",
      "Epoch 5 Batch 200 Loss 1.142630 Acc 0.609375\n",
      "Epoch 5 Batch 300 Loss 1.121977 Acc 0.648438\n",
      "Epoch 6 Batch 0 Loss 1.208088 Acc 0.593750\n",
      "Epoch 6 Batch 100 Loss 1.140468 Acc 0.656250\n",
      "Epoch 6 Batch 200 Loss 1.104744 Acc 0.632812\n",
      "Epoch 6 Batch 300 Loss 1.081330 Acc 0.656250\n",
      "Epoch 7 Batch 0 Loss 1.175674 Acc 0.601562\n",
      "Epoch 7 Batch 100 Loss 1.101330 Acc 0.664062\n",
      "Epoch 7 Batch 200 Loss 1.075476 Acc 0.648438\n",
      "Epoch 7 Batch 300 Loss 1.048988 Acc 0.671875\n",
      "Epoch 8 Batch 0 Loss 1.149839 Acc 0.609375\n",
      "Epoch 8 Batch 100 Loss 1.069414 Acc 0.656250\n",
      "Epoch 8 Batch 200 Loss 1.052117 Acc 0.656250\n",
      "Epoch 8 Batch 300 Loss 1.022541 Acc 0.679688\n",
      "Epoch 9 Batch 0 Loss 1.128621 Acc 0.625000\n",
      "Epoch 9 Batch 100 Loss 1.042766 Acc 0.664062\n",
      "Epoch 9 Batch 200 Loss 1.032989 Acc 0.664062\n",
      "Epoch 9 Batch 300 Loss 1.000447 Acc 0.679688\n"
     ]
    }
   ],
   "source": [
    "# 训练新加的层\n",
    "main = fluid.Program()\n",
    "startup = fluid.Program()\n",
    "exe = fluid.Executor(place)\n",
    "\n",
    "with fluid.unique_name.guard():\n",
    "    with fluid.program_guard(main, startup):\n",
    "\n",
    "            # 预测\n",
    "            image = fluid.layers.data('image', [3, 32, 32], dtype='float32')\n",
    "            predict, base_model_program = network(image, False) # 只训练新加的层\n",
    "           \n",
    "            # 损失\n",
    "            label = fluid.layers.data('label', [1], dtype='int64')\n",
    "            loss = fluid.layers.cross_entropy(label=label, input=predict)\n",
    "            loss = fluid.layers.mean(loss)\n",
    "            \n",
    "            # Metric\n",
    "            acc = fluid.layers.accuracy(predict, label)\n",
    "            \n",
    "            # Feeder\n",
    "            feeder = fluid.data_feeder.DataFeeder([image, label], place)\n",
    "            reader = feeder.decorate_reader(paddle.batch(paddle.dataset.cifar.train10(), 128), None)\n",
    "\n",
    "            # opt\n",
    "            optimizer = fluid.optimizer.Adam(learning_rate=0.001)\n",
    "            optimizer.minimize(loss)\n",
    "\n",
    "            # 初始化变量\n",
    "            exe.run(startup)\n",
    "\n",
    "            # 加载参数\n",
    "            load_pretrained_params(exe, base_model_program)\n",
    "\n",
    "            optimized_main = fluid.transpiler.memory_optimize(main, print_log=False)\n",
    "\n",
    "           \n",
    "            for epoch in range(10):\n",
    "                for batch, data in enumerate(reader()):\n",
    "                    result = exe.run(fetch_list=[loss, acc], feed=data)\n",
    "                    if batch % 100 == 0:\n",
    "                        print(\"Epoch %d Batch %d Loss %f Acc %f\" % (epoch, batch, result[0][0], result[1][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存全部网络参数\n",
    "fluid.io.save_params(exe, \"cifar10.model\", main_program=main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Batch 0 Loss 1.110775 Acc 0.640625\n",
      "Epoch 0 Batch 100 Loss 0.675765 Acc 0.781250\n",
      "Epoch 0 Batch 200 Loss 0.561026 Acc 0.804688\n",
      "Epoch 0 Batch 300 Loss 0.399023 Acc 0.843750\n",
      "Epoch 0 Test Acc 0.838241\n",
      "Epoch 1 Batch 0 Loss 0.339029 Acc 0.875000\n",
      "Epoch 1 Batch 100 Loss 0.324468 Acc 0.875000\n",
      "Epoch 1 Batch 200 Loss 0.180062 Acc 0.914062\n",
      "Epoch 1 Batch 300 Loss 0.110104 Acc 0.968750\n",
      "Epoch 1 Test Acc 0.843349\n",
      "Epoch 2 Batch 0 Loss 0.078147 Acc 0.992188\n",
      "Epoch 2 Batch 100 Loss 0.078254 Acc 0.976562\n",
      "Epoch 2 Batch 200 Loss 0.013229 Acc 1.000000\n",
      "Epoch 2 Batch 300 Loss 0.021719 Acc 0.992188\n",
      "Epoch 2 Test Acc 0.846955\n",
      "Epoch 3 Batch 0 Loss 0.007396 Acc 1.000000\n",
      "Epoch 3 Batch 100 Loss 0.008246 Acc 1.000000\n",
      "Epoch 3 Batch 200 Loss 0.012331 Acc 1.000000\n",
      "Epoch 3 Batch 300 Loss 0.010200 Acc 1.000000\n",
      "Epoch 3 Test Acc 0.844050\n",
      "Epoch 4 Batch 0 Loss 0.034095 Acc 0.976562\n",
      "Epoch 4 Batch 100 Loss 0.043675 Acc 0.984375\n",
      "Epoch 4 Batch 200 Loss 0.059676 Acc 0.984375\n",
      "Epoch 4 Batch 300 Loss 0.028524 Acc 1.000000\n",
      "Epoch 4 Test Acc 0.842849\n",
      "Epoch 5 Batch 0 Loss 0.055586 Acc 0.968750\n",
      "Epoch 5 Batch 100 Loss 0.057326 Acc 0.976562\n",
      "Epoch 5 Batch 200 Loss 0.037495 Acc 0.984375\n",
      "Epoch 5 Batch 300 Loss 0.025395 Acc 0.992188\n"
     ]
    }
   ],
   "source": [
    "# 训练整个网络\n",
    "main = fluid.Program()\n",
    "startup = fluid.Program()\n",
    "exe = fluid.Executor(place)\n",
    "\n",
    "with fluid.unique_name.guard():\n",
    "    with fluid.program_guard(main, startup):\n",
    "\n",
    "            # 预测\n",
    "            image = fluid.layers.data('image', [3, 32, 32], dtype='float32')\n",
    "            predict, base_model_program = network(image, True)\n",
    "            \n",
    "            # 损失\n",
    "            label = fluid.layers.data('label', [1], dtype='int64')\n",
    "            loss = fluid.layers.cross_entropy(label=label, input=predict)\n",
    "            loss = fluid.layers.mean(loss)\n",
    "            \n",
    "            # Metric\n",
    "            acc = fluid.layers.accuracy(predict, label)\n",
    "            \n",
    "            test_program = main.clone(for_test=True)\n",
    "            \n",
    "            # Feeder\n",
    "            feeder = fluid.data_feeder.DataFeeder([image, label], place)\n",
    "            reader = feeder.decorate_reader(paddle.batch(paddle.dataset.cifar.train10(), 128), None)\n",
    "            \n",
    "            test_feeder = fluid.data_feeder.DataFeeder([image, label], place)\n",
    "            test_reader = test_feeder.decorate_reader(paddle.batch(paddle.dataset.cifar.test10(), 128), None)  \n",
    "\n",
    "            # opt\n",
    "            optimizer = fluid.optimizer.Adam(learning_rate=0.0001)\n",
    "            optimizer.minimize(loss)\n",
    "\n",
    "            # 初始化变量\n",
    "            exe.run(startup)\n",
    "\n",
    "            # 加载参数\n",
    "            fluid.io.load_params(exe, \"cifar10.model\", main_program=main)\n",
    "\n",
    "            optimized_main = fluid.transpiler.memory_optimize(main, print_log=False)\n",
    "\n",
    "           \n",
    "            for epoch in range(10):\n",
    "                # 训练\n",
    "                for batch, data in enumerate(reader()):\n",
    "                    result = exe.run(fetch_list=[loss, acc], feed=data)\n",
    "                    if batch % 100 == 0:\n",
    "                        print(\"Epoch %d Batch %d Loss %f Acc %f\" % (epoch, batch, result[0][0], result[1][0]))\n",
    "                fluid.io.save_params(exe, \"cifar10.model\", main_program=main)\n",
    "                \n",
    "                # 测试\n",
    "                test_acc = []\n",
    "                for test_data in test_reader():\n",
    "                    result = exe.run(test_program, fetch_list=[acc], feed=test_data)\n",
    "                    test_acc.append(result[0][0])\n",
    "                \n",
    "                print(\"Epoch %d Test Acc %f\" % (epoch, np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}