DeepFM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import *\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import tensorflow.keras.backend as K \n",
    "from tensorflow.keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('./criteo_sampled_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>...</th>\n",
       "      <th>C17</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1382.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>...</td>\n",
       "      <td>e5ba7672</td>\n",
       "      <td>f54016b9</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>b1252a9d</td>\n",
       "      <td>07b5194c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>c5c50484</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>9727dd16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>07c540c4</td>\n",
       "      <td>b04e4670</td>\n",
       "      <td>21ddcdc9</td>\n",
       "      <td>5840adea</td>\n",
       "      <td>60f6221e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>43f13e8b</td>\n",
       "      <td>e8b83407</td>\n",
       "      <td>731c3655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>767.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>245.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8efede7f</td>\n",
       "      <td>3412118d</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>e587c466</td>\n",
       "      <td>ad3062eb</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>3b183c5c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>893</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4392.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>74ef3502</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6b3a5ca6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a171ecb</td>\n",
       "      <td>9117a34a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1e88c74f</td>\n",
       "      <td>26b3c7a7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21c9516a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32c7478e</td>\n",
       "      <td>b34f3128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label   I1   I2    I3    I4      I5    I6    I7   I8     I9  ...       C17  \\\n",
       "0      0  1.0    1   5.0   0.0  1382.0   4.0  15.0  2.0  181.0  ...  e5ba7672   \n",
       "1      0  2.0    0  44.0   1.0   102.0   8.0   2.0  2.0    4.0  ...  07c540c4   \n",
       "2      0  2.0    0   1.0  14.0   767.0  89.0   4.0  2.0  245.0  ...  8efede7f   \n",
       "3      0  NaN  893   NaN   NaN  4392.0   NaN   0.0  0.0    0.0  ...  1e88c74f   \n",
       "4      0  3.0   -1   NaN   0.0     2.0   0.0   3.0  0.0    0.0  ...  1e88c74f   \n",
       "\n",
       "        C18       C19       C20       C21       C22       C23       C24  \\\n",
       "0  f54016b9  21ddcdc9  b1252a9d  07b5194c       NaN  3a171ecb  c5c50484   \n",
       "1  b04e4670  21ddcdc9  5840adea  60f6221e       NaN  3a171ecb  43f13e8b   \n",
       "2  3412118d       NaN       NaN  e587c466  ad3062eb  3a171ecb  3b183c5c   \n",
       "3  74ef3502       NaN       NaN  6b3a5ca6       NaN  3a171ecb  9117a34a   \n",
       "4  26b3c7a7       NaN       NaN  21c9516a       NaN  32c7478e  b34f3128   \n",
       "\n",
       "        C25       C26  \n",
       "0  e8b83407  9727dd16  \n",
       "1  e8b83407  731c3655  \n",
       "2       NaN       NaN  \n",
       "3       NaN       NaN  \n",
       "4       NaN       NaN  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "600000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = train.columns.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义特征组\n",
    "dense_feats = [f for f in cols if f[0] == 'I']\n",
    "sparse_feats = [f for f in cols if f[0] == 'C']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dense_feats(data,feats):\n",
    "    d  = data.copy()\n",
    "    d = d[feats].fillna(0.0)\n",
    "    for f in feats:\n",
    "        d[f] = d[f].apply(lambda x: np.log(x+1) if x>-1 else -1)\n",
    "    return d\n",
    "data_dense = process_dense_feats(train, dense_feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_spares_feats(data,feats):\n",
    "    d = data.copy()\n",
    "    d = d[feats].fillna('-1')\n",
    "    for f in feats:\n",
    "        d[f] = LabelEncoder().fit_transform(d[f])\n",
    "    return d\n",
    "data_sparse = process_spares_feats(train,sparse_feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_data = pd.concat([data_dense,data_sparse],axis=1)\n",
    "total_data['label'] = train['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>I1</th>\n",
       "      <th>I2</th>\n",
       "      <th>I3</th>\n",
       "      <th>I4</th>\n",
       "      <th>I5</th>\n",
       "      <th>I6</th>\n",
       "      <th>I7</th>\n",
       "      <th>I8</th>\n",
       "      <th>I9</th>\n",
       "      <th>I10</th>\n",
       "      <th>...</th>\n",
       "      <th>C18</th>\n",
       "      <th>C19</th>\n",
       "      <th>C20</th>\n",
       "      <th>C21</th>\n",
       "      <th>C22</th>\n",
       "      <th>C23</th>\n",
       "      <th>C24</th>\n",
       "      <th>C25</th>\n",
       "      <th>C26</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.693147</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>1.791759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.232010</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>2.772589</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>5.204007</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>...</td>\n",
       "      <td>3439</td>\n",
       "      <td>213</td>\n",
       "      <td>3</td>\n",
       "      <td>4954</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>24768</td>\n",
       "      <td>52</td>\n",
       "      <td>14364</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.098612</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.806662</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>4.634729</td>\n",
       "      <td>2.197225</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>...</td>\n",
       "      <td>2465</td>\n",
       "      <td>213</td>\n",
       "      <td>1</td>\n",
       "      <td>60664</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>8432</td>\n",
       "      <td>52</td>\n",
       "      <td>10835</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.098612</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>2.708050</td>\n",
       "      <td>6.643790</td>\n",
       "      <td>4.499810</td>\n",
       "      <td>1.609438</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>5.505332</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>...</td>\n",
       "      <td>738</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>143786</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>7344</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.795706</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.387768</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1648</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>67107</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>18107</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.386294</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.098612</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.386294</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>...</td>\n",
       "      <td>556</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21257</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>22439</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 40 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         I1        I2        I3        I4        I5        I6        I7  \\\n",
       "0  0.693147  0.693147  1.791759  0.000000  7.232010  1.609438  2.772589   \n",
       "1  1.098612  0.000000  3.806662  0.693147  4.634729  2.197225  1.098612   \n",
       "2  1.098612  0.000000  0.693147  2.708050  6.643790  4.499810  1.609438   \n",
       "3  0.000000  6.795706  0.000000  0.000000  8.387768  0.000000  0.000000   \n",
       "4  1.386294 -1.000000  0.000000  0.000000  1.098612  0.000000  1.386294   \n",
       "\n",
       "         I8        I9       I10  ...   C18  C19  C20     C21  C22  C23    C24  \\\n",
       "0  1.098612  5.204007  0.693147  ...  3439  213    3    4954    0    3  24768   \n",
       "1  1.098612  1.609438  0.693147  ...  2465  213    1   60664    0    3   8432   \n",
       "2  1.098612  5.505332  0.693147  ...   738    0    0  143786    9    3   7344   \n",
       "3  0.000000  0.000000  0.000000  ...  1648    0    0   67107    0    3  18107   \n",
       "4  0.000000  0.000000  0.693147  ...   556    0    0   21257    0    2  22439   \n",
       "\n",
       "   C25    C26  label  \n",
       "0   52  14364      0  \n",
       "1   52  10835      0  \n",
       "2    0      0      0  \n",
       "3    0      0      0  \n",
       "4    0      0      0  \n",
       "\n",
       "[5 rows x 40 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型的构建与训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一阶特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dense特征  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_inputs = []\n",
    "for f in dense_feats:\n",
    "    _input = Input([1],name=f)\n",
    "    dense_inputs.append(_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'I1:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I2:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I3:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I4:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I5:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I6:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I7:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I8:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I9:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I10:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I11:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I12:0' shape=(None, 1) dtype=float32>,\n",
       " <tf.Tensor 'I13:0' shape=(None, 1) dtype=float32>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_dense_inputs = Concatenate(axis=1)(dense_inputs)  # (?,13)\n",
    "fst_order_dense_layer = Dense(1)(concat_dense_inputs) # (?,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sparse 特征 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_inputs = []\n",
    "for f in sparse_feats:\n",
    "    _input = Input([1],name=f)\n",
    "    sparse_inputs.append(_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_ld_embed = []\n",
    "for i,_input in enumerate(sparse_inputs):\n",
    "    f = sparse_feats[i]\n",
    "    voc_size  = train[f].nunique()\n",
    "    reg = tf.keras.regularizers.l2(0.5)\n",
    "#     _embed = Flatten()(Embedding(voc_size+1,1,embeddings_regularizer = tf.keras.regularizers.l2(0.5))(_input))\n",
    "    _embed = Embedding(voc_size+1, 1, embeddings_regularizer=reg)(_input)\n",
    "    # 由于 Embedding 的结果是二维的，\n",
    "    # 因此如果需要在 Embedding 之后加入 Dense 层，则需要先连接上 Flatten 层\n",
    "    _embed = Flatten()(_embed)\n",
    "    sparse_ld_embed.append(_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "fst_order_sparse_layer = Add()(sparse_ld_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add_3/Identity:0' shape=(None, 1) dtype=float32>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fst_order_sparse_layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear 部分合并\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_part = Add()([fst_order_dense_layer, fst_order_sparse_layer])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二阶特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "k = 8  # embeding size \n",
    "# 只考虑sparse的二阶交叉\n",
    "sparse_kd_embed = []\n",
    "for i,_input in enumerate(sparse_inputs):\n",
    "    f = sparse_feats[i]\n",
    "    voc_size = train[f].nunique()\n",
    "    _embed = Embedding(voc_size+1, k, embeddings_regularizer=tf.keras.regularizers.l2(0.7))(_input)\n",
    "    sparse_kd_embed.append(_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor 'embedding_80/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_81/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_82/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_83/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_84/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_85/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_86/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_87/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_88/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_89/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_90/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_91/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_92/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_93/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_94/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_95/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_96/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_97/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_98/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_99/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_100/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_101/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_102/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_103/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_104/Identity:0' shape=(None, 1, 8) dtype=float32>,\n",
       " <tf.Tensor 'embedding_105/Identity:0' shape=(None, 1, 8) dtype=float32>]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_kd_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.将所有sparse的embedding拼接起来，得到 (n, k)的矩阵，其中n为特征数，k为embedding大小\n",
    "\n",
    "concat_sparse_kd_embed = Concatenate(axis=1)(sparse_kd_embed) # ?, n, k（None,26,8）\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.先求和再平方\n",
    "sum_kd_embed = Lambda(lambda x: K.sum(x,axis=1))(concat_sparse_kd_embed)\n",
    "square_sum_kd_embed = Multiply()([sum_kd_embed, sum_kd_embed])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'multiply_2/Identity:0' shape=(None, 8) dtype=float32>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "square_sum_kd_embed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  3.先平方再求和\n",
    "square_kd_embed = Multiply()([concat_sparse_kd_embed, concat_sparse_kd_embed])\n",
    "sum_square_kd_embed = Lambda(lambda x: K.sum(x,axis=1))(square_kd_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4.相减除以2\n",
    "sub = Subtract()([square_sum_kd_embed,sum_square_kd_embed])\n",
    "sub = Lambda(lambda x: x*0.5)(sub)\n",
    "snd_order_sparse_layer = Lambda(lambda x: K.sum(x, axis=1,keepdims=True))(sub)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DNN \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "flatten_sparse_embed = Flatten()(concat_sparse_kd_embed)   # (None,n*k)\n",
    "fc_layer = Dropout(0.5)(Dense(256,activation='relu')(flatten_sparse_embed))  #(?,256)\n",
    "fc_layer = Dropout(0.3)(Dense(256,activation='relu')(fc_layer))\n",
    "fc_layer = Dropout(0.1)(Dense(256,activation='relu')(fc_layer))\n",
    "fc_layer_output = Dense(1)(fc_layer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 输出结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'activation_1/Identity:0' shape=(None, 1) dtype=float32>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_layer = Add()([linear_part, snd_order_sparse_layer, fc_layer_output])\n",
    "output_layer = Activation('sigmoid')(output_layer)\n",
    "output_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(dense_inputs + sparse_inputs, output_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "C1 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C2 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C3 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C4 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C5 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C6 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C7 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C8 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C9 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C10 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C11 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C12 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C13 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C14 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C15 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C16 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C17 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C18 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C19 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C20 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C21 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C22 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C23 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C24 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C25 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "C26 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_80 (Embedding)        (None, 1, 8)         8728        C1[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_81 (Embedding)        (None, 1, 8)         4224        C2[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_82 (Embedding)        (None, 1, 8)         1660632     C3[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_83 (Embedding)        (None, 1, 8)         687288      C4[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_84 (Embedding)        (None, 1, 8)         1976        C5[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_85 (Embedding)        (None, 1, 8)         112         C6[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_86 (Embedding)        (None, 1, 8)         82872       C7[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_87 (Embedding)        (None, 1, 8)         4104        C8[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_88 (Embedding)        (None, 1, 8)         32          C9[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_89 (Embedding)        (None, 1, 8)         201440      C10[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_90 (Embedding)        (None, 1, 8)         36456       C11[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_91 (Embedding)        (None, 1, 8)         1417232     C12[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_92 (Embedding)        (None, 1, 8)         24336       C13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_93 (Embedding)        (None, 1, 8)         216         C14[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_94 (Embedding)        (None, 1, 8)         65088       C15[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_95 (Embedding)        (None, 1, 8)         1112024     C16[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_96 (Embedding)        (None, 1, 8)         88          C17[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_97 (Embedding)        (None, 1, 8)         28920       C18[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_98 (Embedding)        (None, 1, 8)         13992       C19[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_99 (Embedding)        (None, 1, 8)         32          C20[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_100 (Embedding)       (None, 1, 8)         1282816     C21[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_101 (Embedding)       (None, 1, 8)         120         C22[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_102 (Embedding)       (None, 1, 8)         128         C23[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_103 (Embedding)       (None, 1, 8)         256768      C24[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_104 (Embedding)       (None, 1, 8)         512         C25[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_105 (Embedding)       (None, 1, 8)         195448      C26[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 26, 8)        0           embedding_80[0][0]               \n",
      "                                                                 embedding_81[0][0]               \n",
      "                                                                 embedding_82[0][0]               \n",
      "                                                                 embedding_83[0][0]               \n",
      "                                                                 embedding_84[0][0]               \n",
      "                                                                 embedding_85[0][0]               \n",
      "                                                                 embedding_86[0][0]               \n",
      "                                                                 embedding_87[0][0]               \n",
      "                                                                 embedding_88[0][0]               \n",
      "                                                                 embedding_89[0][0]               \n",
      "                                                                 embedding_90[0][0]               \n",
      "                                                                 embedding_91[0][0]               \n",
      "                                                                 embedding_92[0][0]               \n",
      "                                                                 embedding_93[0][0]               \n",
      "                                                                 embedding_94[0][0]               \n",
      "                                                                 embedding_95[0][0]               \n",
      "                                                                 embedding_96[0][0]               \n",
      "                                                                 embedding_97[0][0]               \n",
      "                                                                 embedding_98[0][0]               \n",
      "                                                                 embedding_99[0][0]               \n",
      "                                                                 embedding_100[0][0]              \n",
      "                                                                 embedding_101[0][0]              \n",
      "                                                                 embedding_102[0][0]              \n",
      "                                                                 embedding_103[0][0]              \n",
      "                                                                 embedding_104[0][0]              \n",
      "                                                                 embedding_105[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_55 (Flatten)            (None, 208)          0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 256)          53504       flatten_55[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 256)          0           dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 8)            0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 26, 8)        0           concatenate_2[0][0]              \n",
      "                                                                 concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 256)          65792       dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "I1 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I2 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I3 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I4 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I5 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I6 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I7 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I8 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I9 (InputLayer)                 [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I10 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I11 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I12 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "I13 (InputLayer)                [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_54 (Embedding)        (None, 1, 1)         1091        C1[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_55 (Embedding)        (None, 1, 1)         528         C2[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_56 (Embedding)        (None, 1, 1)         207579      C3[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_57 (Embedding)        (None, 1, 1)         85911       C4[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_58 (Embedding)        (None, 1, 1)         247         C5[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_59 (Embedding)        (None, 1, 1)         14          C6[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_60 (Embedding)        (None, 1, 1)         10359       C7[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_61 (Embedding)        (None, 1, 1)         513         C8[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_62 (Embedding)        (None, 1, 1)         4           C9[0][0]                         \n",
      "__________________________________________________________________________________________________\n",
      "embedding_63 (Embedding)        (None, 1, 1)         25180       C10[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_64 (Embedding)        (None, 1, 1)         4557        C11[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_65 (Embedding)        (None, 1, 1)         177154      C12[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_66 (Embedding)        (None, 1, 1)         3042        C13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_67 (Embedding)        (None, 1, 1)         27          C14[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_68 (Embedding)        (None, 1, 1)         8136        C15[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_69 (Embedding)        (None, 1, 1)         139003      C16[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_70 (Embedding)        (None, 1, 1)         11          C17[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_71 (Embedding)        (None, 1, 1)         3615        C18[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_72 (Embedding)        (None, 1, 1)         1749        C19[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_73 (Embedding)        (None, 1, 1)         4           C20[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_74 (Embedding)        (None, 1, 1)         160352      C21[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_75 (Embedding)        (None, 1, 1)         15          C22[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_76 (Embedding)        (None, 1, 1)         16          C23[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_77 (Embedding)        (None, 1, 1)         32096       C24[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_78 (Embedding)        (None, 1, 1)         64          C25[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_79 (Embedding)        (None, 1, 1)         24431       C26[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 8)            0           lambda_4[0][0]                   \n",
      "                                                                 lambda_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, 8)            0           multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 256)          0           dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 13)           0           I1[0][0]                         \n",
      "                                                                 I2[0][0]                         \n",
      "                                                                 I3[0][0]                         \n",
      "                                                                 I4[0][0]                         \n",
      "                                                                 I5[0][0]                         \n",
      "                                                                 I6[0][0]                         \n",
      "                                                                 I7[0][0]                         \n",
      "                                                                 I8[0][0]                         \n",
      "                                                                 I9[0][0]                         \n",
      "                                                                 I10[0][0]                        \n",
      "                                                                 I11[0][0]                        \n",
      "                                                                 I12[0][0]                        \n",
      "                                                                 I13[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "flatten_29 (Flatten)            (None, 1)            0           embedding_54[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_30 (Flatten)            (None, 1)            0           embedding_55[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_31 (Flatten)            (None, 1)            0           embedding_56[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_32 (Flatten)            (None, 1)            0           embedding_57[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_33 (Flatten)            (None, 1)            0           embedding_58[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_34 (Flatten)            (None, 1)            0           embedding_59[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_35 (Flatten)            (None, 1)            0           embedding_60[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_36 (Flatten)            (None, 1)            0           embedding_61[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_37 (Flatten)            (None, 1)            0           embedding_62[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_38 (Flatten)            (None, 1)            0           embedding_63[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_39 (Flatten)            (None, 1)            0           embedding_64[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_40 (Flatten)            (None, 1)            0           embedding_65[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_41 (Flatten)            (None, 1)            0           embedding_66[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_42 (Flatten)            (None, 1)            0           embedding_67[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_43 (Flatten)            (None, 1)            0           embedding_68[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_44 (Flatten)            (None, 1)            0           embedding_69[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_45 (Flatten)            (None, 1)            0           embedding_70[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_46 (Flatten)            (None, 1)            0           embedding_71[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_47 (Flatten)            (None, 1)            0           embedding_72[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_48 (Flatten)            (None, 1)            0           embedding_73[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_49 (Flatten)            (None, 1)            0           embedding_74[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_50 (Flatten)            (None, 1)            0           embedding_75[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_51 (Flatten)            (None, 1)            0           embedding_76[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_52 (Flatten)            (None, 1)            0           embedding_77[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_53 (Flatten)            (None, 1)            0           embedding_78[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "flatten_54 (Flatten)            (None, 1)            0           embedding_79[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "subtract_1 (Subtract)           (None, 8)            0           multiply_2[0][0]                 \n",
      "                                                                 lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 256)          65792       dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1)            14          concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 1)            0           flatten_29[0][0]                 \n",
      "                                                                 flatten_30[0][0]                 \n",
      "                                                                 flatten_31[0][0]                 \n",
      "                                                                 flatten_32[0][0]                 \n",
      "                                                                 flatten_33[0][0]                 \n",
      "                                                                 flatten_34[0][0]                 \n",
      "                                                                 flatten_35[0][0]                 \n",
      "                                                                 flatten_36[0][0]                 \n",
      "                                                                 flatten_37[0][0]                 \n",
      "                                                                 flatten_38[0][0]                 \n",
      "                                                                 flatten_39[0][0]                 \n",
      "                                                                 flatten_40[0][0]                 \n",
      "                                                                 flatten_41[0][0]                 \n",
      "                                                                 flatten_42[0][0]                 \n",
      "                                                                 flatten_43[0][0]                 \n",
      "                                                                 flatten_44[0][0]                 \n",
      "                                                                 flatten_45[0][0]                 \n",
      "                                                                 flatten_46[0][0]                 \n",
      "                                                                 flatten_47[0][0]                 \n",
      "                                                                 flatten_48[0][0]                 \n",
      "                                                                 flatten_49[0][0]                 \n",
      "                                                                 flatten_50[0][0]                 \n",
      "                                                                 flatten_51[0][0]                 \n",
      "                                                                 flatten_52[0][0]                 \n",
      "                                                                 flatten_53[0][0]                 \n",
      "                                                                 flatten_54[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 8)            0           subtract_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 256)          0           dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 1)            0           dense[0][0]                      \n",
      "                                                                 add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, 1)            0           lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 1)            257         dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 1)            0           add_4[0][0]                      \n",
      "                                                                 lambda_7[0][0]                   \n",
      "                                                                 dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 1)            0           add_5[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 8,156,641\n",
      "Trainable params: 8,156,641\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['binary_crossentropy',tf.keras.metrics.AUC(name='auc')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = total_data.loc[:500000-1]\n",
    "valid_data = total_data.loc[500000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dense_x = [train_data[f].values for f in dense_feats]\n",
    "train_sparse_x = [train_data[f].values for f in sparse_feats]\n",
    "\n",
    "train_label = [train_data['label'].values]\n",
    "\n",
    "val_dense_x = [valid_data[f].values for f in dense_feats]\n",
    "val_sparse_x = [valid_data[f].values for f in sparse_feats]\n",
    "\n",
    "val_label = [valid_data['label'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 500000 samples, validate on 100000 samples\n",
      "Epoch 1/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n",
      "D:\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500000/500000 [==============================] - 113s 225us/sample - loss: 32.7361 - binary_crossentropy: 0.5236 - auc: 0.6997 - val_loss: 0.5901 - val_binary_crossentropy: 0.5122 - val_auc: 0.7324\n",
      "Epoch 2/5\n",
      "500000/500000 [==============================] - 105s 210us/sample - loss: 0.5788 - binary_crossentropy: 0.4981 - auc: 0.7368 - val_loss: 0.5836 - val_binary_crossentropy: 0.5043 - val_auc: 0.7357\n",
      "Epoch 3/5\n",
      "500000/500000 [==============================] - 106s 212us/sample - loss: 0.5917 - binary_crossentropy: 0.4963 - auc: 0.7396 - val_loss: 0.6094 - val_binary_crossentropy: 0.5036 - val_auc: 0.7346\n",
      "Epoch 4/5\n",
      "500000/500000 [==============================] - 105s 210us/sample - loss: 0.6028 - binary_crossentropy: 0.4956 - auc: 0.7407 - val_loss: 0.6022 - val_binary_crossentropy: 0.5029 - val_auc: 0.7352\n",
      "Epoch 5/5\n",
      "500000/500000 [==============================] - 106s 212us/sample - loss: 0.6076 - binary_crossentropy: 0.4949 - auc: 0.7417 - val_loss: 0.6261 - val_binary_crossentropy: 0.5005 - val_auc: 0.7370\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x22eac3ffac8>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_dense_x+train_sparse_x, train_label,\n",
    "          epochs=5, batch_size=256,\n",
    "         validation_data = (val_dense_x+val_sparse_x, val_label),\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}