try-bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Input-Token (InputLayer)        (None, 512)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Input-Segment (InputLayer)      (None, 512)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token (TokenEmbedding [(None, 512, 768), ( 24576000    Input-Token[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Segment (Embedding)   (None, 512, 768)     1536        Input-Segment[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Token-Segment (Add)   (None, 512, 768)     0           Embedding-Token[0][0]            \n",
      "                                                                 Embedding-Segment[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Position (PositionEmb (None, 512, 768)     393216      Embedding-Token-Segment[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Dropout (Dropout)     (None, 512, 768)     0           Embedding-Position[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "Embedding-Norm (LayerNormalizat (None, 512, 768)     1536        Embedding-Dropout[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     2362368     Embedding-Norm[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-1-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     0           Embedding-Norm[0][0]             \n",
      "                                                                 Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-1-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-1-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-1-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-1-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-1-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-1-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-1-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-2-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-1-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-2-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-2-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-2-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-2-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-2-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-2-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-2-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-3-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-2-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-3-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-3-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-3-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-3-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-3-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-3-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-3-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-4-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-3-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-4-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-4-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-4-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-4-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-4-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-4-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-4-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-5-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-4-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-5-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-5-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-5-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-5-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-5-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-5-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-5-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-6-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-5-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-6-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-6-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-6-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-6-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-6-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-6-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-6-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-7-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-6-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-7-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-7-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-7-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-7-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-7-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-7-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-7-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-8-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-7-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-8-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-8-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-8-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-8-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-8-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-8-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     2362368     Encoder-8-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-9-MultiHeadSelfAttention[\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     0           Encoder-8-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-MultiHeadSelfAttentio (None, 512, 768)     1536        Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward (FeedForw (None, 512, 768)     4722432     Encoder-9-MultiHeadSelfAttention-\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Dropout ( (None, 512, 768)     0           Encoder-9-FeedForward[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Add (Add) (None, 512, 768)     0           Encoder-9-MultiHeadSelfAttention-\n",
      "                                                                 Encoder-9-FeedForward-Dropout[0][\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-9-FeedForward-Norm (Lay (None, 512, 768)     1536        Encoder-9-FeedForward-Add[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     2362368     Encoder-9-FeedForward-Norm[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-9-FeedForward-Norm[0][0] \n",
      "                                                                 Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-MultiHeadSelfAttenti (None, 512, 768)     1536        Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward (FeedFor (None, 512, 768)     4722432     Encoder-10-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Dropout  (None, 512, 768)     0           Encoder-10-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Add (Add (None, 512, 768)     0           Encoder-10-MultiHeadSelfAttention\n",
      "                                                                 Encoder-10-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-10-FeedForward-Norm (La (None, 512, 768)     1536        Encoder-10-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     2362368     Encoder-10-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-10-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-MultiHeadSelfAttenti (None, 512, 768)     1536        Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward (FeedFor (None, 512, 768)     4722432     Encoder-11-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Dropout  (None, 512, 768)     0           Encoder-11-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Add (Add (None, 512, 768)     0           Encoder-11-MultiHeadSelfAttention\n",
      "                                                                 Encoder-11-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-11-FeedForward-Norm (La (None, 512, 768)     1536        Encoder-11-FeedForward-Add[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     2362368     Encoder-11-FeedForward-Norm[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     0           Encoder-11-FeedForward-Norm[0][0]\n",
      "                                                                 Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-MultiHeadSelfAttenti (None, 512, 768)     1536        Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward (FeedFor (None, 512, 768)     4722432     Encoder-12-MultiHeadSelfAttention\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Dropout  (None, 512, 768)     0           Encoder-12-FeedForward[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Add (Add (None, 512, 768)     0           Encoder-12-MultiHeadSelfAttention\n",
      "                                                                 Encoder-12-FeedForward-Dropout[0]\n",
      "__________________________________________________________________________________________________\n",
      "Encoder-12-FeedForward-Norm (La (None, 512, 768)     1536        Encoder-12-FeedForward-Add[0][0] \n",
      "==================================================================================================\n",
      "Total params: 110,026,752\n",
      "Trainable params: 0\n",
      "Non-trainable params: 110,026,752\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras_bert import load_trained_model_from_checkpoint\n",
    "\n",
    "config_path = './model/bert_config.json'\n",
    "checkpoint_path = './model/model.ckpt-1400000'\n",
    "\n",
    "bert = load_trained_model_from_checkpoint(config_path, checkpoint_path)\n",
    "bert.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "import numpy as np\n",
    "\n",
    "spp = spm.SentencePieceProcessor()\n",
    "spp.Load('./model/wiki-ja.model')\n",
    "\n",
    "def texts2matrix(texts):\n",
    "    maxlen = 512 # from BERT config\n",
    "    common_seg_input = np.zeros((len(texts), maxlen), dtype = np.float32)\n",
    "    matrix = np.zeros((len(texts), maxlen), dtype = np.float32)\n",
    "    for i, text in enumerate(texts):\n",
    "        tok = [w for w in spp.encode_as_pieces(text.replace(\" \", \"\"))]\n",
    "        if tok == [] or len(tok) > maxlen:\n",
    "            print(\"skip processing\", tok)\n",
    "        else:\n",
    "            tokens = []\n",
    "            tokens.append('[CLS]')\n",
    "            tokens.extend(tok)\n",
    "            tokens.append('[SEP]')\n",
    "            for t, token in enumerate(tokens):\n",
    "                try:\n",
    "                    matrix[i, t] = spp.piece_to_id(token)\n",
    "                except:\n",
    "                    print(token+\"is unknown\")\n",
    "                    matrix[i, t] = spp.piece_to_id('<unk>')\n",
    "    return bert.predict([matrix, common_seg_input])[:,0] # embedding of [CLS]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['▁', '隣の', '客', 'は', 'よく', '柿', '食', 'う', '客', 'だ']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spp.encode_as_pieces(\"隣の客はよく柿食う客だ\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-3.41196954e-01,  1.03131640e+00,  1.53629273e-01,\n",
       "        -4.40728009e-01,  1.48483515e-02,  8.22932497e-02,\n",
       "        -2.90441006e-01,  7.41560638e-01,  9.66181457e-01,\n",
       "        -5.53366125e-01, -3.13522905e-01,  4.32044834e-01,\n",
       "        -3.25102150e-01, -2.92775273e-01, -3.10505331e-02,\n",
       "        -4.70475703e-01, -5.41527793e-02, -5.28014079e-02,\n",
       "         5.45616820e-02, -4.55384225e-01, -1.48637548e-01,\n",
       "        -2.70385772e-01,  3.95462364e-01,  7.42649853e-01,\n",
       "         3.23779643e-01, -1.18695222e-01,  2.62434155e-01,\n",
       "        -3.65382016e-01,  3.53820950e-01,  9.89902139e-01,\n",
       "         5.05595505e-01,  5.27373552e-01,  3.30920160e-01,\n",
       "        -6.04039207e-02, -9.53062773e-02, -2.08897784e-01,\n",
       "        -3.60898823e-01, -4.41889614e-01, -1.66381931e+00,\n",
       "         8.48136604e-01,  9.16482806e-02,  1.83489114e-01,\n",
       "         2.44988471e-01, -1.15357660e-01,  4.47331876e-01,\n",
       "        -1.04940629e+00,  1.93154171e-01,  1.48590133e-02,\n",
       "         1.32922947e-01,  1.21201493e-01, -2.23365307e-01,\n",
       "        -8.09091270e-01, -4.60399389e-01,  2.00671121e-01,\n",
       "         6.47910535e-01, -2.68025309e-01, -9.11149025e-01,\n",
       "         2.19839424e-01,  4.54713076e-01, -1.84328794e-01,\n",
       "         1.50350094e-01, -1.30243357e-02, -9.76140052e-03,\n",
       "         1.31779209e-01, -3.14491838e-01,  4.01578903e-01,\n",
       "         1.75621295e+00, -8.31146315e-02,  1.70686990e-01,\n",
       "        -1.58521140e+00,  2.43181840e-01, -1.89781934e-01,\n",
       "        -2.57623762e-01, -4.38549995e-01,  2.97371179e-01,\n",
       "        -3.85520011e-01, -4.75291044e-01, -2.49779727e-02,\n",
       "         1.09317386e+00,  4.57887113e-01, -8.17321301e-01,\n",
       "         3.41786504e-01, -1.48914129e-01,  3.06992143e-01,\n",
       "         7.91547894e-02, -1.29809761e+00,  5.80829918e-01,\n",
       "         1.92527235e-01,  5.51201366e-02, -8.72347891e-01,\n",
       "        -7.36681342e-01,  1.42902568e-01,  6.13276064e-01,\n",
       "         4.32970226e-01,  1.18495613e-01,  6.88618362e-01,\n",
       "        -3.01248878e-02, -1.29240185e-01, -9.97886658e-02,\n",
       "        -8.74036551e-01, -1.07920095e-01, -2.64189541e-01,\n",
       "         9.46380734e-01, -3.35174203e-01,  4.83301878e-01,\n",
       "         4.31580096e-01,  3.39167893e-01,  4.15103316e-01,\n",
       "        -2.94816911e-01, -7.05796003e-01,  5.07722676e-01,\n",
       "        -2.80282013e-02, -8.94954205e-01, -2.71270927e-02,\n",
       "         6.44469321e-01, -1.62413001e-01, -1.06403530e-01,\n",
       "         3.22235227e-01, -6.72499910e-02,  1.87533274e-02,\n",
       "         2.53466852e-02,  4.28025097e-01, -5.15553355e-01,\n",
       "        -8.07807267e-01,  4.81993765e-01, -1.28574237e-01,\n",
       "         9.45348740e-01,  1.26531649e+00, -1.08862448e+00,\n",
       "        -3.92194539e-01, -3.89907882e-02,  3.90915066e-01,\n",
       "         5.79836190e-01,  1.39137283e-01, -2.93854892e-01,\n",
       "        -4.50849831e-01, -3.58672976e-01,  7.13262737e-01,\n",
       "         1.26347288e-01,  4.16903019e-01, -5.03438056e-01,\n",
       "        -3.22168082e-01, -4.16828185e-01, -7.44621992e-01,\n",
       "        -7.26766765e-01,  2.50595987e-01, -4.69411612e-01,\n",
       "         3.62520933e-01, -8.73152837e-02,  3.11666369e-01,\n",
       "        -2.03936934e-01,  3.08193803e-01, -2.12300718e-02,\n",
       "        -1.58420160e-01, -1.04484236e+00,  5.47882795e-01,\n",
       "         9.43869472e-01, -7.92398080e-02, -6.23048581e-02,\n",
       "        -3.37976784e-01, -4.37112838e-01,  5.88934839e-01,\n",
       "         8.12318563e-01, -2.80522779e-02,  7.60511935e-01,\n",
       "        -1.51935682e-01,  1.83395622e-03,  5.50399601e-01,\n",
       "        -6.84230983e-01, -4.97771770e-01,  3.05638462e-01,\n",
       "        -2.59333141e-02,  6.26489639e-01,  5.53940177e-01,\n",
       "         4.67159837e-01, -7.60172367e-01, -8.28081146e-02,\n",
       "        -8.31299543e-01, -9.93815511e-02, -3.30541193e-01,\n",
       "         5.98473907e-01, -5.02232671e-01,  4.92046773e-01,\n",
       "         8.40816557e-01, -9.73056108e-02, -9.80954468e-01,\n",
       "         5.69156110e-01, -9.28330719e-01,  3.24458718e-01,\n",
       "         2.98354238e-01, -1.91841453e-01, -4.63392973e-01,\n",
       "        -6.60693824e-01, -4.78488445e-01, -5.74834824e-01,\n",
       "        -7.34087706e-01,  1.89678013e-01,  2.42255293e-02,\n",
       "        -5.65084875e-01,  1.04403949e+00, -2.08863676e-01,\n",
       "         6.47452950e-01,  6.26616359e-01, -4.15137559e-01,\n",
       "        -7.10043982e-02,  2.68883139e-01, -2.18708411e-01,\n",
       "        -4.29461926e-01,  5.33697605e-01,  1.87634811e-01,\n",
       "        -2.00889558e-02, -2.95353293e-01,  9.91718024e-02,\n",
       "         1.43453792e-01, -6.24500290e-02, -2.07394496e-01,\n",
       "         2.83592463e-01,  5.59615970e-01,  9.82661173e-02,\n",
       "        -3.09230447e-01, -7.34772719e-03,  3.46308202e-01,\n",
       "        -1.75663978e-01,  1.30158231e-01,  2.75899887e-01,\n",
       "        -3.47817123e-01,  6.17740691e-01,  4.75798547e-01,\n",
       "        -2.04927474e-02,  1.45841181e-01,  5.54816723e-01,\n",
       "         2.15199172e-01, -5.54657318e-02,  5.07273450e-02,\n",
       "         1.45997077e-01,  4.48609740e-01, -3.35066915e-01,\n",
       "        -1.11587310e+00,  5.53072751e-01,  7.67248988e-01,\n",
       "        -1.78028628e-01,  1.07554650e+00,  2.12198496e-01,\n",
       "         5.71074188e-01,  3.44125092e-01,  3.85336816e-01,\n",
       "         2.88215905e-01,  5.58551192e-01,  7.39583194e-01,\n",
       "        -1.31670117e+00, -1.94250215e-02, -5.93101680e-01,\n",
       "        -1.97017506e-01,  9.26928297e-02, -3.58675748e-01,\n",
       "        -1.56278722e-02, -2.40840644e-01, -1.29527166e-01,\n",
       "        -5.60039103e-01,  4.72759426e-01,  1.19632810e-01,\n",
       "         3.69585216e-01, -1.09749116e-01,  3.79618704e-02,\n",
       "        -4.29493263e-02,  8.13191235e-01,  8.37159455e-01,\n",
       "         9.42881405e-02, -1.80082336e-01,  3.25201660e-01,\n",
       "        -8.67232800e-01, -2.25841492e-01,  6.24148965e-01,\n",
       "         4.13805127e-01,  2.45852366e-01, -2.45398536e-01,\n",
       "        -1.50448084e-01, -4.98111635e-01,  6.32965147e-01,\n",
       "        -1.38484374e-01, -1.78183213e-01, -3.07016373e-01,\n",
       "         2.03114241e-01,  4.92489040e-01, -7.87337571e-02,\n",
       "         1.99491501e-01,  5.00335574e-01,  7.60092437e-01,\n",
       "        -4.54912744e-02,  2.91224480e-01,  1.33285239e-01,\n",
       "         5.37026167e-01,  1.75356776e-01, -3.08089286e-01,\n",
       "         2.68935353e-01, -6.21328712e-01, -6.51691020e-01,\n",
       "        -3.50849509e-01, -4.82045889e-01,  2.37454847e-01,\n",
       "         9.17111456e-01,  4.47165310e-01, -7.88008422e-03,\n",
       "        -7.27694631e-01,  3.25741410e-01,  1.02858114e+00,\n",
       "        -7.21012279e-02,  6.11205220e-01, -1.81915551e-01,\n",
       "        -2.92757079e-02,  6.60989881e-01, -2.14850038e-01,\n",
       "        -2.72726923e-01,  3.53152603e-01,  8.76680538e-02,\n",
       "         3.69301736e-02,  3.67751867e-01, -4.73997563e-01,\n",
       "        -2.95697004e-01, -4.08319354e-01, -3.47831249e-02,\n",
       "        -3.67018491e-01,  3.32422405e-01, -6.86930537e-01,\n",
       "         2.19339877e-01,  7.23144710e-01,  1.01503754e+00,\n",
       "        -4.13619429e-01, -3.02973211e-01,  5.06616771e-01,\n",
       "        -3.78170982e-04,  2.09247425e-01, -3.90816748e-01,\n",
       "        -6.94561601e-02, -8.10936093e-01,  3.94879244e-02,\n",
       "         1.30733699e-01, -5.29211938e-01, -5.38954794e-01,\n",
       "         2.66199887e-01,  7.52545476e-01,  6.90532207e-01,\n",
       "         2.51175910e-01,  3.04693371e-01,  5.43156266e-01,\n",
       "         3.06274831e-01,  5.95492184e-01, -5.42922139e-01,\n",
       "         4.05604154e-01,  2.22407952e-01,  4.24313545e-01,\n",
       "        -2.70783395e-01,  1.18448511e-01, -1.63319916e-01,\n",
       "        -2.89082438e-01, -6.04633335e-03, -4.32550237e-02,\n",
       "        -3.94020051e-01,  3.56435895e-01, -4.89008009e-01,\n",
       "        -5.92878163e-01, -5.32847904e-02, -1.82254195e-01,\n",
       "        -8.78034711e-01, -4.30920899e-01,  4.04655188e-01,\n",
       "        -3.81023407e-01, -1.71280161e-01,  1.38276398e-01,\n",
       "        -9.09301221e-01,  3.83267045e-01, -4.21728224e-01,\n",
       "        -1.22696213e-01, -6.89740002e-01, -2.86078006e-01,\n",
       "         2.81837583e-03, -6.48831606e-01, -3.34766209e-01,\n",
       "         4.64410454e-01,  4.42226529e-01, -2.83131927e-01,\n",
       "         6.46065474e-01, -8.13250065e-01,  1.00194776e+00,\n",
       "        -6.23493910e-01,  1.83197826e-01,  3.57828178e-02,\n",
       "        -4.44781363e-01,  2.06424236e-01,  1.39159383e-02,\n",
       "        -1.76182479e-01,  3.54258239e-01, -4.58734810e-01,\n",
       "        -3.46053094e-01,  2.40359589e-01,  4.56818044e-02,\n",
       "        -4.76316363e-03,  6.37176573e-01,  3.11916918e-01,\n",
       "        -1.00646782e+00,  4.41091508e-01,  5.84388852e-01,\n",
       "         5.56233108e-01, -4.35185760e-01, -1.80617750e-01,\n",
       "        -1.92396492e-02,  4.78572249e-01, -2.64197111e-01,\n",
       "         8.84815231e-02,  3.94915670e-01, -3.39671552e-01,\n",
       "        -4.33810234e-01, -8.84257674e-01,  7.22541869e-01,\n",
       "         3.28476042e-01,  1.24967501e-01, -2.53009826e-01,\n",
       "         5.59427559e-01, -2.79681142e-02, -2.38478988e-01,\n",
       "        -1.41505465e-01,  1.26825973e-01,  3.79392296e-01,\n",
       "         1.80786625e-02,  3.85819048e-01,  7.80409038e-01,\n",
       "         8.04184750e-03,  1.79238772e+00,  4.26952600e-01,\n",
       "        -1.12128928e-01, -9.23069537e-01,  4.58373547e-01,\n",
       "        -3.76697510e-01, -5.84068239e-01, -8.95970225e-01,\n",
       "         2.83646673e-01,  1.34370893e-01, -2.34166712e-01,\n",
       "        -6.96452633e-02, -3.04343283e-01, -1.18256651e-01,\n",
       "         1.00144576e-02,  4.93974268e-01, -2.11021140e-01,\n",
       "        -6.01036191e-01, -5.56520522e-01, -8.23370874e-01,\n",
       "        -3.45890224e-03, -4.75362688e-01, -6.51902139e-01,\n",
       "        -2.01324016e-01, -2.70224959e-02, -1.00335801e+00,\n",
       "         4.96688545e-01, -7.56352544e-01,  1.15239173e-02,\n",
       "         8.36816356e-02, -4.68099296e-01, -6.61606431e-01,\n",
       "        -8.56553614e-02, -4.55347836e-01,  8.11865568e-01,\n",
       "        -1.87989771e-02, -2.63478935e-01,  1.07976779e-01,\n",
       "        -1.53609663e-01,  3.24554801e-01, -4.24242884e-01,\n",
       "         8.65549147e-02,  6.17535114e-01,  8.53862882e-01,\n",
       "         1.19150028e-01, -1.14588365e-01,  1.19078064e+00,\n",
       "         1.20398805e-01,  9.09195095e-02, -1.43212438e-01,\n",
       "        -4.10865426e-01,  3.27569425e-01, -1.35679573e-01,\n",
       "         3.78244996e-01,  6.28936648e-01, -5.02290428e-01,\n",
       "        -2.64282346e-01,  5.54659069e-01, -3.91025424e-01,\n",
       "         7.54323155e-02, -2.95518339e-01, -3.55624557e-02,\n",
       "        -3.63153011e-01,  2.05787018e-01, -2.03549683e-01,\n",
       "         7.02224314e-01,  1.23427832e+00,  9.57435742e-02,\n",
       "        -1.52445495e-01,  7.68611059e-02, -9.01131988e-01,\n",
       "         7.83828795e-02,  3.06580439e-02,  5.93883991e-01,\n",
       "        -3.66236493e-02,  6.91558197e-02,  1.51393011e-01,\n",
       "         7.27045536e-01, -7.44491756e-01,  6.20405793e-01,\n",
       "         6.55160248e-01, -5.21443307e-01,  8.50050151e-02,\n",
       "        -4.54568863e-01,  2.59443045e-01,  8.44585657e-01,\n",
       "        -3.35721344e-01, -1.38397086e-02,  5.45821249e-01,\n",
       "        -2.33279824e-01, -1.01633936e-01,  1.77354857e-01,\n",
       "        -6.31835938e-01,  9.28710699e-02,  1.11194479e+00,\n",
       "         4.58208174e-01,  3.80069584e-01, -9.73609239e-02,\n",
       "        -9.83000517e-01, -1.90630585e-01,  5.14976084e-01,\n",
       "         3.26729625e-01, -1.42130756e+00,  2.68814862e-01,\n",
       "        -3.86303008e-01,  1.77386224e-01,  6.89821661e-01,\n",
       "         4.05710578e-01,  8.46539021e-01, -3.82536918e-01,\n",
       "         1.21904090e-01,  1.39078528e-01,  1.83218032e-01,\n",
       "         6.47338033e-01, -3.57320786e-01, -2.79620111e-01,\n",
       "        -1.87447786e-01, -3.31504911e-01, -2.06307292e-01,\n",
       "        -3.36240202e-01,  7.83670098e-02,  1.11741498e-01,\n",
       "        -7.64224410e-01, -4.76161361e-01,  2.85779648e-02,\n",
       "        -6.27083659e-01, -1.09508492e-01,  6.64650798e-02,\n",
       "         3.01101267e-01,  3.84633899e-01,  5.52473009e-01,\n",
       "        -6.81100428e-01, -2.86236376e-01,  4.07056451e-01,\n",
       "         4.43867326e-01,  7.77267098e-01,  4.09620721e-03,\n",
       "        -6.92541480e-01, -7.94005334e-01,  2.97854155e-01,\n",
       "        -1.94963813e-02, -4.89406198e-01,  2.81771943e-02,\n",
       "         1.20716847e-01,  4.52844322e-01,  3.90465736e-01,\n",
       "        -5.00926137e-01,  5.88028908e-01, -2.18286052e-01,\n",
       "        -6.90258026e-01, -2.26257876e-01, -7.10235983e-02,\n",
       "         7.89520703e-03, -7.00282216e-01,  1.30513310e-01,\n",
       "        -4.36396450e-02, -2.99966246e-01,  2.93400198e-01,\n",
       "         8.78202692e-02, -4.77716953e-01, -1.05323708e+00,\n",
       "        -3.22917700e-01, -1.84710249e-01,  2.71997213e-01,\n",
       "         4.60757554e-01,  6.62419945e-04,  9.24677253e-02,\n",
       "        -1.53675042e-02, -7.96687603e-01, -2.06498995e-01,\n",
       "        -4.80436385e-01, -7.59519875e-01, -1.05072010e+00,\n",
       "        -6.39558733e-01,  7.63463318e-01,  5.91687083e-01,\n",
       "        -9.63577271e+00,  4.44700271e-01,  1.31367251e-01,\n",
       "         6.10854805e-01, -1.18617558e+00,  3.13978732e-01,\n",
       "         2.49987781e-01, -6.96834102e-02, -1.49432167e-01,\n",
       "        -1.09629415e-01, -6.13219440e-02, -4.78914231e-02,\n",
       "        -1.41790450e-01, -2.40927696e-01,  3.50070059e-01,\n",
       "         4.99029547e-01,  7.19281435e-02,  6.84727311e-01,\n",
       "        -2.77015626e-01,  1.99780226e-01,  2.76441962e-01,\n",
       "        -3.99392247e-01, -2.99042314e-01,  2.67265588e-01,\n",
       "        -7.61341631e-01, -7.84940898e-01, -5.27213037e-01,\n",
       "         1.97685331e-01,  5.44225276e-01, -2.60761708e-01,\n",
       "         3.52040827e-02, -4.80326600e-02,  1.32204592e-03,\n",
       "        -5.91455460e-01, -2.17847340e-02,  5.66341728e-02,\n",
       "        -3.94499958e-01,  4.14105237e-01,  1.99507941e-02,\n",
       "        -1.99712098e-01, -7.05869570e-02, -5.41249752e-01,\n",
       "         7.11569637e-02,  1.02350354e+00,  4.81278837e-01,\n",
       "         1.97529629e-01,  3.99961889e-01,  4.17143792e-01,\n",
       "         1.69111088e-01,  4.15739447e-01,  2.24750027e-01,\n",
       "        -5.12715936e-01, -8.03451985e-02,  7.04362154e-01,\n",
       "         1.70494199e-01, -1.07044518e+00, -9.79289636e-02,\n",
       "         3.56196344e-01,  4.53739166e-01,  3.76853824e-01,\n",
       "         1.12506062e-01, -3.66817415e-01, -1.30538374e-01,\n",
       "         8.32410693e-01,  7.48137087e-02,  7.15898156e-01,\n",
       "         3.58702064e-01, -2.55321208e-02, -3.00934538e-03,\n",
       "        -5.37060022e-01, -9.33886878e-03,  3.79176438e-01,\n",
       "         3.71628225e-01, -1.47640318e-01,  3.39386433e-01,\n",
       "         2.09279627e-01, -1.06807029e+00,  2.37923786e-01,\n",
       "         7.06852376e-02, -8.61725807e-02,  2.56560057e-01,\n",
       "         7.42613137e-01,  5.88195205e-01, -7.94894159e-01,\n",
       "        -2.86156446e-01, -2.63243258e-01, -1.73382401e-01,\n",
       "         6.02445781e-01,  2.02903077e-01, -2.07980677e-01,\n",
       "        -2.38176823e-01,  4.95059371e-01, -9.45077598e-01,\n",
       "         4.79143932e-02, -6.03120923e-01, -5.32997623e-02,\n",
       "        -4.42588776e-02,  6.09056234e-01, -2.73310125e-01,\n",
       "         6.22116089e-01, -5.84838569e-01, -5.69955967e-02,\n",
       "         1.23023987e+00,  8.34992230e-01,  1.28113687e-01,\n",
       "         1.66580409e-01, -3.62012863e-01,  7.63240308e-02,\n",
       "         7.53488243e-01,  4.42869127e-01, -6.31386936e-02,\n",
       "        -5.60628414e-01,  7.20693052e-01,  2.32281625e-01,\n",
       "         5.94475627e-01, -6.95820570e-01,  3.63762736e-01,\n",
       "         1.10444427e-01,  5.20017684e-01, -6.72089279e-01,\n",
       "         7.55287051e-01, -7.14739144e-01, -4.38253522e-01,\n",
       "         8.22951496e-01, -1.97228998e-01,  1.30304351e-01,\n",
       "        -3.87558311e-01, -2.05499053e-01,  2.21723795e-01,\n",
       "        -3.42101790e-02, -3.02060932e-01,  3.65348645e-02,\n",
       "        -1.12452352e+00,  1.80890128e-01,  3.34478796e-01,\n",
       "         1.03679216e+00, -1.97073758e-01, -1.82493255e-01,\n",
       "        -3.47629160e-01, -4.82896924e-01,  1.70449123e-01,\n",
       "         3.12438279e-01, -1.02226591e+00, -3.08820724e-01,\n",
       "        -2.86246628e-01, -6.58816934e-01,  7.44849265e-01,\n",
       "         6.15298271e-01, -1.10467508e-01, -1.04591981e-01,\n",
       "         6.63146734e-01,  1.06748417e-01, -9.10821259e-01,\n",
       "         5.24114907e-01, -3.32358837e-01, -1.54572055e-01,\n",
       "         3.60905588e-01, -2.97609478e-01, -3.91984910e-01,\n",
       "        -1.39377341e-01,  7.65620917e-03, -4.57305431e-01,\n",
       "         1.02569170e-01,  1.20893091e-01,  6.64056718e-01]], dtype=float32)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts2matrix([\"隣の客はよく柿食う客だ\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}