Jupyter file/question_TF_2_7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "186fe388",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.9.7 (tags/v3.9.7:1016ef3, Aug 30 2021, 20:19:38) [MSC v.1929 64 bit (AMD64)]\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "print(sys.version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1e37c006",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.7.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "11dc86f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from flask import jsonify\n",
    "\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras_preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.utils import to_categorical  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23cad0dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Поло Print Bar</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Футболка VseMayki.Ru</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Майка Print Bar</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Friedrich Wilhelm Hermann Wagener \"Staats- un...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Шорты Quiksilver</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                name  category\n",
       "0                                     Поло Print Bar        10\n",
       "1                               Футболка VseMayki.Ru        10\n",
       "2                                    Майка Print Bar        10\n",
       "3   Friedrich Wilhelm Hermann Wagener \"Staats- un...         7\n",
       "4                                   Шорты Quiksilver        10"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('goods_10K.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e7ad53ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [\n",
    "    'Авто',\n",
    "    'Товары для здоровья',\n",
    "    'Электроника',\n",
    "    'Бытовая техника',\n",
    "    'Строительство и ремонт',\n",
    "    'Товары для дома',\n",
    "    'Детские товары',\n",
    "    'Досуг и развлечения',\n",
    "    'Компьютерная техника',\n",
    "    'Товары для красоты',\n",
    "    'Одежда, обувь и аксессуары',\n",
    "    'Продукты',\n",
    "    'Спорт и отдых',\n",
    "    'Дача, сад и огород',\n",
    "    'Товары для животных',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c53bae02",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data['name']\n",
    "y = data['category']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fe7e741b",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15bc36b6",
   "metadata": {},
   "source": [
    "Пока идем нормально"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "12cae412",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5985    10\n",
       "3641    10\n",
       "6765    10\n",
       "3578    10\n",
       "5491    10\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "048e41b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_final = to_categorical(y_train)\n",
    "y_test_final = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d4eb28d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "print(y_train_final[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "65ff40f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7000,) (7000, 15)\n",
      "(3000,) (3000, 15)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, y_train_final.shape)\n",
    "print(X_test.shape, y_test_final.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "dc90bcbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Максимальное количество уникальных токетов (слов) в нашем словаре\n",
    "MAX_WORDS = 50000\n",
    "# Это специальное значение для слов, которых нет в нашем словаре\n",
    "OOV_TOKEN = '<OOV>'\n",
    "# Максимальная длина предложения\n",
    "MAX_SEQUENCE_LENGTH = 10\n",
    "EMBEDDING_DIM = 100\n",
    "# Сколько раз будем прогонять весь тренировочный набор данных при обучении\n",
    "EPOCHS = 3\n",
    "BATCH_SIZE = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9372012e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Всего 7540 уникальных токенов\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(\n",
    "    num_words=MAX_WORDS, \n",
    "    filters='!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~0123456789', # Фильтруем спецсимволы и цифры\n",
    "    lower=True, \n",
    "    oov_token=OOV_TOKEN)\n",
    "tokenizer.fit_on_texts(X_train)\n",
    "word_index = tokenizer.word_index\n",
    "print('Всего %s уникальных токенов' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "397bf290",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(texts):\n",
    "    texts = tokenizer.texts_to_sequences(texts)\n",
    "    return pad_sequences(texts, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6afca189",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7000, 10)\n",
      "(3000, 10)\n"
     ]
    }
   ],
   "source": [
    "X_train_final = preprocess_text(X_train)\n",
    "X_test_final = preprocess_text(X_test)\n",
    "print(X_train_final.shape)\n",
    "print(X_test_final.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "30157839",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0    0    0    0    0    0    0    2    4    3]\n",
      " [   0    0    0    0    0    0    0    2    4    3]\n",
      " [   0    0    0    0    0    0    0   16    4    3]\n",
      " [   0    0    0    0    0    0    0    0   67 2200]\n",
      " [   0    0    0    0    0    0    0    8  835  517]\n",
      " [   0    0    0    0    0    0    0  211  638  212]\n",
      " [   0    0    0  134  124 2201 2202   71  164 2203]\n",
      " [   0    0    0    0    0 1244 2204   19 2205   93]\n",
      " [   0    0    0    0    0    0    0    7    4    3]\n",
      " [   0    0    0    0    0    0    0    0   34 2206]]\n"
     ]
    }
   ],
   "source": [
    "print(X_train_final[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2e06d24a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 10, 100)           5000000   \n",
      "                                                                 \n",
      " spatial_dropout1d (SpatialD  (None, 10, 100)          0         \n",
      " ropout1D)                                                       \n",
      "                                                                 \n",
      " lstm (LSTM)                 (None, 100)               80400     \n",
      "                                                                 \n",
      " dense (Dense)               (None, 15)                1515      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 5,081,915\n",
      "Trainable params: 5,081,915\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=MAX_WORDS, output_dim=EMBEDDING_DIM, input_length=MAX_SEQUENCE_LENGTH))\n",
    "model.add(layers.SpatialDropout1D(0.2))\n",
    "model.add(layers.LSTM(100, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(layers.Dense(len(labels), activation=\"softmax\"))\n",
    "model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "881b9db9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "print(len(model.layers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c605ab96",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "394/394 [==============================] - 17s 39ms/step - loss: 0.7441 - accuracy: 0.8113 - val_loss: 0.3747 - val_accuracy: 0.8929\n",
      "Epoch 2/3\n",
      "394/394 [==============================] - 14s 37ms/step - loss: 0.3122 - accuracy: 0.9038 - val_loss: 0.2976 - val_accuracy: 0.9100\n",
      "Epoch 3/3\n",
      "394/394 [==============================] - 14s 36ms/step - loss: 0.1575 - accuracy: 0.9519 - val_loss: 0.2557 - val_accuracy: 0.9486\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x21b541a2940>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train_final, y_train_final, epochs=EPOCHS, batch_size=BATCH_SIZE, validation_split=0.1, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7dcab5c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(model, path):\n",
    "    tf.keras.models.save_model(\n",
    "        model,\n",
    "        path,\n",
    "        overwrite=True,\n",
    "        include_optimizer=True,\n",
    "        save_format=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8b90fb95",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "def save_tokenizer(tokenizer, path):\n",
    "    with open(path, 'wb') as file:\n",
    "        pickle.dump(tokenizer, file, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4185ed50",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(model, \"goods_classifier.h5\")\n",
    "save_tokenizer(tokenizer, \"tokenizer.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6f2c1eb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate on test data\n",
      "94/94 [==============================] - 0s 1ms/step - loss: 0.2606 - accuracy: 0.9310\n",
      "test loss, test acc: [0.2606315612792969, 0.9309999942779541]\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the model on the test data using `evaluate`\n",
    "print(\"Evaluate on test data\")\n",
    "results = model.evaluate(X_test_final, y_test_final, batch_size=32)\n",
    "print(\"test loss, test acc:\", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ea2c7286",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'label': 'Одежда, обувь и аксессуары', 'confidence': 0.5731292366981506}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.5783060789108276}, {'label': 'Детские товары', 'confidence': 0.5758166909217834}, {'label': 'Детские товары', 'confidence': 0.43981531262397766}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.5783060789108276}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.6860765814781189}, {'label': 'Детские товары', 'confidence': 0.5666932463645935}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.5783060789108276}, {'label': 'Детские товары', 'confidence': 0.4702834188938141}, {'label': 'Детские товары', 'confidence': 0.6185156106948853}, {'label': 'Детские товары', 'confidence': 0.43981531262397766}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.492744117975235}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.6860765814781189}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.822939932346344}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.7559226751327515}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.5125983953475952}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.822939932346344}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.36244484782218933}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.5810175538063049}, {'label': 'Одежда, обувь и аксессуары', 'confidence': 0.3853687047958374}]\n"
     ]
    }
   ],
   "source": [
    "    x = preprocess_text(\"Карта памяти Samsung\")\n",
    "    y = model.predict(x)\n",
    "    result = []\n",
    "    for item in y:\n",
    "        index = np.argmax(item)\n",
    "        result.append({\n",
    "            \"label\": labels[index],\n",
    "            \"confidence\": item.tolist()[index],\n",
    "        })\n",
    "\n",
    "    print(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}