binary_class_and_validation_files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>code_block</th>\n",
       "      <th>tag</th>\n",
       "      <th>imports</th>\n",
       "      <th>load_data</th>\n",
       "      <th>visualize</th>\n",
       "      <th>preprocessing</th>\n",
       "      <th>model</th>\n",
       "      <th>train</th>\n",
       "      <th>predict</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>[' plotting', ' linear algebra', ' accessing d...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>[' plotting', ' linear algebra', ' accessing d...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>[' plotting', ' linear algebra', ' accessing d...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>[' plotting', ' linear algebra', ' accessing d...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>from sklearn.preprocessing import StandardScal...</td>\n",
       "      <td>[' plotting', ' linear algebra', ' accessing d...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                         code_block  \\\n",
       "0           0  from mpl_toolkits.mplot3d import Axes3D\\nfrom ...   \n",
       "1           1  from mpl_toolkits.mplot3d import Axes3D\\nfrom ...   \n",
       "2           2  from mpl_toolkits.mplot3d import Axes3D\\nfrom ...   \n",
       "3           3  from mpl_toolkits.mplot3d import Axes3D\\nfrom ...   \n",
       "4           4  from sklearn.preprocessing import StandardScal...   \n",
       "\n",
       "                                                 tag  imports  load_data  \\\n",
       "0  [' plotting', ' linear algebra', ' accessing d...        0          1   \n",
       "1  [' plotting', ' linear algebra', ' accessing d...        0          1   \n",
       "2  [' plotting', ' linear algebra', ' accessing d...        0          1   \n",
       "3  [' plotting', ' linear algebra', ' accessing d...        0          1   \n",
       "4  [' plotting', ' linear algebra', ' accessing d...        0          1   \n",
       "\n",
       "   visualize  preprocessing  model  train  predict  \n",
       "0          1              0      0      0        0  \n",
       "1          1              0      0      0        0  \n",
       "2          1              0      0      0        0  \n",
       "3          1              0      0      0        0  \n",
       "4          1              0      0      0        0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('/home/kirill/Desktop/regex_by_comm.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Уникальных тегов:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64975"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['tag'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Размер:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(107328,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['tag'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.drop(columns='Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Убираем данные без какой либо метки"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "unlabeled_data = data[data['imports'] + data['load_data'] + data['preprocessing'] + data['model'] + data['train'] + data['predict'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.drop(index=unlabeled_data.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>code_block</th>\n",
       "      <th>imports</th>\n",
       "      <th>load_data</th>\n",
       "      <th>visualize</th>\n",
       "      <th>preprocessing</th>\n",
       "      <th>model</th>\n",
       "      <th>train</th>\n",
       "      <th>predict</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>from sklearn.preprocessing import StandardScal...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107323</th>\n",
       "      <td>\\nclf_list = [('Perceptron', Perceptron(eta0=0...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107324</th>\n",
       "      <td>def test_set_performances(clf_tuple_list, X_tr...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107325</th>\n",
       "      <td>\\nmodel_names, test_acc, test_f1 = test_set_pe...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107326</th>\n",
       "      <td>def plot_confusion_matrix(true_y, pred_y, titl...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107327</th>\n",
       "      <td>\\ngb_clf = GradientBoostingClassifier()\\ngb_cl...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>107328 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               code_block  imports  load_data  \\\n",
       "0       from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0          1   \n",
       "1       from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0          1   \n",
       "2       from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0          1   \n",
       "3       from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0          1   \n",
       "4       from sklearn.preprocessing import StandardScal...        0          1   \n",
       "...                                                   ...      ...        ...   \n",
       "107323  \\nclf_list = [('Perceptron', Perceptron(eta0=0...        0          0   \n",
       "107324  def test_set_performances(clf_tuple_list, X_tr...        0          0   \n",
       "107325  \\nmodel_names, test_acc, test_f1 = test_set_pe...        0          0   \n",
       "107326  def plot_confusion_matrix(true_y, pred_y, titl...        0          0   \n",
       "107327  \\ngb_clf = GradientBoostingClassifier()\\ngb_cl...        0          0   \n",
       "\n",
       "        visualize  preprocessing  model  train  predict  \n",
       "0               1              0      0      0        0  \n",
       "1               1              0      0      0        0  \n",
       "2               1              0      0      0        0  \n",
       "3               1              0      0      0        0  \n",
       "4               1              0      0      0        0  \n",
       "...           ...            ...    ...    ...      ...  \n",
       "107323          1              0      0      0        1  \n",
       "107324          0              0      0      1        1  \n",
       "107325          0              0      0      0        1  \n",
       "107326          1              0      0      1        0  \n",
       "107327          1              0      0      1        1  \n",
       "\n",
       "[107328 rows x 8 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ready_data = data\n",
    "ready_data = ready_data.drop(columns=['tag'], axis=1)\n",
    "ready_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_drop = ready_data[ready_data['code_block'].str.len() > 10000].index\n",
    "\n",
    "ready_data = ready_data.drop(index=to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "ready_data = ready_data.rename(columns={\"load_data\":\"loaddata\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "ready_data = ready_data.reset_index().drop(columns=['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>code_block</th>\n",
       "      <th>imports</th>\n",
       "      <th>loaddata</th>\n",
       "      <th>visualize</th>\n",
       "      <th>preprocessing</th>\n",
       "      <th>model</th>\n",
       "      <th>train</th>\n",
       "      <th>predict</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>from mpl_toolkits.mplot3d import Axes3D\\nfrom ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>from sklearn.preprocessing import StandardScal...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96625</th>\n",
       "      <td>\\nsns.set(rc={'figure.figsize':(12,8)})\\npalet...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96626</th>\n",
       "      <td>sns.set(rc={'figure.figsize':(12,8)})\\npalette...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96627</th>\n",
       "      <td>\\nclf_list = [('Perceptron', Perceptron(eta0=0...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96628</th>\n",
       "      <td>\\nmodel_names, test_acc, test_f1 = test_set_pe...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96629</th>\n",
       "      <td>\\ngb_clf = GradientBoostingClassifier()\\ngb_cl...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>96630 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              code_block  imports  loaddata  \\\n",
       "0      from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0         1   \n",
       "1      from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0         1   \n",
       "2      from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0         1   \n",
       "3      from mpl_toolkits.mplot3d import Axes3D\\nfrom ...        0         1   \n",
       "4      from sklearn.preprocessing import StandardScal...        0         1   \n",
       "...                                                  ...      ...       ...   \n",
       "96625  \\nsns.set(rc={'figure.figsize':(12,8)})\\npalet...        1         0   \n",
       "96626  sns.set(rc={'figure.figsize':(12,8)})\\npalette...        1         0   \n",
       "96627  \\nclf_list = [('Perceptron', Perceptron(eta0=0...        0         0   \n",
       "96628  \\nmodel_names, test_acc, test_f1 = test_set_pe...        0         0   \n",
       "96629  \\ngb_clf = GradientBoostingClassifier()\\ngb_cl...        0         0   \n",
       "\n",
       "       visualize  preprocessing  model  train  predict  \n",
       "0              1              0      0      0        0  \n",
       "1              1              0      0      0        0  \n",
       "2              1              0      0      0        0  \n",
       "3              1              0      0      0        0  \n",
       "4              1              0      0      0        0  \n",
       "...          ...            ...    ...    ...      ...  \n",
       "96625          1              0      0      0        0  \n",
       "96626          1              0      0      0        0  \n",
       "96627          1              0      0      0        1  \n",
       "96628          0              0      0      0        1  \n",
       "96629          1              0      0      1        1  \n",
       "\n",
       "[96630 rows x 8 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_del = []\n",
    "for i in range(ready_data.shape[0]):\n",
    "    if 'def' in ready_data.loc[i, 'code_block']:\n",
    "        to_del.append(i)\n",
    "\n",
    "ready_data = ready_data.drop(index=to_del, axis=1)\n",
    "ready_data = ready_data.reset_index().drop(columns=['index'], axis=1)\n",
    "ready_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Обозначим вершины графа знаний:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = ['imports', 'loaddata', 'visualize', 'preprocessing', 'model', 'train', 'predict']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Разделение на обучающую, тестовую и валидационную выборки, а также переработка сниппетов в файлы под требования проекта code2vec:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "SEED = 42001\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(ready_data.drop(targets, axis=1),\n",
    "                                                   ready_data[targets],\n",
    "                                                   test_size=3/11, random_state=SEED)\n",
    "\n",
    "X_val, X_test, y_val, y_test = train_test_split(X_test, y_test,\n",
    "                                               test_size=0.5, random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_set = X_train.reset_index().drop(columns=['index'], axis=1)\n",
    "test_set = X_test.reset_index().drop(columns=['index'], axis=1)\n",
    "valid_set = X_val.reset_index().drop(columns=['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = y_train.reset_index().drop(columns=['index'], axis=1)\n",
    "y_test = y_test.reset_index().drop(columns=['index'], axis=1)\n",
    "y_val = y_val.reset_index().drop(columns=['index'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Далее все '\\n' заменяются на '\\n\\t', а также вначале каждого сниппета приписывается\n",
    "\n",
    "'def tag():\\n\\t'\n",
    "\n",
    "Каждый сниппет записывается в **отдельный файл** с названием номер_итерации.py в каталог трей, тест или валидейшн."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_tabs(exmpl):\n",
    "    new_exmpl = str()\n",
    "    for i in range(len(exmpl)):\n",
    "        if (exmpl[i]=='\\n'):\n",
    "            new_exmpl += '\\n\\t'\n",
    "        else:\n",
    "            new_exmpl += exmpl[i]\n",
    "    return new_exmpl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_name(target):\n",
    "    name = ''\n",
    "    if target[3] == 1:\n",
    "        return 'preprocessing_block'\n",
    "    else:\n",
    "        return 'no_name_block'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(train_set.shape[0]):\n",
    "        exmpl = train_set.loc[i]['code_block'][:-1]\n",
    "        exmpl = add_tabs(exmpl)\n",
    "        print('def ' + func_name(y_train.loc[i]) +'():\\n\\t' + exmpl\n",
    "              ,  file=open('/home/kirill/code2vec/cd2vec/python/my_train/func'+str(i)+'.py', 'w'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "test:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(test_set.shape[0]):\n",
    "        exmpl = test_set.loc[i]['code_block'][:-1]\n",
    "        exmpl = add_tabs(exmpl)\n",
    "        print('def ' + func_name(y_test.loc[i]) +'():\\n\\t' + exmpl\n",
    "              ,  file=open('/home/kirill/code2vec/cd2vec/python/my_test/func'+str(i)+'.py', 'w'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "valid:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(valid_set.shape[0]):\n",
    "        exmpl = valid_set.loc[i]['code_block'][:-1]\n",
    "        exmpl = add_tabs(exmpl)\n",
    "        print('def ' + func_name(y_val.loc[i]) +'():\\n\\t' + exmpl\n",
    "              ,  file=open('/home/kirill/code2vec/cd2vec/python/my_val/func'+str(i)+'.py', 'w'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Дальше загружаем и распределяем для code2vec валидационную выборку, размеченную экспертно:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation = pd.read_csv('/home/kirill/Desktop/all_google_sheet.csv')\n",
    "validation = validation['Code']\n",
    "validation = validation[validation.isna() != True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation = validation.reset_index().drop(columns=['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>for dataset in full_data:\\n    dataset['Embark...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>for dataset in data_cleaner:\\n        dataset[...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>label = LabelEncoder()\\nfor dataset in data_cl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>for dataset in combine:\\n    dataset['Title'] ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>from sklearn.feature_extraction.text import Co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>\\nfrom textblob import TextBlob\\n\\ndf['sentime...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>\\n\\n\\n\\nimport numpy as np\\nfrom gensim.models...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>from gensim.scripts.glove2word2vec import glov...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>from sklearn.feature_extraction.text import Tf...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>186 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Code\n",
       "0    for dataset in full_data:\\n    dataset['Embark...\n",
       "1    for dataset in data_cleaner:\\n        dataset[...\n",
       "2    label = LabelEncoder()\\nfor dataset in data_cl...\n",
       "3    for dataset in combine:\\n    dataset['Title'] ...\n",
       "4    title_mapping = {\"Mr\": 1, \"Miss\": 2, \"Mrs\": 3,...\n",
       "..                                                 ...\n",
       "181  from sklearn.feature_extraction.text import Co...\n",
       "182  \\nfrom textblob import TextBlob\\n\\ndf['sentime...\n",
       "183  \\n\\n\\n\\nimport numpy as np\\nfrom gensim.models...\n",
       "184  from gensim.scripts.glove2word2vec import glov...\n",
       "185  from sklearn.feature_extraction.text import Tf...\n",
       "\n",
       "[186 rows x 1 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_del = []\n",
    "for i in range(validation.shape[0]):\n",
    "    if 'def' in validation.loc[i]['Code']:\n",
    "        to_del.append(i)\n",
    "\n",
    "validation = validation.drop(index=to_del, axis=1)\n",
    "validation = validation.reset_index().drop(columns=['index'], axis=1)\n",
    "validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(validation.shape[0]):\n",
    "    exmpl = validation.loc[i]['Code']\n",
    "    exmpl = add_tabs(exmpl)\n",
    "    print('def ' + 'f' +'():\\n\\t' + exmpl\n",
    "              ,  file=open('/home/kirill/code2vec/cd2vec/python/real_validation/func'+str(i)+'.py', 'w'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}