Data Dictionary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Dictionary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "A Web Scraping Dictionary Data Struxture For Collecting All Of The Features On Every Article HTML Page.\n",
    "\n",
    "''' \n",
    "\n",
    "master_dict = {\n",
    "'HTML_Content': [],\n",
    "'Full_Text': [],\n",
    "'Url': [],\n",
    "\n",
    "#### Article Information \n",
    "'Authors': [],\n",
    "'Publish_Date': [],\n",
    "'Article_Text': [],\n",
    "'Article_Text_Length': [],\n",
    "'Has_Top_Image': [],\n",
    "'Number_of_Movies': [],\n",
    "'Article_Is_Media_News': [],\n",
    "'Number_Of_Images':[],\n",
    "'Is_Valid_Body': [],\n",
    "    \n",
    "\n",
    "#### NLP Features\n",
    "'Setences_Text': [],\n",
    "'Number_Of_Sentences': [],\n",
    "'Lexicon_Count': [],\n",
    "    \n",
    "\n",
    "#### Readability Scores\n",
    "'Flesch_Reading_Ease_formula': [],\n",
    "'Flesch_Kincaid_Grade_Level': [],\n",
    "'FOG_Scale': [],\n",
    "'SMOG_Index': [],\n",
    "'ARI_Index': [],\n",
    "    \n",
    "\n",
    "#### Meta_Data\n",
    "'Title_Text': [],\n",
    "'Title_Tag_Length' : [],\n",
    "'Meta_Description' : [],\n",
    "'Meta_Description_Length': [],\n",
    "\n",
    "\n",
    "#### Extract_page_features\n",
    "'Body_Content_Links': [],\n",
    "'Number_Of_Links': [],\n",
    "'Links_To_Text_Ratio': [],\n",
    "\n",
    "\n",
    "#### Technical Page Metrics\n",
    "'Page_Size_In_Bytes': [],\n",
    "'Plain_Text_Size': [],\n",
    "'Plain_Text_Rate': [],\n",
    "'Encoding': [],\n",
    "'SSL': []\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target Variable "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The target variable for my data set is the <strong> total number of article shares. </strong>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Type of Machine Learning Problem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This type of machine learning problem is defined as a regression problem, I used a mixture of text and numerical data to predict the number of article shares for a given article (HTML) web page."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_df = pd.read_pickle('Pickled_Files/final_df.pkl')\n",
    "cleaned_df.dropna(axis= 0, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Evergreen_Score</th>\n",
       "      <th>Total_Shares</th>\n",
       "      <th>Word_Count</th>\n",
       "      <th>num_linking_domains</th>\n",
       "      <th>Article_Text_Length</th>\n",
       "      <th>Has_Top_Image</th>\n",
       "      <th>Number_of_Movies</th>\n",
       "      <th>Article_Is_Media_News</th>\n",
       "      <th>Number_Of_Images</th>\n",
       "      <th>Is_Valid_Body</th>\n",
       "      <th>...</th>\n",
       "      <th>PART</th>\n",
       "      <th>PRON</th>\n",
       "      <th>PROPN</th>\n",
       "      <th>PUNCT</th>\n",
       "      <th>SPACE</th>\n",
       "      <th>SYM</th>\n",
       "      <th>VERB</th>\n",
       "      <th>X</th>\n",
       "      <th>sentiment_polarity_score</th>\n",
       "      <th>subjectivity_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.54</td>\n",
       "      <td>8021</td>\n",
       "      <td>4767</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27301</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>508.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>603.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.281762</td>\n",
       "      <td>0.559670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.44</td>\n",
       "      <td>2569</td>\n",
       "      <td>1181</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6519</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.269613</td>\n",
       "      <td>0.476775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5.68</td>\n",
       "      <td>844</td>\n",
       "      <td>996</td>\n",
       "      <td>12.0</td>\n",
       "      <td>5916</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>149.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.175732</td>\n",
       "      <td>0.485097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.30</td>\n",
       "      <td>775</td>\n",
       "      <td>8996</td>\n",
       "      <td>3.0</td>\n",
       "      <td>39589</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>113</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>706.0</td>\n",
       "      <td>1202.0</td>\n",
       "      <td>241.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>779.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.195131</td>\n",
       "      <td>0.489689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.53</td>\n",
       "      <td>614</td>\n",
       "      <td>2226</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12657</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.252478</td>\n",
       "      <td>0.494659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.18</td>\n",
       "      <td>538</td>\n",
       "      <td>1489</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9050</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>113.0</td>\n",
       "      <td>223.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.327259</td>\n",
       "      <td>0.496381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.05</td>\n",
       "      <td>533</td>\n",
       "      <td>698</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4741</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>114.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>97.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.233514</td>\n",
       "      <td>0.553033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.43</td>\n",
       "      <td>514</td>\n",
       "      <td>2364</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13014</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>87</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>267.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.155326</td>\n",
       "      <td>0.456686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.07</td>\n",
       "      <td>486</td>\n",
       "      <td>1188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>176</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.93</td>\n",
       "      <td>389</td>\n",
       "      <td>1629</td>\n",
       "      <td>15.0</td>\n",
       "      <td>9606</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>194.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.116757</td>\n",
       "      <td>0.446654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.93</td>\n",
       "      <td>389</td>\n",
       "      <td>1629</td>\n",
       "      <td>15.0</td>\n",
       "      <td>9606</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>194.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.116757</td>\n",
       "      <td>0.446654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.68</td>\n",
       "      <td>374</td>\n",
       "      <td>2601</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14710</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>122.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>365.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.184251</td>\n",
       "      <td>0.449516</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 135 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Evergreen_Score  Total_Shares  Word_Count  num_linking_domains  \\\n",
       "0              1.54          8021        4767                  1.0   \n",
       "1              1.44          2569        1181                  2.0   \n",
       "2              5.68           844         996                 12.0   \n",
       "3              1.30           775        8996                  3.0   \n",
       "4              1.53           614        2226                  1.0   \n",
       "5              0.18           538        1489                  0.0   \n",
       "6              2.05           533         698                  0.0   \n",
       "7              2.43           514        2364                  0.0   \n",
       "8              2.07           486        1188                  0.0   \n",
       "9              2.93           389        1629                 15.0   \n",
       "10             2.93           389        1629                 15.0   \n",
       "11             0.68           374        2601                  1.0   \n",
       "\n",
       "    Article_Text_Length  Has_Top_Image  Number_of_Movies  \\\n",
       "0                 27301              1                 0   \n",
       "1                  6519              1                 0   \n",
       "2                  5916              1                 0   \n",
       "3                 39589              1                 0   \n",
       "4                 12657              1                 0   \n",
       "5                  9050              1                 0   \n",
       "6                  4741              1                 0   \n",
       "7                 13014              1                 0   \n",
       "8                   176              1                 0   \n",
       "9                  9606              1                 0   \n",
       "10                 9606              1                 0   \n",
       "11                14710              1                 0   \n",
       "\n",
       "    Article_Is_Media_News  Number_Of_Images  Is_Valid_Body  ...  PART  PRON  \\\n",
       "0                       0                42              1  ...   6.0   0.0   \n",
       "1                       0                24              1  ...   0.0   0.0   \n",
       "2                       0                12              1  ...   0.0   0.0   \n",
       "3                       0               113              1  ...  21.0   2.0   \n",
       "4                       0                28              1  ...   4.0   1.0   \n",
       "5                       0                10              1  ...   3.0   0.0   \n",
       "6                       0                10              1  ...   1.0   0.0   \n",
       "7                       0                87              1  ...   4.0   3.0   \n",
       "8                       0                15              0  ...   0.0   0.0   \n",
       "9                       0                20              1  ...   6.0   3.0   \n",
       "10                      0                20              1  ...   6.0   3.0   \n",
       "11                      0                10              1  ...   0.0   0.0   \n",
       "\n",
       "    PROPN   PUNCT  SPACE    SYM   VERB     X  sentiment_polarity_score  \\\n",
       "0   172.0   508.0  227.0   27.0  603.0   0.0                  0.281762   \n",
       "1    42.0   145.0   46.0    4.0  156.0   1.0                  0.269613   \n",
       "2    24.0   149.0   46.0    5.0  121.0   1.0                  0.175732   \n",
       "3   706.0  1202.0  241.0  118.0  779.0  40.0                  0.195131   \n",
       "4    85.0   286.0   66.0   16.0  276.0   0.0                  0.252478   \n",
       "5   113.0   223.0   66.0    5.0  192.0   0.0                  0.327259   \n",
       "6    42.0   114.0   23.0    1.0   97.0   2.0                  0.233514   \n",
       "7   163.0   267.0  102.0   16.0  290.0   1.0                  0.155326   \n",
       "8     2.0     7.0    0.0    0.0    7.0   0.0                  0.500000   \n",
       "9    57.0   224.0   44.0    1.0  194.0   0.0                  0.116757   \n",
       "10   57.0   224.0   44.0    1.0  194.0   0.0                  0.116757   \n",
       "11   87.0   265.0  122.0    5.0  365.0   0.0                  0.184251   \n",
       "\n",
       "    subjectivity_score  \n",
       "0             0.559670  \n",
       "1             0.476775  \n",
       "2             0.485097  \n",
       "3             0.489689  \n",
       "4             0.494659  \n",
       "5             0.496381  \n",
       "6             0.553033  \n",
       "7             0.456686  \n",
       "8             0.500000  \n",
       "9             0.446654  \n",
       "10            0.446654  \n",
       "11            0.449516  \n",
       "\n",
       "[12 rows x 135 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14469, 135)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}