DAY 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the necessary libraries\n",
    "import re\n",
    "import nltk \n",
    "import string\n",
    "import pandas as pd\n",
    "from scipy.sparse import hstack\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data from CSV file\n",
    "train = pd.read_csv('dataset_challenge_DAY2/train_set.csv')\n",
    "test = pd.read_csv('dataset_challenge_DAY2/new_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a variable 'y_train' containing the 'Directory code' column from the train dataset\n",
    "y_train = train['Directory code']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a text cleaning function that extracts a substring from the input text\n",
    "def take_some_text(text):\n",
    "    text = text[149:8000]  # Extract a substring from index 149 to 7999\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to remove stopwords and punctuation from a text column in a dataframe\n",
    "def remove_stopwords_and_punkt(df, text_column=\"text\", legal_stopwords=False):\n",
    "    # This function takes as input a dataframe, a string containing the text column name, and an optional list of legal words\n",
    "    # The function returns a dataframe with the specified text column cleaned from stopwords and punctuation\n",
    "    nltk.download('stopwords')  # Download the NLTK stopwords dataset\n",
    "    nltk.download('punkt')      # Download the NLTK punctuation dataset\n",
    "    \n",
    "    stop_words = set(stopwords.words('english'))  # Create a set of English stopwords\n",
    "    \n",
    "    if not legal_stopwords == False:\n",
    "        stop_words = stop_words.union(legal_stopwords)  # If legal stopwords are provided, add them to the set\n",
    "    \n",
    "    def remove_stop_words(text):\n",
    "        words = word_tokenize(text.lower())          # Tokenize the text and convert to lowercase\n",
    "        clean_words = [word for word in words if word not in stop_words]  # Remove stopwords\n",
    "        \n",
    "        return \" \".join(clean_words)  # Join the clean words back into a string\n",
    "    \n",
    "    df[text_column] = df[text_column].apply(remove_stop_words)  # Apply the remove_stop_words function to the text column\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to remove punctuation and convert text to lowercase in a dataframe column\n",
    "def remove_punct(df):\n",
    "    # Apply lambda function to remove punctuation and convert text to lowercase in the 'Text' column\n",
    "    df['Text'] = df['Text'].apply(lambda x: x.translate(str.maketrans('', '', string.punctuation)).lower())\n",
    "    \n",
    "    return df['Text']  # Return the cleaned 'Text' column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to remove digits from text in a dataframe column\n",
    "def remove_digit(df):\n",
    "    # Apply regex to remove digits from the 'Text' column\n",
    "    df['Text'] = df['Text'].apply(lambda x: re.sub(r'\\d+', '', x))\n",
    "    \n",
    "    return df['Text']  # Return the text column with digits removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to remove specified symbols from text in a dataframe column\n",
    "def remove_symbols(df):\n",
    "    symbols_to_remove = ['/', '$', '@', '\\\\', '\\.+' ]  # List of symbols to remove\n",
    "    pattern = '|'.join(re.escape(symbol) for symbol in symbols_to_remove)  # Create a regex pattern\n",
    "    \n",
    "    # Apply regex to remove specified symbols from the 'Text' column\n",
    "    df['Text'] = df['Text'].apply(lambda x: re.sub(pattern, '', x))  \n",
    "    \n",
    "    return df['Text']  # Return the text column with specified symbols removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/simonemaiorani/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/simonemaiorani/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "# Apply various text cleaning functions to the 'Text' column in the train set\n",
    "X_train = remove_stopwords_and_punkt(train, text_column=\"Text\")  # Remove stopwords and punctuation\n",
    "X_train['Text'] = train['Text'].apply(take_some_text)  # Extract a substring\n",
    "X_train['Text'] = remove_digit(train)  # Remove digits\n",
    "X_train['Text'] = remove_symbols(train)  # Remove specified symbols\n",
    "X_train['Text'] = remove_punct(train)  # Remove punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/simonemaiorani/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/simonemaiorani/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# Apply various text cleaning functions to the 'Text' column in the test set\n",
    "test = remove_stopwords_and_punkt(test, text_column=\"Text\")  # Remove stopwords and punctuation\n",
    "test['Text'] = test['Text'].apply(take_some_text)  # Extract a substring\n",
    "test['Text'] = remove_digit(test)  # Remove digits\n",
    "test['Text'] = remove_symbols(test)  # Remove specified symbols\n",
    "test['Text'] = remove_punct(test)  # Remove punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9941, 80310)\n"
     ]
    }
   ],
   "source": [
    "# Extract the 'Citations' columns from train and test\n",
    "cit_train = train[\"Citations\"]\n",
    "cit_test = test[\"Citations\"]\n",
    "\n",
    "# Create a TF-IDF vectorizer for citations\n",
    "tfidf_vectorizer_citazioni = TfidfVectorizer()\n",
    "\n",
    "# Transform 'Citations' for train and test datasets\n",
    "cit_tf_train = tfidf_vectorizer_citazioni.fit_transform(cit_train)\n",
    "cit_tf_test = tfidf_vectorizer_citazioni.transform(cit_test)\n",
    "\n",
    "# Create a TF-IDF vectorizer for text\n",
    "vectorizer = TfidfVectorizer()\n",
    "X_train_idf = vectorizer.fit_transform(X_train['Text'])\n",
    "\n",
    "# Print the shape of the TF-IDF matrix for text\n",
    "print(X_train_idf.shape)\n",
    "\n",
    "# Combine TF-IDF matrices for text and citations for train and test\n",
    "X_train_idf_final = hstack([X_train_idf, cit_tf_train])\n",
    "X_test_idf = vectorizer.transform(test['Text'])\n",
    "X_test_idf_final = hstack([X_test_idf, cit_tf_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 2.56357047\n",
      "Iteration 2, loss = 0.88582948\n",
      "Iteration 3, loss = 0.45714378\n",
      "Iteration 4, loss = 0.27514736\n",
      "Iteration 5, loss = 0.17607049\n",
      "Iteration 6, loss = 0.11367899\n",
      "Iteration 7, loss = 0.07532807\n",
      "Iteration 8, loss = 0.05174722\n",
      "Iteration 9, loss = 0.03807115\n",
      "Iteration 10, loss = 0.02897765\n",
      "Iteration 11, loss = 0.02231242\n",
      "Iteration 12, loss = 0.01808678\n",
      "Iteration 13, loss = 0.01447403\n",
      "Iteration 14, loss = 0.01286682\n",
      "Iteration 15, loss = 0.01124758\n",
      "Iteration 16, loss = 0.00942494\n",
      "Iteration 17, loss = 0.00888397\n",
      "Iteration 18, loss = 0.00770508\n",
      "Iteration 19, loss = 0.00731101\n",
      "Iteration 20, loss = 0.00657001\n",
      "Iteration 21, loss = 0.00692997\n",
      "Iteration 22, loss = 0.00628749\n",
      "Iteration 23, loss = 0.00563029\n",
      "Iteration 24, loss = 0.00522301\n",
      "Iteration 25, loss = 0.00487423\n",
      "Iteration 26, loss = 0.00518856\n",
      "Iteration 27, loss = 0.00533468\n",
      "Iteration 28, loss = 0.00491238\n",
      "Iteration 29, loss = 0.00466680\n",
      "Iteration 30, loss = 0.00429896\n",
      "Iteration 31, loss = 0.00414201\n",
      "Iteration 32, loss = 0.00392905\n",
      "Iteration 33, loss = 0.00396426\n",
      "Iteration 34, loss = 0.00361961\n",
      "Iteration 35, loss = 0.00328327\n",
      "Iteration 36, loss = 0.00326171\n",
      "Iteration 37, loss = 0.00314519\n",
      "Iteration 38, loss = 0.00311756\n",
      "Iteration 39, loss = 0.00314437\n",
      "Iteration 40, loss = 0.00291910\n",
      "Iteration 41, loss = 0.00297893\n",
      "Iteration 42, loss = 0.00317738\n",
      "Iteration 43, loss = 0.00331251\n",
      "Iteration 44, loss = 0.00349115\n",
      "Iteration 45, loss = 0.00253187\n",
      "Iteration 46, loss = 0.00245864\n",
      "Iteration 47, loss = 0.00242726\n",
      "Iteration 48, loss = 0.00233888\n",
      "Iteration 49, loss = 0.00235795\n",
      "Iteration 50, loss = 0.00227509\n",
      "Iteration 51, loss = 0.00224754\n",
      "Iteration 52, loss = 0.00223686\n",
      "Iteration 53, loss = 0.00219598\n",
      "Iteration 54, loss = 0.00215536\n",
      "Iteration 55, loss = 0.00215943\n",
      "Iteration 56, loss = 0.00209819\n",
      "Training loss did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MLPClassifier(activation=&#x27;identity&#x27;, hidden_layer_sizes=(150, 150),\n",
       "              learning_rate=&#x27;adaptive&#x27;, max_iter=150, random_state=42,\n",
       "              verbose=True)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MLPClassifier</label><div class=\"sk-toggleable__content\"><pre>MLPClassifier(activation=&#x27;identity&#x27;, hidden_layer_sizes=(150, 150),\n",
       "              learning_rate=&#x27;adaptive&#x27;, max_iter=150, random_state=42,\n",
       "              verbose=True)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "MLPClassifier(activation='identity', hidden_layer_sizes=(150, 150),\n",
       "              learning_rate='adaptive', max_iter=150, random_state=42,\n",
       "              verbose=True)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the MLP classifier with specified parameters\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(150, 150), max_iter=150, activation='identity', solver='adam', random_state=42, learning_rate='adaptive', verbose=True)\n",
    "\n",
    "# Fit the MLP model using the TF-IDF matrix for text and 'y_train'\n",
    "mlp.fit(X_train_idf_final, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions on the test set using the trained MLP model\n",
    "y_pred = mlp.predict(X_test_idf_final)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the predictions to a CSV file named 'predictions_NN.csv'\n",
    "import csv\n",
    "\n",
    "with open('predictions_NN.csv', 'w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    \n",
    "    # Write the header row with the column name 'label'\n",
    "    writer.writerow([\"label\"])\n",
    "    \n",
    "    # Write each prediction value from 'y_pred' into the 'label' column\n",
    "    for i in range(len(y_pred)):\n",
    "        writer.writerow([y_pred[i]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}