03_tfidf.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","import scipy\n",""]},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[],"source":["def to_list(words):\n","    if words == '[]':\n","        return []\n","    else:\n","        words = words.strip('\"')\n","        words = words.strip('[')\n","        words = words.strip(']')\n","        words = words.split(',')\n","        words = [w.lstrip(' ') for w in words]\n","        words = [w.strip(\"'\") for w in words]\n","        words = [w for w in words if w != '']\n","\n","        return words\n",""]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":["df_train = pd.read_csv('./dataset/2_with_cleaned_text/train.min.csv')\n","df_train['words'] = df_train['words'].apply(to_list)\n","\n","X_train = df_train['words'].copy()\n","X_train = X_train.apply(lambda words: ' '.join(words))\n","\n","df_test = pd.read_csv('./dataset/2_with_cleaned_text/test.min.csv')\n","df_test['words'] = df_test['words'].apply(to_list)\n","\n","X_test = df_test['words'].copy()\n","X_test = X_test.apply(lambda words: ' '.join(words))\n",""]},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":["from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n","\n","tfidf_vectorizer = TfidfVectorizer(lowercase=False, norm='l2', sublinear_tf=True)\n","\n","X_train_tfidf = tfidf_vectorizer.fit_transform(X_train)\n","X_test_tfidf = tfidf_vectorizer.transform(X_test)\n","\n","scipy.sparse.csr_matrix.sort_indices(X_train_tfidf)\n","scipy.sparse.csr_matrix.sort_indices(X_test_tfidf)\n",""]},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[],"source":["words_train = []\n","words_test = []\n","\n","for i in range(X_train.shape[0]):\n","    words_train.append(X_train_tfidf[i].toarray().reshape((6633,)))\n","for i in range(X_test.shape[0]):\n","    words_test.append(X_test_tfidf[i].toarray().reshape((6633,)))\n",""]},{"cell_type":"code","execution_count":6,"metadata":{},"outputs":[],"source":["words_train = pd.Series(words_train)\n","words_test = pd.Series(words_test)\n",""]},{"cell_type":"code","execution_count":7,"metadata":{},"outputs":[],"source":["del df_train['words']\n","del df_test['words']\n",""]},{"cell_type":"code","execution_count":8,"metadata":{},"outputs":[],"source":["df_train = pd.concat([df_train, words_train.to_frame(name='words')], axis=1)\n","df_test = pd.concat([df_test, words_test.to_frame(name='words')], axis=1)\n",""]},{"cell_type":"code","execution_count":9,"metadata":{},"outputs":[],"source":["df_train['category'] = df_train['category']\n","df_test['category'] = df_test['category']\n",""]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[],"source":["df_train.to_parquet('./dataset/3_with_tfidf/train.parquet', engine='pyarrow')\n","df_test.to_parquet('./dataset/3_with_tfidf/test.parquet', engine='pyarrow')\n","\n",""]}],"nbformat":4,"nbformat_minor":2,"metadata":{"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":3},"orig_nbformat":2}}