text-cleaner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "flying-judge",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-05-08T04:50:39.951604Z",
     "iopub.status.busy": "2021-05-08T04:50:39.950747Z",
     "iopub.status.idle": "2021-05-08T04:50:50.315845Z",
     "shell.execute_reply": "2021-05-08T04:50:50.314439Z"
    },
    "papermill": {
     "duration": 10.38825,
     "end_time": "2021-05-08T04:50:50.316046",
     "exception": false,
     "start_time": "2021-05-08T04:50:39.927796",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# !pip -q install ../input/pysastrawi/Sastrawi-1.0.1-py2.py3-none-any.whl \n",
    "\n",
    "## for data\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn .model_selection import StratifiedKFold, GroupKFold\n",
    "## for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "## for processing\n",
    "import string\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "## for bag-of-words\n",
    "from sklearn import feature_extraction, model_selection, naive_bayes, pipeline, manifold, preprocessing\n",
    "## for explainer\n",
    "from lime import lime_text\n",
    "## for word embedding\n",
    "import gensim\n",
    "\n",
    "import gensim.downloader as gensim_api\n",
    "## for deep learning\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import models, layers, preprocessing as kprocessing\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n",
    "## for bert language model\n",
    "import transformers\n",
    "from transformers import TFAutoModel, AutoTokenizer\n",
    "from transformers import RobertaTokenizer, TFRobertaModel\n",
    "\n",
    "# from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "unlikely-lighter",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T04:50:50.346930Z",
     "iopub.status.busy": "2021-05-08T04:50:50.346114Z",
     "iopub.status.idle": "2021-05-08T04:50:50.367816Z",
     "shell.execute_reply": "2021-05-08T04:50:50.367120Z"
    },
    "papermill": {
     "duration": 0.045186,
     "end_time": "2021-05-08T04:50:50.367955",
     "exception": false,
     "start_time": "2021-05-08T04:50:50.322769",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Preprocessing function helper\n",
    "# replace word that concatenate with other word\n",
    "def remove_concatenate_2_words(text):\n",
    "    list_words = ['khusus']\n",
    "    for w in list_words:\n",
    "        text = text.replace(w, '')\n",
    "    return text\n",
    "\n",
    "PUNCT_TO_REMOVE = string.punctuation\n",
    "def remove_punctuation(text):\n",
    "    return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))\n",
    "\n",
    "STOPWORDS_ID = set(stopwords.words('indonesian'))\n",
    "STOPWORDS_EN = set(stopwords.words('english'))\n",
    "def remove_stopwords(list_text):\n",
    "    text_not_in_ID = [word for word in list_text if word not in STOPWORDS_EN]\n",
    "    text = [word for word in text_not_in_ID if word not in STOPWORDS_ID]\n",
    "    return text\n",
    "\n",
    "# remove big number and split text that contains word and number\n",
    "def remove_big_number(list_text):\n",
    "    words = []\n",
    "    for w in list_text:\n",
    "        sub_w = re.split('(\\d+)',w)\n",
    "        for item in sub_w:\n",
    "            try:\n",
    "                tmp = int(item)\n",
    "                if tmp < 7000:\n",
    "                    if (tmp>1000) and (tmp % 100 == 0): # for even number\n",
    "                        words.append(str(tmp))\n",
    "                    elif (tmp<=1000) and (tmp>100) and (tmp % 10 == 0 ):\n",
    "                        words.append(str(tmp))\n",
    "                    elif (tmp<=100) and (tmp % 2 == 0):\n",
    "                        words.append(str(tmp))\n",
    "            except:\n",
    "                words.append(item)\n",
    "    return words\n",
    "\n",
    "def remove_zero_val(list_text):\n",
    "    return [w for w in list_text if w not in ['0']]\n",
    "\n",
    "def remove_common_words(list_text):\n",
    "    common_words = \"hari keren kere kw super baik jual jualan quality best free  kwalitas berkualitas kualitas bagus terbaik kembali dijamin beli gratis murah free diskon ongkir cek berkualitas original asli kualitas uang jaminan jamin terjamin buatan buat kirim wilayah luar kota jawa bali jakarta surabaya bulan month year day tahun hari harian anda your nikmat singapore malaysia indonesia vietnam thailand filipina bangkok jepang buy one get dapat dua two satu meriah kirim send pengiriman paket hemat uang kembali dapat guarantee buatan lokal dalam internasional karya termurah paling murah terbaik cheap murah biaya\".split(' ')\n",
    "    return [w for w in list_text if w not in common_words]\n",
    "\n",
    "def remove_strange_words(list_text):\n",
    "    strange_words = ['aaa', 'aaaa', 'aaaaa', 'abc', 'abcd', 'bb', 'bbb', 'bbbb', 'ccc', 'cccc', 'thn', 'th', 'bln']\n",
    "    return [w for w in list_text if w not in strange_words]\n",
    "\n",
    "def text_vectorizer(max_features, max_len, vocab):\n",
    "    # max_features: Maximum vocab size.\n",
    "    # max_len: Sequence length to pad the outputs to.\n",
    "    \n",
    "    text_dataset = tf.data.Dataset.from_tensor_slices(vocab)\n",
    "    \n",
    "    # Create the layer.\n",
    "    vectorize_layer = TextVectorization(\n",
    "        max_tokens = max_features,\n",
    "        output_mode = 'int',\n",
    "        output_sequence_length = max_len\n",
    "    )\n",
    "\n",
    "    vectorize_layer.adapt(text_dataset.batch(64))\n",
    "\n",
    "    model = tf.keras.models.Sequential()\n",
    "    model.add(tf.keras.Input(shape=(1,), dtype=tf.string))\n",
    "    model.add(vectorize_layer)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "parliamentary-bookmark",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T04:50:50.392711Z",
     "iopub.status.busy": "2021-05-08T04:50:50.391967Z",
     "iopub.status.idle": "2021-05-08T04:50:50.395475Z",
     "shell.execute_reply": "2021-05-08T04:50:50.394838Z"
    },
    "papermill": {
     "duration": 0.021429,
     "end_time": "2021-05-08T04:50:50.395610",
     "exception": false,
     "start_time": "2021-05-08T04:50:50.374181",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def utils_preprocess_text(text, flg_stemm=False, flg_lemm=True, lst_stopwords=None):\n",
    "    ## clean (convert to lowercase and remove punctuations and characters and then strip\n",
    "    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n",
    "            \n",
    "    ## Tokenize (convert from string to list)\n",
    "    lst_text = text.split()\n",
    "    ## remove Stopwords\n",
    "    if lst_stopwords is not None:\n",
    "        for stopwords in lst_stopwords:\n",
    "            lst_text = [word for word in lst_text if word not in \n",
    "                        stopwords]\n",
    "                \n",
    "    ## Stemming (remove -ing, -ly, ...)\n",
    "    if flg_stemm == True:\n",
    "        # english stemming\n",
    "        ps = nltk.stem.porter.PorterStemmer()\n",
    "        lst_text = [ps.stem(word) for word in lst_text]\n",
    "        \n",
    "        # indonesian stemming\n",
    "#         factory = StemmerFactory()\n",
    "#         id_stemmer = factory.create_stemmer()\n",
    "\n",
    "#         lst_text = [id_stemmer.stem(word) for word in lst_text]\n",
    "                \n",
    "    ## Lemmatisation (convert the word into root word)\n",
    "    if flg_lemm == True:\n",
    "        lem = nltk.stem.wordnet.WordNetLemmatizer()\n",
    "        lst_text = [lem.lemmatize(word) for word in lst_text]\n",
    "        \n",
    "    # remove_zero_val\n",
    "    lst_text = [w for w in lst_text if w not in ['0']]\n",
    "    \n",
    "    # remove strange words\n",
    "    strange_words = ['aaa', 'aaaa', 'aaaaa', 'abc', 'abcd', 'bb', 'bbb', 'bbbb', 'ccc', 'cccc', 'thn', 'th', 'bln']\n",
    "    lst_text = [w for w in lst_text if w not in strange_words]\n",
    "            \n",
    "    ## back to string from list\n",
    "    text = \" \".join(lst_text)\n",
    "    return text\n",
    "\n",
    "def string_escape(s, encoding='utf-8'):\n",
    "    return (\n",
    "        s.encode('latin1')  # To bytes, required by 'unicode-escape'\n",
    "        .decode('unicode-escape')  # Perform the actual octal-escaping decode\n",
    "        .encode('latin1')  # 1:1 mapping back to bytes\n",
    "        .decode(encoding)\n",
    "    )  # Decode original encoding\n",
    "\n",
    "lst_stopwords_en = nltk.corpus.stopwords.words(\"english\")\n",
    "lst_stopwords_id = nltk.corpus.stopwords.words(\"indonesian\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "growing-battery",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T04:50:50.411185Z",
     "iopub.status.busy": "2021-05-08T04:50:50.410593Z",
     "iopub.status.idle": "2021-05-08T04:50:50.643560Z",
     "shell.execute_reply": "2021-05-08T04:50:50.642965Z"
    },
    "papermill": {
     "duration": 0.241859,
     "end_time": "2021-05-08T04:50:50.643702",
     "exception": false,
     "start_time": "2021-05-08T04:50:50.401843",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('../input/shopee-product-matching/train.csv')\n",
    "df['label_group'], _ = df['label_group'].factorize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "biblical-round",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T04:50:50.666558Z",
     "iopub.status.busy": "2021-05-08T04:50:50.665898Z",
     "iopub.status.idle": "2021-05-08T04:50:50.838314Z",
     "shell.execute_reply": "2021-05-08T04:50:50.837713Z"
    },
    "papermill": {
     "duration": 0.188102,
     "end_time": "2021-05-08T04:50:50.838480",
     "exception": false,
     "start_time": "2021-05-08T04:50:50.650378",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n",
    "    \"\"\"\n",
    "    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n",
    "    \"\"\"\n",
    "    tokenizer.enable_truncation(max_length=maxlen)\n",
    "    tokenizer.enable_padding(max_length=maxlen)\n",
    "    all_ids = []\n",
    "    \n",
    "    for i in tqdm(range(0, len(texts), chunk_size)):\n",
    "        text_chunk = texts[i:i+chunk_size].tolist()\n",
    "        encs = tokenizer.encode_batch(text_chunk)\n",
    "        all_ids.extend([enc.ids for enc in encs])\n",
    "    \n",
    "    return np.array(all_ids)\n",
    "        \n",
    "def regular_encode(texts, tokenizer, maxlen=512):\n",
    "    enc_di = tokenizer.batch_encode_plus(\n",
    "        texts, \n",
    "#         add_special_tokens = True,\n",
    "        return_attention_mask = True,\n",
    "        return_token_type_ids=True,\n",
    "        pad_to_max_length=True,\n",
    "        max_length=maxlen\n",
    "        )\n",
    "    \n",
    "    return np.array(enc_di['input_ids']), np.array(enc_di['attention_mask'])\n",
    "\n",
    "MAX_LEN = 105\n",
    "MODEL = '../input/tfroberta-base-indonesian/roberta-base-indonesian-522M'\n",
    "tokenizer = RobertaTokenizer.from_pretrained(MODEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "understanding-pillow",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T04:50:50.872857Z",
     "iopub.status.busy": "2021-05-08T04:50:50.867629Z",
     "iopub.status.idle": "2021-05-08T04:51:05.312278Z",
     "shell.execute_reply": "2021-05-08T04:51:05.311645Z"
    },
    "papermill": {
     "duration": 14.467359,
     "end_time": "2021-05-08T04:51:05.312448",
     "exception": false,
     "start_time": "2021-05-08T04:50:50.845089",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_classes: 11014\n"
     ]
    }
   ],
   "source": [
    "# preprocess df_train title & phash\n",
    "df['tmp'] = df['title'].apply(lambda x: string_escape(x))\n",
    "df[\"tmp\"] = df[\"tmp\"].apply(lambda x: utils_preprocess_text(\n",
    "    x, flg_stemm=False, flg_lemm=False, lst_stopwords=None))\n",
    "\n",
    "# for BERT\n",
    "ids, att_mask = regular_encode(list(df[\"tmp\"].values), tokenizer, maxlen=MAX_LEN)\n",
    "df['input_ids'] = list(ids)\n",
    "df['att_mask'] = list(att_mask)\n",
    "del ids, att_mask\n",
    "\n",
    "df['tmp'] = df['title'].apply(lambda x: string_escape(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_concatenate_2_words(x))\n",
    "df['tmp'] = df['tmp'].str.lower()\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_punctuation(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: str(x).split())\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_stopwords(x))\n",
    "#     df['tmp'] = df['tmp'].apply(lambda x: remove_big_number(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_zero_val(x))\n",
    "#     df['tmp'] = df['tmp'].apply(lambda x: remove_common_words(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_strange_words(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: list(np.unique(x)))\n",
    "\n",
    "# for mlp input\n",
    "# title vocab\n",
    "words = list(df['tmp'])\n",
    "words = list(np.unique(np.concatenate(words)))\n",
    "# Text vectorizer\n",
    "model = text_vectorizer(max_features = 25000, max_len = 100, vocab = words)\n",
    "list_text = [' '.join(x) for x in df['tmp']]\n",
    "title_vec = model.predict(list_text)\n",
    "df['title_vec'] = list(title_vec)\n",
    "del model, list_text, title_vec, words\n",
    "\n",
    "n_classes = df['label_group'].nunique()\n",
    "print(f'n_classes: {n_classes}')\n",
    "\n",
    "df.to_parquet(f'/kaggle/working/train.parquet', engine='pyarrow')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 35.872472,
   "end_time": "2021-05-08T04:51:07.333694",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-05-08T04:50:31.461222",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}