preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/share/anaconda2/lib/python2.7/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.309 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "# -*- encoding:utf-8 -*-\n",
    "\n",
    "from __future__ import print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "import os\n",
    "import cPickle as pkl\n",
    "from collections import Counter\n",
    "\n",
    "from nltk import sent_tokenize, word_tokenize\n",
    "from nltk.corpus import stopwords, wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from stanfordcorenlp import StanfordCoreNLP\n",
    "import jieba\n",
    "jieba.enable_parallel(8)\n",
    "\n",
    "import sys\n",
    "stdout = sys.stdout\n",
    "reload(sys)\n",
    "sys.stdout = stdout\n",
    "sys.setdefaultencoding('utf8')\n",
    "\n",
    "stopwords = stopwords.words('english')\n",
    "english_punctuations = [',','.',':',';','?','(',')','[',']','&','!','*','@','#','$','%']\n",
    "stopwords += english_punctuations\n",
    "\n",
    "# nlp = StanfordCoreNLP(r'/home/linxuming/nltk_data/stanford/stanford-corenlp-full-2016-10-31/', memory='8g')\n",
    "\n",
    "\n",
    "raw_data_path = 'data/UM-Corpus/data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_preprocess():\n",
    "\ttrain_en_corpus = []\n",
    "\ttrain_ch_corpus = []\n",
    "\ttest_en_corpus = []\n",
    "\ttest_ch_corpus = []\n",
    "\tcount = 0\n",
    "\n",
    "\tfor dirname in ('Bilingual', 'Testing'):\n",
    "\t\tif dirname == 'Bilingual':\n",
    "\t\t\tsub_dir_names = ['Education', 'Laws', 'Microblog', 'News', \n",
    "\t\t\t\t\t\t\t\t'Science', 'Spoken', 'Subtitles', 'Thesis']\n",
    "\t\t\tfor filename in sub_dir_names:\n",
    "\t\t\t\twith open(os.path.join(raw_data_path, dirname, filename, ''.join(['Bi-', filename, '.txt']))) as fr:\n",
    "\t\t\t\t\tfor i, line in enumerate(fr):\n",
    "\t\t\t\t\t\tline = line.strip().decode()\n",
    "\t\t\t\t\t\tcount += 1\n",
    "\t\t\t\t\t\tif i % 2 == 0:\n",
    "\t\t\t\t\t\t\ttrain_en_corpus.append(line)\n",
    "\t\t\t\t\t\telse:\n",
    "\t\t\t\t\t\t\ttrain_ch_corpus.append(line)\n",
    "\t\t\t\t\tprint('Finished {}'.format(count))\n",
    "\t\telse:\n",
    "\t\t\twith open(os.path.join(raw_data_path, dirname, 'Testing-Data.txt')) as fr:\n",
    "\t\t\t\tfor i, line in enumerate(fr):\n",
    "\t\t\t\t\tline = line.strip().decode()\n",
    "\t\t\t\t\tcount += 1\n",
    "\t\t\t\t\tif i % 2 == 0:\n",
    "\t\t\t\t\t\ttest_en_corpus.append(line)\n",
    "\t\t\t\t\telse:\n",
    "\t\t\t\t\t\ttest_ch_corpus.append(line)\n",
    "\t\t\t\tprint('Finished {}'.format(count))\n",
    "\ttrain_en_corpus = '\\n'.join(train_en_corpus)\n",
    "\ttrain_ch_corpus = '\\n'.join(train_ch_corpus)\n",
    "\ttest_en_corpus = '\\n'.join(test_en_corpus)\n",
    "\ttest_ch_corpus = '\\n'.join(test_ch_corpus)\n",
    "\treturn train_en_corpus, train_ch_corpus, test_en_corpus, test_ch_corpus\n",
    "\n",
    "\n",
    "def segment(corpus, tokenizer, savepath=None):\n",
    "\ttokenized_corpus = []\n",
    "\tcount = 0\n",
    "\ttokenized_corpus = ' '.join([_ for _ in tokenizer(corpus) if _.strip(' ')])\n",
    "\ttokenized_corpus = tokenized_corpus.split(' \\n ')\n",
    "\t# for sentence in corpus:\n",
    "\t# \tcount += 1\n",
    "\t# \ttokenized_corpus.append(' '.join(tokenizer(sentence)))\n",
    "\t# \tif count % 1000 == 0:\n",
    "\t# \t\tprint('Finished cutting {}'.format(count))\n",
    "\tif savepath:\n",
    "\t\twith open(savepath, 'w') as fw:\n",
    "\t\t\tpkl.dump(tokenized_corpus, fw)\n",
    "\treturn tokenized_corpus\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train_en_corpus, train_ch_corpus, test_en_corpus, test_ch_corpus = data_preprocess()\n",
    "train_en_corpus = segment(train_en_corpus, jieba.cut, 'data/preprocess/train_en_segment.pkl')\n",
    "train_ch_corpus = segment(train_ch_corpus, lambda k: iter(k.strip()), 'data/preprocess/train_ch_segment.pkl')\n",
    "test_en_corpus = segment(test_en_corpus, jieba.cut, 'data/preprocess/test_en_segment.pkl')\n",
    "test_ch_corpus = segment(test_ch_corpus, lambda k: iter(k.strip()), 'data/preprocess/test_ch_segment.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data/preprocess/train_en_segment.pkl') as fr:\n",
    "    train_en_corpus = pkl.load(fr)\n",
    "\n",
    "with open('data/preprocess/train_ch_segment.pkl') as fr:\n",
    "    train_ch_corpus = pkl.load(fr)\n",
    "\n",
    "with open('data/preprocess/test_en_segment.pkl') as fr:\n",
    "    test_en_corpus = pkl.load(fr)\n",
    "\n",
    "with open('data/preprocess/test_ch_segment.pkl') as fr:\n",
    "    test_ch_corpus = pkl.load(fr)\n",
    "\n",
    "def vocab(data, topK=None):\n",
    "    word2id = Counter()\n",
    "    for sentence in data:\n",
    "        for word in sentence.split():\n",
    "            word2id[word] += 1\n",
    "    word2id = word2id.most_common()\n",
    "    if topK:\n",
    "        word2id = word2id[:topK]\n",
    "    word2id, _ = zip(*word2id)\n",
    "    word2id = {word : i + 4 for i, word in enumerate(word2id)}\n",
    "    word2id['<PAD>'] = 0\n",
    "    word2id['<UNK>'] = 1\n",
    "    word2id['<S>'] = 2\n",
    "    word2id['</S>'] = 3\n",
    "    id2word = dict(zip(word2id.values(), word2id.keys()))\n",
    "    return word2id, id2word\n",
    "\n",
    "en_word2id, en_id2word = vocab(train_en_corpus)\n",
    "ch_word2id, ch_id2word = vocab(train_ch_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "537922\n",
      "9893\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "print(len(en_word2id))\n",
    "print(len(ch_word2id))\n",
    "\n",
    "max_src_len = 40\n",
    "max_tgt_len = 40\n",
    "src_embedding_size = 200\n",
    "tgt_embedding_size = 200\n",
    "train_size = 0.8\n",
    "\n",
    "\n",
    "def transform(data, word2id):\n",
    "    ret_data = []\n",
    "    for sentence in data:\n",
    "        ret_data.append([word2id.get(word, 1) for word in sentence.split()])\n",
    "    return ret_data\n",
    "\n",
    "\n",
    "train_en_corpus = transform(train_en_corpus, en_word2id)\n",
    "train_ch_corpus = transform(train_ch_corpus, ch_word2id)\n",
    "with open('data/preprocess/vocab_dict.pkl', 'w') as fw:\n",
    "    pkl.dump([en_word2id, en_id2word, ch_word2id, ch_id2word], fw)\n",
    "with open('data/preprocess/vocab_dict_and_corpus.pkl', 'w') as fw:\n",
    "    pkl.dump([en_word2id, en_id2word, ch_word2id, ch_id2word, train_en_corpus, train_ch_corpus, test_en_corpus, test_ch_corpus], fw)\n",
    "\n",
    "with open('data/preprocess/demo_vocab_dict_and_corpus.pkl', 'w') as fw:\n",
    "    pkl.dump([en_word2id, en_id2word, ch_word2id, ch_id2word, train_en_corpus[:100], train_ch_corpus[:100], test_en_corpus[:100], test_ch_corpus[:100]], fw)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def padding(data, max_len):\n",
    "    return tf.keras.preprocessing.sequence.pad_sequences(data, max_len, padding='post', truncating='post')\n",
    "\n",
    "with open('data/preprocess/vocab_dict.pkl') as fr:\n",
    "    en_word2id, en_id2word, ch_word2id, ch_id2word, \\\n",
    "    train_en_corpus, train_ch_corpus, test_en_corpus, test_ch_corpus = pkl.load(fr)\n",
    "\n",
    "    \n",
    "train_en_corpus = padding(train_en_corpus, max_src_len)\n",
    "train_ch_corpus = padding(train_ch_corpus, max_tgt_len)\n",
    "\n",
    "print(train_en_corpus.shape)\n",
    "print(train_ch_corpus.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_src_vocab_size = 60000\n",
    "max_tgt_vocab_size = 5000\n",
    "\n",
    "print(train_en_corpus[10])\n",
    "print([en_id2word[_] for _ in train_en_corpus[10]])\n",
    "print(' '.join([ch_id2word[_] for _ in train_ch_corpus[10]]))\n",
    "# train_en_corpus[train_en_corpus >= max_src_vocab_size] = 1\n",
    "# train_ch_corpus[train_ch_corpus >= max_tgt_vocab_size] = 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}