tf-idf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from functions.training_functions import merge_articles_tqdfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def merge_essai(theme_doc_folder):\n",
    "    \"\"\" for tqdfs corpus \"\"\" \n",
    "    s = \"\"\n",
    "    for source in os.listdir(theme_doc_folder):\n",
    "        try :\n",
    "            for doc in os.listdir(theme_doc_folder + source):\n",
    "                with open(theme_doc_folder + source + \"/\" + doc ,'r') as f:\n",
    "                    lines = f.readlines()\n",
    "                    s += \"\".join(txt for txt in lines)\n",
    "                s += \" \"\n",
    "        except:\n",
    "            pass\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "theme_doc_folder = \"/home/ubuntu/summarization_query_oriented/data/TD-QFS/alz/alz/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s = merge_articles_tqdfs(theme_doc_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "def make_corpus(theme_doc_folder):\n",
    "    \"\"\" for tqdfs corpus \"\"\" \n",
    "    corpus = []\n",
    "    for source in os.listdir(theme_doc_folder):\n",
    "        try :\n",
    "            for doc in os.listdir(theme_doc_folder + source):\n",
    "                with open(theme_doc_folder + source + \"/\" + doc ,'r') as f:\n",
    "                    corpus.append(f.read().decode('utf-8', 'ignore'))\n",
    "        except:\n",
    "            pass\n",
    "    return corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpus = make_corpus(theme_doc_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "181"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u\"APOE e2 is relatively rare and may provide some protection against the disease. If Alzheimer's disease occurs in a person with this allele, it develops later in life than it would in someone with the APOE e4 gene.\\r\\nAPOE e3, the most common allele, is believed to play a neutral role in the diseaseneither decreasing nor increasing risk.\\r\\nAPOE e4 is present in about 25 to 30 percent of the population and in about 40 percent of all people with late-onset Alzheimer's. People who develop Alzheimer's are more likely to have an APOE e4 allele than people who do not develop the disease.\""
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<181x3639 sparse matrix of type '<type 'numpy.float64'>'\n",
       "\twith 14747 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer(min_df=1)\n",
    "vectorizer.fit_transform(corpus)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}