end2end_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarization query oriented\n",
    "\n",
    " <hr style=\"border-color:#1d539d\"> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import bisect\n",
    "import collections\n",
    "import copy\n",
    "import gensim\n",
    "import json\n",
    "import keras\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import os\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import pandas as pd\n",
    "import random\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# paths to folders\n",
    "\n",
    "data_json = \"/home/ubuntu/summarization_query_oriented/data/json/patch_0/\"\n",
    "data_txt = \"/home/ubuntu/summarization_query_oriented/data/txt/\"\n",
    "model_folder = \"/home/ubuntu/summarization_query_oriented/models/\"\n",
    "nn_models_folder = \"/home/ubuntu/summarization_query_oriented/nn_models/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Doc2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model loaded\n"
     ]
    }
   ],
   "source": [
    "# parameters of doc2vec\n",
    "dm = 0\n",
    "min_count = 5\n",
    "window = 10\n",
    "size = 400\n",
    "sample = 1e-4\n",
    "negative = 5\n",
    "workers = 4\n",
    "epoch = 20\n",
    "\n",
    "# Initialize the model ( IMPORTANT )\n",
    "d2v_model = gensim.models.doc2vec.Doc2Vec(dm=dm,min_count=min_count, window=window, size=size, sample=sample, negative=negative, workers=workers,iter = epoch)\n",
    "\n",
    "# load model\n",
    "model_name =\"dm_\"+str(dm)+\"_mc_\"+str(min_count)+\"_w_\"+str(window)+\"_size_\"+str(size)+\"_neg_\"+str(negative)+\"_ep_\"+str(epoch)\n",
    "d2v_model = d2v_model.load(model_folder+model_name+\".d2v\")\n",
    "print(\"model loaded\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 1 : fully connected model\n",
    "\n",
    "\n",
    "* __Architecture__ : Fully connected model\n",
    "\n",
    "\n",
    "* __Input__ : a vector that is the concatenation of [ query , partial summary, candidate ]\n",
    "    * *query* : here the subtitle of a wikipedia page\n",
    "    * *partial summary* : here a part (eventually void) of the summary attached to this subtitle\n",
    "    * *candidate* : a random sentence\n",
    "\n",
    "\n",
    "* __Output__ : a score describing how much the candidate sentence is completing the partial summary w.r.t the query \n",
    "\n",
    "\n",
    "* __Training mode__ : we sample triplet from wikipedia data to build the training set, we label 1 if the candidate sentence is a sentence of the correct subsection that is not in the partial summary (by building), we label 0 otherwise\n",
    "\n",
    "\n",
    "* __Testing mode__ : Given a document and a query. The partial summary is initialized as the query, then we choose the sentence of the document that is not in the partial summary with the highest score and delete it from the document. We repeat it until we reach the length limit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data Processing \n",
    "\n",
    "Here we build functions to perform end—to-end data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# functions\n",
    "\n",
    "non_selected_keys = [\"title\", \"external links\",\"further reading\",\"references\",\"see also\"]\n",
    "\n",
    "def has_at_least_one_relevant_key(file_as_dict):\n",
    "    \n",
    "    for key in file_as_dict.keys():\n",
    "        b = True\n",
    "        for unwanted_key in non_selected_keys:\n",
    "            if unwanted_key in key.lower() :\n",
    "                b = False    \n",
    "        if b :\n",
    "            return True\n",
    "    return False\n",
    "        \n",
    "def has_irrelevant_content(file_as_dict):\n",
    "    # remove articles with mathematics of chemics\n",
    "    for key in file_as_dict.keys():\n",
    "        if \"{\\\\\" in file_as_dict[key]:\n",
    "            return True        \n",
    "\n",
    "    # check that there is at least one interesting key\n",
    "    if not has_at_least_one_relevant_key(file_as_dict):\n",
    "        return True\n",
    "\n",
    "    return False\n",
    "\n",
    "\n",
    "def relevant_articles(article_folder_path, min_size = 10000) : \n",
    "    \"\"\"\n",
    "    inputs :\n",
    "        - absolute path of the folder containing all the json articles\n",
    "        - min_size : retaining only file with at least size = min_size*10^-4 ko\n",
    "    output : \n",
    "        - article_names: nd array of the names of the relevant articles (absolute paths)\n",
    "        - article_weights : nd array normalized of the weights of each files\n",
    "    \"\"\"\n",
    "    all_names =  [f for f in listdir(article_folder_path)]\n",
    "    article_names = []\n",
    "    article_weights = []\n",
    "    for name in all_names:\n",
    "        article_weight = os.path.getsize(article_folder_path+name)\n",
    "        if article_weight > min_size:\n",
    "            # the size of the article meets the requirement\n",
    "            \n",
    "            with open(article_folder_path+name) as f :\n",
    "                file_as_dict = json.load(f) # get article as dict\n",
    "            \n",
    "            if not has_irrelevant_content(file_as_dict):\n",
    "                article_names.append(article_folder_path+name)\n",
    "                article_weights.append(article_weight)\n",
    "    \n",
    "    article_names = np.asarray(article_names)\n",
    "    article_weights = (np.asarray(article_weights) + 0.0) / np.sum(article_weights)\n",
    "        \n",
    "    return article_names, article_weights\n",
    "            \n",
    "def select_key(file_as_dict):\n",
    "    assert has_at_least_one_relevant_key(file_as_dict), \"the file has no relevant key\"\n",
    "\n",
    "    keys = file_as_dict.keys()\n",
    "    rand_idx = np.random.randint(0,len(keys))\n",
    "    selected_key = keys[rand_idx]\n",
    "    \n",
    "    if len(file_as_dict[selected_key].split(\".\"))<=2:\n",
    "        return select_key(file_as_dict)\n",
    "    \n",
    "    for unwanted_key in non_selected_keys :\n",
    "        if unwanted_key in selected_key.lower() :\n",
    "            return select_key(file_as_dict)\n",
    "        \n",
    "    return selected_key\n",
    "\n",
    "def create_triplets(d2v_model, article_names, article_weights, nb_triplets=20, triplets_per_file=5, neg_ratio=0.5, str_mode = False) :\n",
    "    \"\"\"\n",
    "    inputs :    \n",
    "        - d2v_model : paragraph vector model \n",
    "        - article_names : ndarray containing the names of the json files (absolute path !)\n",
    "        - article_weights: ndarray normalized of the weight of each files \n",
    "        - nb_triplets : nb of triplets to generate\n",
    "        - triplets_per_file : number of triplet built for each selected file\n",
    "        - neg_ratio : ratio of positives / negative examples. Negative examples are taken inside the article !\n",
    "        \n",
    "    output : \n",
    "        - triplets : nd_array of triplets of shape (nb_triplets+ , embed_dim)\n",
    "        - labels : nd_array of labels of shape (nb_triplets+ ,)\n",
    "\n",
    "    \"\"\"\n",
    "    triplets = []\n",
    "    labels = []\n",
    "    \n",
    "    assert nb_triplets>=triplets_per_file, \"you should have nb_triplets > triplets_per_file\"\n",
    "    \n",
    "    # nb of pos / neg triplets per file\n",
    "    neg_per_file = np.floor(triplets_per_file*neg_ratio) #number of negative triplets to generate given(query + partial summary)\n",
    "    assert neg_per_file >= 1, \"you have to increase your neg_ratio\"\n",
    "    \n",
    "    nb_files = nb_triplets / triplets_per_file\n",
    "    selected_files_array = np.random.choice(article_names, size=nb_files, p=article_weights, replace = False)\n",
    "    \n",
    "    for full_name in selected_files_array :\n",
    "        with open(full_name) as f :\n",
    "            file_as_dict = json.load(f)\n",
    "        \n",
    "        counter = 0\n",
    "        while counter < triplets_per_file :\n",
    "            \n",
    "            # select a key for positive examples\n",
    "            key_pos = select_key(file_as_dict)\n",
    "            \n",
    "            triplet = build_triplet(d2v_model, file_as_dict, key_pos, positive = True, str_mode = str_mode)\n",
    "            label = 1\n",
    "            \n",
    "            triplets.append(triplet)\n",
    "            labels.append(label)\n",
    "            counter += 1 \n",
    "            \n",
    "            if neg_ratio < 1 : \n",
    "                \n",
    "                if np.random.rand() < neg_ratio :\n",
    "                    \n",
    "                    triplet = build_triplet(d2v_model, file_as_dict, key_pos, positive = False, str_mode = str_mode)\n",
    "                    label = 0\n",
    "                    \n",
    "                    triplets.append(triplet)\n",
    "                    labels.append(label)\n",
    "                    counter += 1 \n",
    "\n",
    "            else :\n",
    "                \n",
    "                for n in range(int(np.floor(neg_ratio))):\n",
    "                    \n",
    "                    triplet = build_triplet(d2v_model, file_as_dict, key_pos, positive = False, str_mode = str_mode)\n",
    "                    label = 0\n",
    "                    \n",
    "                    triplets.append(triplet)\n",
    "                    labels.append(label)\n",
    "                    counter += 1 \n",
    "\n",
    "            \n",
    "    triplets = np.asarray(triplets)[:nb_triplets]\n",
    "    labels = np.asarray(labels)[:nb_triplets]\n",
    "    \n",
    "    return triplets, labels\n",
    "\n",
    "def build_triplet(d2v_model, file_as_dict, key_pos, positive = True, str_mode = False):\n",
    "\n",
    "    query_str = key_pos\n",
    "    query_prep = gensim.utils.simple_preprocess(query_str, deacc=True)\n",
    "    query_vector = d2v_model.infer_vector(query_prep)\n",
    "    \n",
    "    summary_str = file_as_dict[key_pos]\n",
    "    sentences = summary_str.split(\".\")\n",
    "    \n",
    "    partial_summary = []\n",
    "    candidates = []\n",
    "    \n",
    "    size_partial_summary = np.random.rand()\n",
    "    \n",
    "    for sentence in sentences: \n",
    "        if np.random.rand() < size_partial_summary :\n",
    "            partial_summary.append(sentence)\n",
    "        else :\n",
    "            candidates.append(sentence)\n",
    "    \n",
    "    candidate = \"\"\n",
    "    counter_candidate = 0\n",
    "    while (candidate == \"\" or partial_summary == \"\") and counter_candidate < 10:\n",
    "        counter_candidate += 1\n",
    "        \n",
    "        if positive : \n",
    "            if len(candidates) > 0:\n",
    "                random_candidate_index = np.random.randint(0,len(candidates))\n",
    "                candidate = candidates[random_candidate_index]\n",
    "            else :\n",
    "                random_candidate_index = np.random.randint(0,len(partial_summary))\n",
    "                candidate = partial_summary[random_candidate_index]\n",
    "                partial_summary[random_candidate_index] = \"\"\n",
    "\n",
    "\n",
    "            candidate_prep = gensim.utils.simple_preprocess(candidate, deacc=True)\n",
    "            candidate_vector = d2v_model.infer_vector(candidate_prep)\n",
    "\n",
    "        else :\n",
    "\n",
    "            key_neg = select_key(file_as_dict)\n",
    "            counter = 0\n",
    "\n",
    "            while key_neg == key_pos and counter<10 : # the counter is for the preproduction code \n",
    "                counter += 1\n",
    "                key_neg = select_key(file_as_dict)\n",
    "\n",
    "            summary_str = file_as_dict[key_neg]\n",
    "\n",
    "            sentences = summary_str.split('.')\n",
    "            random_candidate_index = np.random.randint(0,len(sentences))\n",
    "            candidate = sentences[random_candidate_index]\n",
    "            candidate_prep = gensim.utils.simple_preprocess(candidate, deacc=True)\n",
    "            candidate_vector = d2v_model.infer_vector(candidate_prep)\n",
    "        \n",
    "        partial_summary_str = \"\".join(partial_summary)\n",
    "        partial_summary_prep = gensim.utils.simple_preprocess(partial_summary_str, deacc=True)\n",
    "        partial_summary_vector = d2v_model.infer_vector(partial_summary_prep)\n",
    "    \n",
    "    if str_mode :\n",
    "        return query_str, partial_summary_str, candidate\n",
    "    else :\n",
    "        return np.hstack( [query_vector, partial_summary_vector, candidate_vector] )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data processing\n",
    "article_names, article_weights = relevant_articles(data_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEACAYAAABVtcpZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFyBJREFUeJzt3X+MVed95/H3B9t44oRQtoFhM5CEXXcIWLQO3Z2ktbJ7\nK1MIaQW0f6SkkbALq/0DtrbaVRUm/3jmj9XEK1Uh1S5IUVMYIqd07G4WpCLACN1WXTWBEFKcDIHR\nevkx43BJli5Zr9UE7O/+cZ+B0/GM517PnXvGfj4v6WrO/c7znPucK5vPPc+5Zx5FBGZmlqd5ZQ/A\nzMzK4xAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8tYQyEg6WlJL6XHU6m2SNIJSRclHZe0sNC+V9KI\npAuS1hfqayWdl3RJ0p7WH46ZmTVj2hCQ9AiwA/hXwKPAb0r6l8Bu4GRErAROAb2p/WrgM8AqYCOw\nV5LS7vYBOyKiG+iWtKHFx2NmZk1o5ExgFfCtiPhpRLwO/A3w28AmYDC1GQS2pO1NwKGIuBMRl4ER\noEfSUmBBRJxJ7Q4W+piZWQkaCYHvAZ9M0z8PAZ8GlgOdEVEDiIjrwJLUvgu4Vug/lmpdwGihPppq\nZmZWkvunaxARP5D0LPAi8CpwDnh9sqYtHpuZmc2yaUMAICL2A/sBJP0n6p/0a5I6I6KWpnpupOZj\n1M8Uxi1LtanqbyLJgWJm9jZEhKZvdU+j3w5anH5+CPgt4OvAEeDJ1OQJ4HDaPgJslTRf0grgYeB0\nmjK6JaknXSjeVugz2YH4EcEzzzxT+hjmysPvhd8Lvxdv/Xg7GjoTAP5S0j8DbgM7I+InaYpoSNJ2\n4Ar1bwQREcOShoDhQvvx0e0CDgAdwNGIOPa2Rm1mZi3R6HTQv5mkdhNYN0X7AWBgkvpZYE2TYzQz\ns1niO4bnuEqlUvYQ5gy/F/f4vbjH78XM6O3OI80mSTEXx2VmNpdJImbjwrCZmb07OQTMzDLmEDAz\ny5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTMzDLmEDAzy5hDwMwsYw4BM7OMOQTM\nzDLW6PKSfyDpe5LOS3ouLR25SNIJSRclHZe0sNC+V9KIpAuS1hfqa9M+LknaMxsHZGZmjZs2BCR9\nEPh9YG1E/CL11cg+C+wGTkbESuAU0Jvar6a+1OQqYCOwN60pDLAP2BER3UC3pA0tPh4zM2tCo9NB\n9wHvlXQ/8B5gDNgMDKbfDwJb0vYm4FBE3ImIy8AI0CNpKbAgIs6kdgcLfczMrATTrjEcEa9I+mPg\nKvAacCIiTkrqjIhaanNd0pLUpQv4u8IuxlLtDjBaqI+meku88sorvPjii63aXcOWLFnCxo0b2/66\nZmatMG0ISPo56p/6PwzcAp6X9Dlg4vqPLV0Psq+v7+52pVKZdh3RZ54Z4ODB0zzwwEdbOYxp/eM/\n/gXXr4/ygQ98oK2va2ZWrVapVqsz2se0IQCsA16OiJsAkr4B/CpQGz8bSFM9N1L7MWB5of+yVJuq\nPqliCDTi9dff4Gc/28bPfrarqX4z1dFxFK+HbGZlmPgBub+/v+l9NHJN4CrwCUkd6QLv48AwcAR4\nMrV5Ajicto8AW9M3iFYADwOnI+I6cEtST9rPtkIfMzMrQSPXBE5LegE4B9xOP78CLACGJG0HrlD/\nRhARMSxpiHpQ3AZ2xr2PyruAA0AHcDQijrX2cMzMrBmNTAcREf3AxPOMm9SniiZrPwAMTFI/C6xp\ncoxmZjZLfMewmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJm\nZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWsWlDQFK3pHOSvpN+3pL0lKRFkk5Iuijp\nuKSFhT69kkYkXZC0vlBfK+m8pEuS9szWQZmZWWOmDYGIuBQRH4uItcAvA/8P+AawGzgZESuBU0Av\ngKTV1JeaXAVsBPamNYUB9gE7IqIb6Ja0odUHZGZmjWt2Omgd8D8j4hqwGRhM9UFgS9reBByKiDsR\ncRkYAXokLQUWRMSZ1O5goY+ZmZWg2RD4HeDrabszImoAEXEdWJLqXcC1Qp+xVOsCRgv10VQzM7OS\nNLTQPICkB6h/yv98KsWEJhOfz0hfX9/d7UqlQqVSaeXuzcze8arVKtVqdUb7aDgEqM/vn42IH6fn\nNUmdEVFLUz03Un0MWF7otyzVpqpPqhgCZmb2ZhM/IPf39ze9j2amgz4L/Hnh+RHgybT9BHC4UN8q\nab6kFcDDwOk0ZXRLUk+6ULyt0MfMzErQ0JmApIeoXxT+94Xys8CQpO3AFerfCCIihiUNAcPAbWBn\nRIxPFe0CDgAdwNGIONaKgzAzs7enoRCIiNeAxRNqN6kHw2TtB4CBSepngTXND9PMzGaD7xg2M8uY\nQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy\n5hAwM8uYQ8DMLGMOATOzjDkEzMwy1lAISFoo6XlJFyR9X9LHJS2SdELSRUnHJS0stO+VNJLary/U\n10o6L+mSpD2zcUBmZta4Rs8Evkx9OchVwC8BPwB2AycjYiVwCugFkLSa+lKTq6gvTr83rSkMsA/Y\nERHdQLekDS07EjMza9q0ISDp/cAnI2I/QETciYhbwGZgMDUbBLak7U3AodTuMjAC9EhaCiyIiDOp\n3cFCHzMzK0EjZwIrgB9L2i/pO5K+khae74yIGkBEXAeWpPZdwLVC/7FU6wJGC/XRVDMzs5I0stD8\n/cBaYFdEfFvSl6hPBcWEdhOfz0hfX9/d7UqlQqVSaeXuzcze8arVKtVqdUb7aCQERoFrEfHt9Pwv\nqYdATVJnRNTSVM+N9PsxYHmh/7JUm6o+qWIImJnZm038gNzf39/0PqadDkpTPtckdafS48D3gSPA\nk6n2BHA4bR8BtkqaL2kF8DBwOk0Z3ZLUky4Ubyv0MTOzEjRyJgDwFPCcpAeAl4HfA+4DhiRtB65Q\n/0YQETEsaQgYBm4DOyNifKpoF3AA6KD+baNjrToQMzNrXkMhEBF/D/zrSX61bor2A8DAJPWzwJpm\nBmhmZrPHdwybmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBm\nljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhoKAUmXJf29pHOSTqfaIkknJF2UdFzSwkL7Xkkj\nki5IWl+or5V0XtIlSXtafzhmZtaMRs8E3gAqEfGxiOhJtd3AyYhYCZwCegEkraa+ytgqYCOwNy0n\nCbAP2BER3UC3pA0tOg4zM3sbGg0BTdJ2MzCYtgeBLWl7E3AoIu5ExGVgBOhJi9EviIgzqd3BQh8z\nMytBoyEQwIuSzkj6d6nWmRahJy0ivyTVu4Brhb5jqdYFjBbqo6lmZmYlaXSh+cci4oeSFgMnJF2k\nHgxFE5+bmdkc1+hC8z9MP38k6b8DPUBNUmdE1NJUz43UfAxYXui+LNWmqk+qr6/v7nalUqFSqTQy\nVDOzbFSrVarV6oz2oYi3/gAv6SFgXkS8Kum9wAmgH3gcuBkRz0r6PLAoInanC8PPAR+nPt3zIvAL\nERGSvgk8BZwB/gr4k4g4NslrxnTjmmj79l3s378a2NVUv5nq6FjM1avDLF68uK2va2Y2kSQiQtO3\nvKeRM4FO4BuSIrV/LiJOSPo2MCRpO3CF+jeCiIhhSUPAMHAb2Fn4F30XcADoAI5OFgBmZtY+04ZA\nRPwv4NFJ6jeBdVP0GQAGJqmfBdY0P0wzM5sNvmPYzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEz\ns4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLWMMh\nIGmepO9IOpKeL5J0QtJFScclLSy07ZU0IumCpPWF+lpJ5yVdkrSntYdiZmbNauZM4GnqS0aO2w2c\njIiVwCmgFyCtMfwZYBWwEdgraXzNy33AjojoBrolbZjh+M3MbAYaCgFJy4BPA39aKG8GBtP2ILAl\nbW8CDkXEnYi4DIwAPZKWAgsi4kxqd7DQx8zMStDomcCXgD8ColDrjIgaQERcB5akehdwrdBuLNW6\ngNFCfTTVzMysJNOGgKTfAGoR8V1Ab9E03uJ3ZmY2B93fQJvHgE2SPg28B1gg6WvAdUmdEVFLUz03\nUvsxYHmh/7JUm6o+qb6+vrvblUqFSqXSwFDNzPJRrVapVqsz2ociGv8AL+nfAv8xIjZJ+s/A/46I\nZyV9HlgUEbvTheHngI9Tn+55EfiFiAhJ3wSeAs4AfwX8SUQcm+R1oplxAWzfvov9+1cDu5rqN1Md\nHYu5enWYxYsXt/V1zcwmkkREvNWMzZs0ciYwlS8CQ5K2A1eofyOIiBiWNET9m0S3gZ2Ff9F3AQeA\nDuDoZAFgZmbt01QIRMRfA3+dtm8C66ZoNwAMTFI/C6xpfphmZjYbfMewmVnGHAJmZhlzCJiZZcwh\nYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlz\nCJiZZcwhYGaWsUYWmn9Q0rcknZP0kqRnUn2RpBOSLko6LmlhoU+vpBFJFyStL9TXSjov6ZKkPbNz\nSGZm1qhpQyAifgr8WkR8DHgU2CipB9gNnIyIlcApoBcgrTH8GWAVsBHYK2l8zct9wI6I6Aa6JW1o\n9QGZmVnjGpoOiojX0uaD1JekDGAzMJjqg8CWtL0JOBQRdyLiMjAC9EhaCiyIiDOp3cFCHzMzK0FD\nISBpnqRzwHXgxfQPeWdE1AAi4jqwJDXvAq4Vuo+lWhcwWqiPppqZmZWkoYXmI+IN4GOS3g98Q9Ij\n1M8G/kmzVg6sr6/v7nalUqFSqbRy92Zm73jVapVqtTqjfTQUAuMi4ieSqsCngJqkzoiopameG6nZ\nGLC80G1Zqk1Vn1QxBMzM7M0mfkDu7+9veh+NfDvoA+Pf/JH0HuDXgQvAEeDJ1OwJ4HDaPgJslTRf\n0grgYeB0mjK6JaknXSjeVuhjZmYlaORM4J8Dg5LmUQ+Nv4iIo5K+CQxJ2g5cof6NICJiWNIQMAzc\nBnZGxPhU0S7gANABHI2IYy09GjMza8q0IRARLwFrJ6nfBNZN0WcAGJikfhZY0/wwzcxsNviOYTOz\njDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DM\nLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGONLC+5TNIpSd+X9JKkp1J9kaQTki5KOj6+BGX6Xa+kEUkX\nJK0v1NdKOi/pkqQ9s3NIZmbWqEbOBO4AfxgRjwC/AuyS9FFgN3AyIlYCp4BeAEmrqS81uQrYCOxN\nawoD7AN2REQ30C1pQ0uPxszMmjJtCETE9Yj4btp+lfoi88uAzcBgajYIbEnbm4BDEXEnIi4DI0CP\npKXAgog4k9odLPQxM7MSNHVNQNJHgEeBbwKdEVGDelAAS1KzLuBaodtYqnUBo4X6aKqZmVlJpl1o\nfpyk9wEvAE9HxKuSYkKTic9npK+v7+52pVKhUqm0cvdmZu941WqVarU6o300FAKS7qceAF+LiMOp\nXJPUGRG1NNVzI9XHgOWF7stSbar6pIohYGZmbzbxA3J/f3/T+2h0OujPgOGI+HKhdgR4Mm0/ARwu\n1LdKmi9pBfAwcDpNGd2S1JMuFG8r9DEzsxJMeyYg6THgc8BLks5Rn/b5AvAsMCRpO3CF+jeCiIhh\nSUPAMHAb2BkR41NFu4ADQAdwNCKOtfZwzMysGdOGQET8D+C+KX69boo+A8DAJPWzwJpmBmhmZrPH\ndwybmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwi0wCOP/DKS2vpYuvQj\nZR+2mb0LNPxXRG1qP/rRNVr8R1SnVatp+kZmZtPwmYCZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYc\nAmZmGZs2BCR9VVJN0vlCbZGkE5IuSjouaWHhd72SRiRdkLS+UF8r6bykS5L2tP5QzMysWY2cCewH\nNkyo7QZORsRK4BTQCyBpNfVlJlcBG4G9aT1hgH3AjojoBrolTdynmZm12bQhEBF/C/zDhPJmYDBt\nDwJb0vYm4FBE3ImIy8AI0CNpKbAgIs6kdgcLfczMrCRv95rAkoioAUTEdWBJqncB1wrtxlKtCxgt\n1EdTzczMStSqPxvR8r+Z0NfXd3e7UqlQqVRa/RJmZu9o1WqVarU6o3283RCoSeqMiFqa6rmR6mPA\n8kK7Zak2VX1KxRAwM7M3m/gBub+/v+l9NDodpPQYdwR4Mm0/ARwu1LdKmi9pBfAwcDpNGd2S1JMu\nFG8r9DEzs5JMeyYg6etABfh5SVeBZ4AvAs9L2g5cof6NICJiWNIQMAzcBnZGxPhU0S7gANABHI2I\nY609FDMza9a0IRARvzvFr9ZN0X4AGJikfhZY09TozMxsVvmOYTOzjDkEzMwy5hAwM8uYQ8DMLGMO\nATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy1qo/JW1t9yD3Fm1rn87OD3P9+uW2v66ZzQ6H\nwDvWT5mFZRymVau1P3jMbPZ4OsjMLGMOATOzjDkEzMwy1vYQkPQpST+QdEnS59v9+mZmdk9bQ0DS\nPOC/ABuAR4DPSvpoO8fwzlMtewAT1L+V1O7H0qUfmfGC2u8mfi/u8XsxM+0+E+gBRiLiSkTcBg4B\nm9s8hneYatkDmGD8W0ntfdRqV/w/e4Hfi3v8XsxMu0OgC7hWeD6aamZmVoJ3zX0CDz74AB0d+5g/\nv73r17/22v9t6+vl60H6+/vp7+9v2yvOm/cQb7zxWtteb5xvyLN2UkT7bjiS9AmgLyI+lZ7vBiIi\nnp3Qrv13QZmZvQtERFN3dLY7BO4DLgKPAz8ETgOfjYgLbRuEmZnd1dbpoIh4XdJ/AE5Qvx7xVQeA\nmVl52nomYGZmc8ucumPYN5LVSVom6ZSk70t6SdJTZY+pbJLmSfqOpCNlj6VMkhZKel7ShfTfx8fL\nHlNZJP2BpO9JOi/pOUnzyx5Tu0j6qqSapPOF2iJJJyRdlHRc0sJG9jVnQsA3kv0Td4A/jIhHgF8B\ndmX8Xox7GhguexBzwJeBoxGxCvglIMvpVEkfBH4fWBsRv0h9antruaNqq/3U/60s2g2cjIiVwCmg\nt5EdzZkQwDeS3RUR1yPiu2n7Ver/o2d7P4WkZcCngT8teyxlkvR+4JMRsR8gIu5ExE9KHlaZ7gPe\nK+l+4CHglZLH0zYR8bfAP0wobwYG0/YgsKWRfc2lEPCNZJOQ9BHgUeBb5Y6kVF8C/ogyFlCYW1YA\nP5a0P02NfUXSe8oeVBki4hXgj4GrwBjwfyLiZLmjKt2SiKhB/YMksKSRTnMpBGwCSe8DXgCeTmcE\n2ZH0G0AtnRkpPXJ1P7AW+K8RsRZ4jfoUQHYk/Rz1T74fBj4IvE/S75Y7qjmnoQ9NcykExoAPFZ4v\nS7UspVPcF4CvRcThssdToseATZJeBv4c+DVJB0seU1lGgWsR8e30/AXqoZCjdcDLEXEzIl4H/hvw\nqyWPqWw1SZ0AkpYCNxrpNJdC4AzwsKQPp6v8W4GcvwnyZ8BwRHy57IGUKSK+EBEfioh/Qf2/iVMR\nsa3scZUhnepfk9SdSo+T78Xyq8AnJHWovtj24+R3kXzimfER4Mm0/QTQ0IfHOfO3g3wj2T2SHgM+\nB7wk6Rz107ovRER7/zCSzUVPAc9JegB4Gfi9ksdTiog4LekF4BxwO/38Srmjah9JXwcqwM9Lugo8\nA3wReF7SduAK8JmG9uWbxczM8jWXpoPMzKzNHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZ\nZcwhYGaWsf8PYkjHymUFcB0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc1f6cbc810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.hist(article_weights*10000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ILLUSTRATION BLOCK : Here you can play with the triplet maker and see what gives triplet labelisation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : New York State Route 98\n",
      "\n",
      "partial summary : S Most of the route passes through rural, undeveloped areas; however, in southern Genesee County, it traverses more urbanized areas that lie in and around Batavia\n",
      "Most of modern NY 98 north of Batavia was originally designated as New York State Route 74 in the mid-1920s It was extended south to US 20 (now US 20A) in Varysburg in the late 1920s Other changes, including a pair of substantial realignments, have also occurred in the time since the route was originally assigned\n",
      "\n",
      "candidate :  The northern end is at an interchange with the Lake Ontario State Parkway in the town of Carlton in Orleans County, near the southern shore of Lake Ontario\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : New York State Route 98\n",
      "\n",
      "partial summary :  The southern terminus of the route is at an intersection with U Most of the route passes through rural, undeveloped areas; however, in southern Genesee County, it traverses more urbanized areas that lie in and around Batavia\n",
      "Most of modern NY 98 north of Batavia was originally designated as New York State Route 74 in the mid-1920s The extension into Salamanca was eliminated in the 1960s, while the route was extended north to meet the Lake Ontario Parkway in the 1970s\n",
      "\n",
      "candidate :  In the 1930 renumbering of state highways in New York, NY 74 became part of NY 98, a new route that extended from Salamanca in the south to Carlton in the north via Great Valley, Franklinville, Arcade, and Varysburg\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : New York State Route 98 Route description\n",
      "\n",
      "partial summary : \n",
      "\n",
      "candidate :  The only non-state-maintained section of the route lies within the city of Batavia, where the route is locally maintained from the southern city line to the south end of the overlap with NY 33\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : New York State Route 98 Route description\n",
      "\n",
      "partial summary : Most of NY 98 is maintained by the New York State Department of Transportation\n",
      "\n",
      "candidate : 0 km) across farmlands and through forests to a point once known as Five Corners, where NY 98 separates from NY 78 and turns northward once more\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : New York State Route 98 Route description Java to Attica\n",
      "\n",
      "partial summary :  NY 98 crosses the railroad at a grade crossing just south of the hamlet before meeting NY 77 and NY 78 at a four-way junction in the center of the community NY 77 continues north from the intersection on Cattaraugus Road while NY 98 joins NY 78 eastward on Main Street, traversing the ARA at another grade crossing on their way out of the hamlet5 miles (2 The route follows the water body along the Sheldon–Orangeville town line to a junction with US 20A just inside of the town of Orangeville NY 98 joins US 20A here, following the latter northwestward to Varysburg, a small creekside hamlet in Sheldon The route enters the village shortly afterward, following High Street through the lightly developed southern half of Attica One block later, NY 98 passes under the Norfolk Southern Railway's Buffalo Line and joins NY 238 at Main Street The junction of Main and High Streets marks the start of Attica's business district, which follows NY 98 and NY 238 north for two blocks along Market Street to Buffalo Street Here, NY 238 leaves NY 98 to continue toward Darien while NY 98 remains on Market Street to the northern village and town lines, which also serves as the Wyoming–Genesee county line, though a small northernmost portion of the village of Attica, which NY 98 returns slightly, is in the town of Alexander\n",
      "\n",
      "candidate : 4 km) into Sheldon, NY 98 passes the southern end of Tonawanda Creek, the source of which is located along the west side of NY 98\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : New York State Route 98 Route description Java to Attica\n",
      "\n",
      "partial summary : 0 km) across farmlands and through forests to a point once known as Five Corners, where NY 98 separates from NY 78 and turns northward once more The route continues across a mixture of cultivated fields and forests to North Java, a small hamlet located at the junction between NY 98 and Perry and Wethersfield Roads\n",
      "South of the village, the route comes within view of the Attica Correctional Facility, the site of the 1971 Attica Prison riot The route enters the village shortly afterward, following High Street through the lightly developed southern half of Attica\n",
      "\n",
      "candidate :  When US 219 was extended into New York c\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : New York State Route 98 History\n",
      "\n",
      "partial summary :  In the mid-1920s, what is now NY 98 between Batavia and modern NY 18 in Carlton was designated as NY 74 The route was extended southward to Varysburg over the former routing of legislative Route 19 in the late 1920s In the 1930 renumbering of state highways in New York, NY 74 became part of NY 98, a new route that extended from Salamanca in the south to Carlton in the north via Great Valley, Franklinville, Arcade, and Varysburg When US 219 was extended into New York c\n",
      "Originally, NY 98 followed a more easterly alignment through the towns of Elba and Barre At the northern edge of the village of Elba, NY 98 left its modern alignment and followed modern CR 24 in Genesee County and CR 98 in Orleans County before rejoining its current alignment near Barre Center Farther south, NY 98 was originally routed on East Arcade Road (now Wyoming County's CR 11) between Arcade and North Java NY 98 was realigned c 1933 to follow Curriers, Chaffee, and Cattaraugus Roads between Arcade and Java Center and overlap with NY 78 between Java Center and its original alignment to the east The portion of the route between Arcade and Java Center was also designated as part of NY 77 NY 98 was relocated back onto its original alignment via East Arcade Road c NY 77 was reextended to the vicinity of Arcade c 1961, following the modern routing of NY 98 between Arcade and Java Center\n",
      "When the Lake Ontario State Parkway was constructed in the 1970s, NY 98 was extended north along NY 18 and Point Breeze Road to meet the parkway at a new interchange near the Carlton hamlet of The Bridges Although the short extension of the route was signed as a state highway, ownership and maintenance of it was not transferred from Orleans County to the state of New York until April 1, 1989 The segment of Point Breeze Road from the parkway to the Lake Ontario shoreline remains county maintained as CR 37\n",
      "\n",
      "candidate :  The overlap between US 219 and NY 98 remained intact until c\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : New York State Route 98 History\n",
      "\n",
      "partial summary : The segment of modern NY 98 between Varysburg and Batavia was originally designated as part of Route 19, an unsigned legislative route, by the New York State Legislature in 1908 In the mid-1920s, what is now NY 98 between Batavia and modern NY 18 in Carlton was designated as NY 74 The route was extended southward to Varysburg over the former routing of legislative Route 19 in the late 1920s In the 1930 renumbering of state highways in New York, NY 74 became part of NY 98, a new route that extended from Salamanca in the south to Carlton in the north via Great Valley, Franklinville, Arcade, and Varysburg When US 219 was extended into New York c 1935, it was routed on NY 98 between Salamanca and Great Valley 1962 when NY 98 was truncated on its southern end to Great Valley At the northern edge of the village of Elba, NY 98 left its modern alignment and followed modern CR 24 in Genesee County and CR 98 in Orleans County before rejoining its current alignment near Barre Center Farther south, NY 98 was originally routed on East Arcade Road (now Wyoming County's CR 11) between Arcade and North Java NY 98 was realigned c NY 77 was reextended to the vicinity of Arcade c NY 98 was realigned in the early 1960s to follow NY 77 between Arcade and Java Center, resulting in the truncation of NY 77 to Java Center\n",
      "\n",
      "candidate :  The route heads northeast across Cattaraugus County, paralleling Forks Creek through northeast Great Valley and the northwestern corner of Humphrey to the town of Franklinville\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Gadsden Purchase Southern route for the Transcontinental Railroad James Gadsden and California\n",
      "\n",
      "partial summary : \n",
      "\n",
      "candidate : ”\n",
      "When the secession proposal failed, Gadsden, working with his cousin Isaac Edward Holmes, a lawyer in San Francisco since 1851, and the California state senator Thomas Jefferson Green, attempted to divide California in two\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Gadsden Purchase Southern route for the Transcontinental Railroad James Gadsden and California\n",
      "\n",
      "partial summary : Gadsden supported nullification in 1831 They proposed that the southern half of the state allow slavery\n",
      "\n",
      "candidate :  The new American Arizona Territory also included most of the lands acquired in the Gadsden Purchase\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Gadsden Purchase Treaty of Guadalupe Hidalgo Isthmus of Tehuantepec\n",
      "\n",
      "partial summary : During negotiations of the treaty, Americans had failed to secure the right of transit across the 125-mile (201 km)-wide Isthmus of Tehuantepec in southern Mexico The idea of building a railroad here had been considered for a long time In 1842 Mexican President Antonio López de Santa Anna sold the rights to build a railroad or canal across the isthmus The deal included land grants 300 miles (480 km)-wide along the right-of-way for future colonization and development In 1847 a British bank bought the rights, raising US fears of British colonization in the hemisphere, in violation of the precepts of the Monroe Doctrine United States interest in the right-of-way was increased by the 1848 finding of gold in the Sierra Nevada, the finding the impetus for the California Gold Rush\n",
      "The Memphis commercial convention of 1849 recommended that the United States pursue the trans-isthmus route, since it appeared unlikely that a transcontinental railroad would be built anytime soon Also showing interest was Peter A Hargous of New York who ran an import-export business between New York and Vera Cruz Hargous purchased the rights to the route for $25,000 ($700,000 today), but realized that the grant had little value unless it was supported by the Mexican and American governments Hughes reported to Secretary of State John M Clayton that a railroad across the isthmus was a “feasible and practical” idea Clayton then instructed Robert P Letcher, the minister to Mexico, to negotiate a treaty to protect Hargous' rights This treaty, however, was never finalized\n",
      "The Clayton–Bulwer Treaty between the United States and Great Britain, which guaranteed the neutrality of any such canal, was finalized in April 1850 Mexican negotiators refused the treaty because it would eliminate Mexico's ability to play the US and Britain against each other They eliminated the right of the United States to unilaterally intervene militarily The United States Senate approved the treaty in early 1851, but the Mexican Congress refused to accept the treaty Judah P Benjamin and a committee of New Orleans businessmen joined with Hargous and secured a charter from the Louisiana legislature to create the Tehuantepec Railroad Company The new company sold stock and sent survey teams to Mexico Hargous started to acquire land even after the Mexican legislature rejected the treaty, a move that led to the Mexicans canceling Hargous' contract to use the right of way Hargous put his losses at $5 million ($142,000,000 today) and asked the United States government to intervene President Fillmore refused to do so G Sloo and Associates in New York for $600,000 ($17,000,000 today) In March 1853 Sloo contracted with a British company to build a railroad and sought an exclusive contract from the new Franklin Pierce Administration to deliver mail from New York to San Francisco However, Sloo soon defaulted on bank loans and the contract was sold back to Hargous\n",
      "\n",
      "candidate :  The United States' proposal gave Mexicans a 20% discount on shipping, guaranteed Mexican rights in the zone, allowed the United States to send in military if necessary, and gave the United States most-favored-nation status for Mexican cargo fees\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Gadsden Purchase Treaty of Guadalupe Hidalgo Isthmus of Tehuantepec\n",
      "\n",
      "partial summary : During negotiations of the treaty, Americans had failed to secure the right of transit across the 125-mile (201 km)-wide Isthmus of Tehuantepec in southern Mexico The idea of building a railroad here had been considered for a long time In 1842 Mexican President Antonio López de Santa Anna sold the rights to build a railroad or canal across the isthmus The deal included land grants 300 miles (480 km)-wide along the right-of-way for future colonization and development In 1847 a British bank bought the rights, raising US fears of British colonization in the hemisphere, in violation of the precepts of the Monroe Doctrine United States interest in the right-of-way was increased by the 1848 finding of gold in the Sierra Nevada, the finding the impetus for the California Gold Rush\n",
      "The Memphis commercial convention of 1849 recommended that the United States pursue the trans-isthmus route, since it appeared unlikely that a transcontinental railroad would be built anytime soon Interests in Louisiana were especially adamant about this option, as they believed that any transcontinental railroad would divert commercial traffic away from the Mississippi and New Orleans, and they at least wanted to secure a southern route Also showing interest was Peter A Hargous of New York who ran an import-export business between New York and Vera Cruz Hughes reported to Secretary of State John M Clayton then instructed Robert P The United States' proposal gave Mexicans a 20% discount on shipping, guaranteed Mexican rights in the zone, allowed the United States to send in military if necessary, and gave the United States most-favored-nation status for Mexican cargo fees This treaty, however, was never finalized\n",
      "The Clayton–Bulwer Treaty between the United States and Great Britain, which guaranteed the neutrality of any such canal, was finalized in April 1850 Mexican negotiators refused the treaty because it would eliminate Mexico's ability to play the US and Britain against each other They eliminated the right of the United States to unilaterally intervene militarily The United States Senate approved the treaty in early 1851, but the Mexican Congress refused to accept the treaty\n",
      "In the meantime, Hargous proceeded as if the treaty would be approved eventually Judah P Benjamin and a committee of New Orleans businessmen joined with Hargous and secured a charter from the Louisiana legislature to create the Tehuantepec Railroad Company The new company sold stock and sent survey teams to Mexico Hargous put his losses at $5 million ($142,000,000 today) and asked the United States government to intervene\n",
      "Mexico sold the canal franchise, without the land grants, to A G Sloo and Associates in New York for $600,000 ($17,000,000 today) In March 1853 Sloo contracted with a British company to build a railroad and sought an exclusive contract from the new Franklin Pierce Administration to deliver mail from New York to San Francisco However, Sloo soon defaulted on bank loans and the contract was sold back to Hargous\n",
      "\n",
      "candidate :  The new stability brought miners and ranchers\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Gadsden Purchase Growth of the region after 1854 Army control\n",
      "\n",
      "partial summary : The residents of the area gained full US citizenship and slowly assimilated into American life over the next half-century The US Army took control of the purchase lands in 1854 but not until 1856 were troops stationed in the troubled region In June 1857 it established Fort Buchanan south of the Gila at the head of the Sonoita Creek Valley The fort protected the area until it was evacuated and destroyed in July 1861 By the late 1850s, mining camps and military posts had not only transformed the Arizona countryside; they had also generated new trade linkages to the state of Sonora, Mexico Magdalena, Sonora, became a supply center for Tubac, wheat from nearby Cucurpe fed the troops at Fort Buchanan, and the town of Santa Cruz sustained the Mowry mines, just miles to the north\n",
      "\n",
      "candidate :  The principal threat to the peace and security of settlers and travelers in the area were raids by Apache Indians\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Gadsden Purchase Growth of the region after 1854 Army control\n",
      "\n",
      "partial summary :  The US Army took control of the purchase lands in 1854 but not until 1856 were troops stationed in the troubled region In June 1857 it established Fort Buchanan south of the Gila at the head of the Sonoita Creek Valley The fort protected the area until it was evacuated and destroyed in July 1861 The new stability brought miners and ranchers By the late 1850s, mining camps and military posts had not only transformed the Arizona countryside; they had also generated new trade linkages to the state of Sonora, Mexico\n",
      "\n",
      "candidate :  He was concerned that the increasing railroad construction in the North was shifting trade in lumber, farm goods, and manufacturing goods from the traditional north-south route based on the Ohio and Mississippi rivers to an east-west axis that would bypass the South\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Gadsden Purchase Bibliography\n",
      "\n",
      "partial summary : \n",
      "\n",
      "candidate :  (1969 2nd\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Gadsden Purchase Bibliography\n",
      "\n",
      "partial summary : Garber, Paul Neff The Gadsden Treaty (1923) 222 pp; the standard diplomatic history\n",
      "luger, Richard]] (1947) SBN 684-10424-5\n",
      "Nichols, Roy Franklin (2007) ISBN 0-307-26520-X\n",
      "Roberson, Jere W The Western Historical Quarterly, Vol 5, No 163–186 JSTOR Journal of the Southwest Volume: 46 online edition\n",
      "\n",
      "candidate :  The new American Arizona Territory also included most of the lands acquired in the Gadsden Purchase\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe Activities Intercultural exchange\n",
      "\n",
      "partial summary :  AEGEE supports mobility and encourages young people to cross borders and make new friends, to break stereotypes and strengthen tolerance, respect and solidarity\n",
      "\n",
      "candidate : AEGEE creates a space for members of different backgrounds to meet and learn about cultural differences, in order to foster mutual understanding\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe Activities Intercultural exchange\n",
      "\n",
      "partial summary :  AEGEE supports mobility and encourages young people to cross borders and make new friends, to break stereotypes and strengthen tolerance, respect and solidarity\n",
      "\n",
      "candidate : AEGEE-Europe is involved in bigger platforms and works with many other international institutions and organizations, in order to bring a students perspective to each one of the processes\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe Activities Personal development and non-formal education\n",
      "\n",
      "partial summary :  By participating in trainings courses, workshops, and active involvement in the organisation, our members improve their competences (such as leadership skills, communication skills and team management) which will be useful for their future lives\n",
      "\n",
      "candidate : AEGEE provides learning opportunities for the development of its members through non-formal education and informal learning\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe Activities Personal development and non-formal education\n",
      "\n",
      "partial summary : AEGEE provides learning opportunities for the development of its members through non-formal education and informal learning By participating in trainings courses, workshops, and active involvement in the organisation, our members improve their competences (such as leadership skills, communication skills and team management) which will be useful for their future lives\n",
      "\n",
      "candidate :  It promotes an equal, democratic and unified Europe, open to all across national borders\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe History\n",
      "\n",
      "partial summary :  It had the aim to overcome the paralysis of the European integration process at the time\n",
      "AEGEE launches its EU and Europe project\n",
      "AEGEE becomes formal member of the European Movement International (EMI)\n",
      "2008 \n",
      "AEGEE launches Y Vote 2009 - European Youth Choice to encourage voting for young people\n",
      "The project Sustaining our Future becomes National winner of the European Charlemagne Youth Prize Award for the Netherlands\n",
      "2009 \n",
      "AEGEE wins 1st place in the European Charlemagne Youth Prize for its project YOUrope Needs YOU!!!\n",
      "Y Vote 2009 becomes National winner of the European Charlemagne Youth Prize award for Greece\n",
      "The AEGEEan the online magazine of AEGEE-Europe is launched\n",
      "AEGEE carries out a big advocacy campaign with other youth NGOs and the European Youth Forum for the new Education, youth and Sport programme of the European Commission (Erasmus +)\n",
      "AEGEE is elected for the Board of the European Youth Forum (YFJ)\n",
      "\n",
      "candidate :  Based on its own experience and the tense relations between Turkish and Greek AEGEE members in those first days, AEGEE focused its activities on peace-building between two countries as an organisation acting for peace and stability, proving at the AEGEE scale that cooperation was possible and that the dialogue between people can happen much more easily that the dialogue between the political level\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe History\n",
      "\n",
      "partial summary : 1985 \n",
      "The association was born the 16 April 1985 as a result of the EGEE 1 conference (États Généraux des Étudiants de l’Europe), when it held its first event in Paris: an assembly of students from Paris, Leiden, London, Madrid, Milan and Munich, organised by founding president Franck Biancheri in cooperation with ﬁve Grandes Écoles in Paris It had the aim to overcome the paralysis of the European integration process at the time Franck Biancheri led the whole process and soon all students involved wanted to turn the EGEE conference into an organisation being the platform for young Europeans\n",
      "Due to the uniqueness of the idea at that time, EGEE was able to stimulate many students to establish antennae in their cities\n",
      "1986 \n",
      "Three EGEE working groups were formed: Sponsoring, Traineeships and Language Study\n",
      "By the start of the academic year, EGEE has 26 branches and 6,000 members\n",
      "EGEE organizes in Heidelberg, a conference on relations between the Far East and Europe\n",
      "EGEE organizes in Paris a conference on the pharmaceutical industry in Europe\n",
      "EGEE organizes, together with the newspaper Le Monde the Night of the 7 European cities, broadcasting political debates about the future of Europe between students and high level politicians in seven different cities\n",
      "1987 \n",
      "EGEE persuades French president François Mitterrand to support funding for the Erasmus programme, a student exchange program financed by the European Commission\n",
      "The Summer University Project was launched It allows young Europeans to have an intense European experience in a city in Europe for 2 weeks, providing them with a real intercultural experience and opening them the doors to Europe\n",
      "1989 \n",
      "After the fall of the Berlin Wall, the Agora in Salerno opens up AEGEE to local antennae outside the European Community, making it one of the first European organisations to expand beyond the old Iron Curtain\n",
      "1990 \n",
      "Les Anciens d'AEGEE-Europe is founded during the EGEE VI meeting in Paris\n",
      "1991 \n",
      "The first European School 1 training is organized in Madrid\n",
      "1992 \n",
      "İstanbul has joined the network as the first Turkish antenna\n",
      "AEGEE gets consultative status at the OSCE (Organization for Security and Cooperation in Europe)\n",
      "1993 \n",
      "AEGEE gets official NGO status in the Council of Europe (CoE)\n",
      "Georg von der Gablentz, receives the Honorary Membership of AEGEE-Europe for his work with the \"East-West WG\" and his initiatives to open AEGEE to Central and Eastern Europe\n",
      "Jeroen Hoogerwerf, receives the Honorary Membership of AEGEE-Europe for his extense contribution to the Network through many of his positions\n",
      "Michael Merker, receives the Honorary Membership of AEGEE-Europe for his involvement in the expansion of AEGEE to the East and founding the first local over the Iron Curtain\n",
      "10 anniversary of AEGEE´s foundation\n",
      "AEGEE strongly advocates against nuclear tests in Europe\n",
      "Zsuzsa Kigyos, receives the Honorary Membership of AEGEE-Europe for her role in establishing AEGEE in Central Eastern Europe, and being the first female president of AEGEE-Europe\n",
      "1996 \n",
      "More than 1000 students are actively involved in the conference series Find Your Way explaining what students can do in the emerging civil society in Central and Eastern Europe Following this, in 2001 an antenna is created in Mağusa\n",
      "David Stulik becomes the first Young European of the Year, an award given by the Heinz-Schwarzkopf Foundation for his active engagement in AEGEE in building a better Europe through volunteer projects\n",
      "AEGEE organizes first case study trip to Moldova and through Former Yugoslavia\n",
      "AEGEE become candidate member organization of the European Youth Forum\n",
      "1998 \n",
      "AEGEE organizes a Case Study trip to Transilvania\n",
      "AEGEE becomes full Member Organization of the European Youth Forum\n",
      "1999 \n",
      "Foundation of the AEGEE-Academy for training and human resources at Agora-Barcelona, prompted by preparations for the European School in Gießen\n",
      "AEGEE organizes its Peace Academy dealing with the topics of conflict in Spain, Northern Ireland, Kosovo, South Africa and Bosnia and Herzegovina\n",
      "2000 \n",
      "\"Education for Democracy\", a new scholarship programme helping students from war-shattered Kosovo to study at universities abroad\n",
      "AEGEE launches EURECA, a contribution to the design of a new educational programme for the enlarging European Union\n",
      "2001 \n",
      "AEGEE organises several major projects focusing on peace and stability in southeastern Europe and the Mediterranean region under the Education for Democracy project\n",
      "Oana Daciana, wins the Young European of the Year award by the Heinz-Schwarzkopf Foundation for her engagement in bringing Europe closer for young people from the Eastern European countries With the establishment of AEGEE locals in Turkey in the 1990s, AEGEE realized the necessity of establishing a dialogue between the neighbors in conflict Based on its own experience and the tense relations between Turkish and Greek AEGEE members in those first days, AEGEE focused its activities on peace-building between two countries as an organisation acting for peace and stability, proving at the AEGEE scale that cooperation was possible and that the dialogue between people can happen much more easily that the dialogue between the political level\n",
      "Launch of AEGEE-Television by AEGEE-Eindhoven\n",
      "Michiel Van Hees, receives the Honorary Membership award of AEGEE-Europe for his long contribution to AEGEE and his role in the projects Find Your Way and Europe and the Euro\n",
      "2003 \n",
      "AEGEE launches Discussing Bologna Process project getting ready for the start of the harmonization of higher education in Europe and bringing a strong student perspective to it\n",
      "AEGEE's first study trip to the Caucasus\n",
      "AEGEE organises the first international student conference in the buffer zone on Cyprus\n",
      "AEGEE launches its EU and Europe project\n",
      "2004 \n",
      "AEGEE starts a dialogue with the new neighbours of the enlarged EU in its project Europe & EU, conducted by the International Politics Working Group\n",
      "AEGEE opens its first local branch in Georgia, in Tiblisi\n",
      "Anan Jahangirl, wins the Young European of the Year award, given by the Heinz-Schwarzkopf Foundation for his commitments of bringing the European Integration process to young people in Azerbaijan\n",
      "2005 \n",
      "AEGEE celebrates its 20th anniversary in Prague and publishes a special edition of its \"Key to Europe\"\n",
      "Olivier Genking, receives the Honorary Membership of AEGEE-Europe for his involvement in the EU & Europe project, his role in establishing many new locals, creating new projects and constant support to the Network\n",
      "2006 \n",
      "AEGEE starts a campaign to include the European Citizens' Initiative (ECI) in the Treaty of Lissabon, and conducts several Election Observation Missions as part of its Flagship Project Take Control! - ways to democracy in Europe\n",
      "AEGEE launches its Youth Globalization project, getting young people closer the global frameworks of youth and cooperation and development\n",
      "AEGEE launches its The BRIDGE- Connecting Mobility and Disability project, dealing with the mobility problem of disabled youth\n",
      "AEGEE launches its Take Control project, organizing a series of training events, conferences, political campaigns, educational and media activities all intended to increase the involvement and engagement of young people in the (European) political process\n",
      "Gunnar Erth,receives the Honorary Membership of AEGEE-Europe, for his contribution to the development of AEGEE’s annual publications\n",
      "2007 \n",
      "AEGEE organised the simulation Model European Union in the premises of the 'European Parliament in Strasbourg\n",
      "AEGEE chairs the IFISO Presidency and organises a stakeholder meeting with the European Commission The Leadership Summerschool (LSS) is born\n",
      "AEGEE plays an instrumental role in enlarging the Erasmus programme to Turkey, thanks to the work of 7 different generations in AEGEE-Ankara\n",
      "2008 \n",
      "AEGEE launches Y Vote 2009 - European Youth Choice to encourage voting for young people\n",
      "The project Sustaining our Future becomes National winner of the European Charlemagne Youth Prize Award for the Netherlands\n",
      "Kamala Schutze, receives the Honorary Membership of AEGEE-Europe, for her long term devotion to the Summer University project of AEGEE\n",
      "2010 \n",
      "AEGEE organises The UN Millennium Development Goals Conference within the framework of the project Beyond Europe Perspectives for Tomorrow's World\n",
      "AEGEE becomes partner organisation of BEST (Board of European Students of Technology)\n",
      "2011 \n",
      "AEGEE initiates the Eastern Partnership Project to give youth a voice and to strengthen civil society in the wider neighbourhood countries\n",
      "AEGEE is one of the founding organizations of the European Year of Volunteering Alliance 2011\n",
      "AEGEE re-defines the concept of European Boards’ Meeting into a thematic conference\n",
      "2012 \n",
      "AEGEE launches its project Europe on Track, sending ambassadors by train all over Europe to gather the opinion of the European Youth on the future they want for the continent\n",
      "AEGEE launches its project Health 4 Youth aiming at providing information about health to students to allow them to take informed decisions about their lifestyle\n",
      "AEGEE launches its EuroArab project, creating bridges between young people from Europe and the Arab countries\n",
      "AEGEE and YEU (Youth for Exchange and Understanding) launch the COY (Certification of Competences of Youth Workers) Project, funded by the European Commission\n",
      "Burcu Bercemen, receives the Honorary Membership of AEGEE-Europe for the devotion to the AEGEE Network and her extraordinary performance with the Turkish-Greek Civic Dialogue project\n",
      "2013 \n",
      "AEGEE introduces a new Visual Identity with the current logo\n",
      "\n",
      "candidate : AEGEE gathers the opinions of European students and represents them towards stakeholders through campaigns, lobby actions, conferences and membership in bigger advocacy platforms\n",
      "--------------------------------------------------\n",
      "label =  1\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe Activities Forum for discussion\n",
      "\n",
      "partial summary : \n",
      "\n",
      "candidate :  This exchange of ideas happens in all our activities but a special place for them is our Working groups, conferences and General Assemblies (EPM & Agora)\n",
      "--------------------------------------------------\n",
      "label =  0\n",
      "\n",
      "query : Association des États Généraux des Étudiants de l'Europe Activities Forum for discussion\n",
      "\n",
      "partial summary :  This exchange of ideas happens in all our activities but a special place for them is our Working groups, conferences and General Assemblies (EPM & Agora)\n",
      "\n",
      "candidate :  AEGEE supports mobility and encourages young people to cross borders and make new friends, to break stereotypes and strengthen tolerance, respect and solidarity\n"
     ]
    }
   ],
   "source": [
    "triplets, labels = create_triplets(d2v_model, article_names, article_weights, nb_triplets=25, triplets_per_file=8, neg_ratio=1, str_mode = True)\n",
    "\n",
    "for i in range(len(labels)):\n",
    "    print 50*'-'\n",
    "    print \"label = \", labels[i]\n",
    "    print \"\\nquery :\", triplets[i][0]\n",
    "    print \"\\npartial summary :\", triplets[i][1]\n",
    "    print \"\\ncandidate :\", triplets[i][2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "\n",
    "fc_model_name = nn_models_folder + time.strftime(\"%Y_%m_%d_\") +'_fc_model.h5' # replace it with hour of training\n",
    "\n",
    "fc_model = Sequential()\n",
    "\n",
    "fc_model.add(Dense(120, input_dim=1200))\n",
    "fc_model.add(Activation('sigmoid'))\n",
    "fc_model.add(Dropout(0.5))\n",
    "\n",
    "fc_model.add(Dense(12))\n",
    "fc_model.add(Activation('sigmoid'))\n",
    "fc_model.add(Dropout(0.5))\n",
    "\n",
    "fc_model.add(Dense(1))\n",
    "fc_model.add(Activation('sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fc_model.compile(loss=\"binary_crossentropy\", optimizer='sgd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Training (we use training per batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10\n",
      "20\n",
      "30\n",
      "40\n",
      "50\n",
      "60\n",
      "70\n",
      "80\n",
      "90\n",
      "100\n",
      "110\n",
      "120\n",
      "130\n",
      "140\n",
      "150\n",
      "160\n",
      "170\n",
      "180\n",
      "190\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128\n",
    "for i in range(200):\n",
    "    if i%10 == 0 : \n",
    "        print(i)\n",
    "    triplets, labels = create_triplets(d2v_model, article_names, article_weights, nb_triplets=batch_size, triplets_per_file=16, neg_ratio=1, str_mode = False)\n",
    "    fc_model.train_on_batch(triplets, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128/128 [==============================] - 0s\n"
     ]
    }
   ],
   "source": [
    "triplets_tests , labels_tests = create_triplets(d2v_model, article_names, article_weights, nb_triplets=128, triplets_per_file=16, neg_ratio=1, str_mode = False)\n",
    "labels_predicted = fc_model.predict(triplets_tests , batch_size=batch_size, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_val = np.load(\"/home/ubuntu/summarization_query_oriented/valset/X_val.npy\")\n",
    "y_val = np.load(\"/home/ubuntu/summarization_query_oriented/valset/y_val.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 9600/10000 [===========================>..] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "loss_and_metrics = fc_model.evaluate(X_val,y_val, batch_size = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.float64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(loss_and_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128/128 [==============================] - 0s\n",
      "0.690462529659\n"
     ]
    }
   ],
   "source": [
    "loss_and_metrics = fc_model.evaluate(triplets_tests, labels_tests, batch_size=batch_size)\n",
    "print(loss_and_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n",
       "       0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,\n",
       "       1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n",
       "       0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0,\n",
       "       1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,\n",
       "       0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "fc_model.save(fc_model_name)  # creates a HDF5 file 'my_model.h5'\n",
    "\n",
    "del fc_model  # deletes the existing model\n",
    "# returns a compiled model\n",
    "# identical to the previous one\n",
    "fc_model = load_model(fc_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fc_model = load_model(fc_model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### General info on the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "dense_1 (Dense)                  (None, 120)           144120      dense_input_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 120)           0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 120)           0           activation_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 12)            1452        dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_2 (Activation)        (None, 12)            0           dense_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 12)            0           activation_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "dense_3 (Dense)                  (None, 1)             13          dropout_2[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "activation_3 (Activation)        (None, 1)             0           dense_3[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 145585\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "[{'class_name': 'Dense', 'config': {'W_constraint': None, 'b_constraint': None, 'name': u'dense_1', 'output_dim': 120, 'activity_regularizer': None, 'trainable': True, 'init': 'glorot_uniform', 'bias': True, 'input_dtype': u'float32', 'input_dim': 1200, 'b_regularizer': None, 'W_regularizer': None, 'activation': 'linear', 'batch_input_shape': (None, 1200)}}, {'class_name': 'Activation', 'config': {'activation': 'sigmoid', 'trainable': True, 'name': u'activation_1'}}, {'class_name': 'Dropout', 'config': {'p': 0.5, 'trainable': True, 'name': u'dropout_1'}}, {'class_name': 'Dense', 'config': {'W_constraint': None, 'b_constraint': None, 'name': u'dense_2', 'activity_regularizer': None, 'trainable': True, 'init': 'glorot_uniform', 'bias': True, 'input_dim': None, 'b_regularizer': None, 'W_regularizer': None, 'activation': 'linear', 'output_dim': 12}}, {'class_name': 'Activation', 'config': {'activation': 'sigmoid', 'trainable': True, 'name': u'activation_2'}}, {'class_name': 'Dropout', 'config': {'p': 0.5, 'trainable': True, 'name': u'dropout_2'}}, {'class_name': 'Dense', 'config': {'W_constraint': None, 'b_constraint': None, 'name': u'dense_3', 'activity_regularizer': None, 'trainable': True, 'init': 'glorot_uniform', 'bias': True, 'input_dim': None, 'b_regularizer': None, 'W_regularizer': None, 'activation': 'linear', 'output_dim': 1}}, {'class_name': 'Activation', 'config': {'activation': 'sigmoid', 'trainable': True, 'name': u'activation_3'}}]\n"
     ]
    }
   ],
   "source": [
    "print(fc_model.summary())\n",
    "print(fc_model.get_config())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Testing the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def summarize(text, query, d2v_model, nn_model, limit = 250):\n",
    "\n",
    "    query_prep = gensim.utils.simple_preprocess(query, deacc=True)\n",
    "    query_vector = d2v_model.infer_vector(query_prep)\n",
    "    \n",
    "    summary  = \"\"\n",
    "    summary_vector = d2v_model.infer_vector([\"\"])\n",
    "    summary_idx = []\n",
    "    \n",
    "    sentences = text.split('.')\n",
    "    sentences = np.asarray(sentences)\n",
    "    \n",
    "    remaining_sentences = copy.copy(sentences)\n",
    "    \n",
    "    size = 0\n",
    "    counter = 0\n",
    "    while size < limit and len(remaining_sentences)>0 :\n",
    "        counter = counter+1\n",
    "        scores = []\n",
    "        for sentence in remaining_sentences :\n",
    "            \n",
    "            \n",
    "            sentence_prep = gensim.utils.simple_preprocess(sentence, deacc=True)\n",
    "            sentence_vector = d2v_model.infer_vector(sentence_prep)\n",
    "\n",
    "            nn_input = np.hstack([query_vector, summary_vector, sentence_vector])\n",
    "            nn_input = np.asarray([nn_input]) # weird but it is important to do it\n",
    "            score = nn_model.predict(nn_input) \n",
    "            scores.append(score)\n",
    "        #print(scores)\n",
    "        max_idx_rem = int(np.argmax(scores))\n",
    "        idx_selected_sentence = np.arange(len(sentences))[sentences == remaining_sentences[max_idx_rem]]\n",
    "        idx_selected_sentence = int(idx_selected_sentence[0])\n",
    "        size += len(remaining_sentences[max_idx_rem].split())\n",
    "        \n",
    "        remaining_sentences = list(remaining_sentences)\n",
    "        del remaining_sentences[max_idx_rem]\n",
    "        bisect.insort_left(summary_idx,idx_selected_sentence)\n",
    "\n",
    "        summary  = \"\"\n",
    "\n",
    "        for idx in summary_idx:\n",
    "            summary = summary + \" \" + sentences[idx]\n",
    "\n",
    "        summary_prep = gensim.utils.simple_preprocess(summary, deacc=True)\n",
    "        summary_vector = d2v_model.infer_vector(summary_prep)\n",
    "\n",
    "    return summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### On wikipedia article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**************************************************\n",
      "query History of Israel Babylonian rule (586–538 BCE)\n",
      "**************************************************\n",
      "real summary\n",
      "\n",
      "In 586 BCE King Nebuchadnezzar II of Babylon conquered Judah. According to the Hebrew Bible, he destroyed Solomon's Temple and exiled the Jews to Babylon. The defeat was also recorded by the Babylonians (see the Babylonian Chronicles). Babylonian and Biblical sources suggest that the Judean king, Jehoiachin, switched allegiances between the Egyptians and the Babylonians and that invasion was a punishment for allying with Babylon's principal rival, Egypt. The exiled Jews may have been restricted to the elite.\n",
      "Jehoiachin was eventually released by the Babylonians (see Jehoiachin's Rations Tablets) and according to both the Bible and the Talmud, the Judean royal family (the Davidic line) continued as head of the exile in Babylon (the Exilarch).\n",
      "**************************************************\n",
      "nn summary\n",
      "\n",
      "31\n",
      "64\n",
      "83\n",
      "116\n",
      "  Most of the expelled Spanish Jews moved to North Africa, Poland, to the Ottoman Empire and to the region of Bilad a-Sham, which roughly corresponds to the ancient Kingdom of Israel (united monarchy)  In 2009, Israeli billionaire Yitzhak Tshuva announced the discovery of huge natural gas reserves off the coast of Israel  Israeli independence in 1948 was marked by massive migration of Jews from both Europe and the Muslim countries to Israel, and of Arabs from Israel, followed by the extensive Arab–Israeli conflict  The principles established objectives relating to a transfer of authority from Israel to an interim Palestinian Authority, as a prelude to a final treaty establishing a Palestinian state, in exchange for mutual recognition\n"
     ]
    }
   ],
   "source": [
    "wikipedia_title = \"History of Israel\"\n",
    "with open(data_json+wikipedia_title+\".json\", 'r') as f:\n",
    "    wiki_as_json = json.load(f)\n",
    "\n",
    "text = \"\"\n",
    "for key in wiki_as_json.keys():\n",
    "    if key not in non_selected_keys:\n",
    "        text += \" \" + wiki_as_json[key]\n",
    "        \n",
    "random_idx = np.random.randint(0,len(wiki_as_json.keys()))\n",
    "query = wiki_as_json.keys()[random_idx]\n",
    "summary_true = wiki_as_json[query]\n",
    "limit_size = len(wiki_as_json[query].split())\n",
    "\n",
    "print 50*\"*\"\n",
    "print 'query', query\n",
    "print 50*\"*\"\n",
    "print \"real summary\\n\\n\", summary_true\n",
    "print 50*\"*\"\n",
    "print \"nn summary\\n\\n\", summarize(text,query,d2v_model, fc_model, limit = limit_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### on duc dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pyrouge\n",
    "import re\n",
    "import os\n",
    "\n",
    "title_file = \"/home/ubuntu/summarization_query_oriented/DUC/duc2005_topics.sgml\"\n",
    "titles_folder = \"/home/ubuntu/summarization_query_oriented/DUC/duc2005_docs/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def doc_title_table(title_file):\n",
    "    with open(title_file , 'r') as f :\n",
    "        lines = f.readlines()\n",
    "        raw_text = \"\".join(l for l in lines)\n",
    "        left_idx_num = [ m.end(0) for m in re.finditer(r\"<num>\",raw_text)]\n",
    "        right_idx_num = [ m.start(0) for m in re.finditer(r\"</num>\",raw_text)]\n",
    "\n",
    "        left_idx_title = [ m.end(0) for m in re.finditer(r\"<title>\",raw_text)]\n",
    "        right_idx_title = [ m.start(0) for m in re.finditer(r\"</title>\",raw_text)]\n",
    "\n",
    "        docs_title_dict = {}\n",
    "        for i in range(len(left_idx_num)):\n",
    "            docs_title_dict[raw_text[left_idx_num[i]+1:right_idx_num[i]-1]] = raw_text[left_idx_title[i]+1:right_idx_title[i]-1]\n",
    "    return docs_title_dict\n",
    "\n",
    "def merge_articles(docs_folder):\n",
    "\n",
    "    s = \"\"\n",
    "    \n",
    "    for doc in os.listdir(docs_folder):\n",
    "\n",
    "        with open(docs_folder + doc ,'r') as f:\n",
    "\n",
    "            lines = f.readlines()\n",
    "            raw_doc = \"\".join(txt for txt in lines)\n",
    "            left_idx_headline = [ m.end(0) for m in re.finditer(r\"<HEADLINE>\",raw_doc)]\n",
    "            right_idx_headline = [ m.start(0) for m in re.finditer(r\"</HEADLINE>\",raw_doc)]\n",
    "\n",
    "            left_idx_text = [ m.end(0) for m in re.finditer(r\"<TEXT>\",raw_doc)]\n",
    "            right_idx_text = [ m.start(0) for m in re.finditer(r\"</TEXT>\",raw_doc)]\n",
    "\n",
    "            raw_headline = raw_doc[left_idx_headline[0]:right_idx_headline[0]]\n",
    "            raw_text = raw_doc[left_idx_text[0]:right_idx_text[0]]\n",
    "\n",
    "            left_idx_paragraph_headline = [ m.end(0) for m in re.finditer(r\"<P>\",raw_headline)]\n",
    "            right_idx_paragraph_headline = [ m.start(0) for m in re.finditer(r\"</P>\",raw_headline)]\n",
    "\n",
    "            left_idx_paragraph_text = [ m.end(0) for m in re.finditer(r\"<P>\",raw_text)]\n",
    "            right_idx_paragraph_text = [ m.start(0) for m in re.finditer(r\"</P>\",raw_text)]\n",
    "\n",
    "            for i in range(len(left_idx_paragraph_headline)):\n",
    "                s += raw_headline[left_idx_paragraph_headline[i]:right_idx_paragraph_headline[i]-2] + \".\"\n",
    "\n",
    "            for i in range(len(left_idx_paragraph_text)):\n",
    "                s += raw_text[left_idx_paragraph_text[i]:right_idx_paragraph_text[i]-1]\n",
    "\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "docs_title = doc_title_table(title_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "docs_key = docs_title.keys()[0]\n",
    "query = docs_title[docs_key]\n",
    "docs_folder = titles_folder+docs_key+\"/\"\n",
    "text = merge_articles(docs_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "journalist risks\n",
      "**************************************************\n",
      "41\n",
      "67\n",
      "98\n",
      "127\n",
      "179\n",
      "225\n",
      "243\n",
      "285\n",
      " \n",
      "After a yearlong reign of terror by government-backed death squads that emptied \n",
      "villages of young men and littered Sri Lanka's beaches with corpses, the \n",
      "fishermen of Moratuwa were hardly surprised when they spotted yet another body \n",
      "floating 250 yards offshore recently  \n",
      "The word was out that De Zoysa had helped produce a satirical play that was \n",
      "savagely critical of Premadasa, who has been so sensitive to personal criticism \n",
      "that he has barred at least two foreign journalists from Sri Lanka simply for \n",
      "writing about his low-caste roots  \n",
      "Last month, Badr, 63, survived what appeared to be an assassination attempt \n",
      "when a car bomb exploded just before he arrived at a police recreational \n",
      "facility  \n",
      "The determination to fight with the pen is certainly what drives the Colombian \n",
      "journalism fraternity, which has made a greater sacrifice than any other \n",
      "national group: 19 journalists were murdered by the Colombian drug mafia last \n",
      "year, 86 in the last decade  \n",
      "The networks' first video footage of the bloodshed, broadcast before dawn \n",
      "Sunday and later that morning, included scenes of bloodied bodies lying on the \n",
      "ground, a tent being crushed by a tank and dramatic shots on ABC of a stalled \n",
      "armored personnel carrier exploding after being set afire by student-thrown \n",
      "Molotov cocktails  \n",
      "Meanwhile, police in Buenaventura, northwest of Cali on the Pacific coast, said \n",
      "journalist Guillermo Gomez Murillo was shot to death Saturday night while \n",
      "watching television in his parents' home  \n",
      "Both the caller and the cartel statement said traffickers are also holding \n",
      "Diana Turbay de Uribe, the editor of a news magazine and daughter of former \n",
      "Colombian president Julio Cesar Turbay  The militants believe officials are \n",
      "too soft on minority Tamil rebels, who are fighting for an independent \n",
      "homeland\n"
     ]
    }
   ],
   "source": [
    "print query\n",
    "print 50*'*'\n",
    "summary = summarize(text,query,d2v_model, fc_model, limit = 250)\n",
    "print summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "285\n"
     ]
    }
   ],
   "source": [
    "print len(summary.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# system dir \n",
    "system_folder = \"/home/ubuntu/summarization_query_oriented/DUC/duc2005_summary_system/first_try/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26\n",
      "67\n",
      "109\n",
      "145\n",
      "176\n",
      "201\n",
      "230\n",
      "276\n",
      "44\n",
      "74\n",
      "114\n",
      "148\n",
      "172\n",
      "213\n",
      "242\n",
      "262\n",
      "34\n",
      "104\n",
      "132\n",
      "166\n",
      "206\n",
      "236\n",
      "266\n",
      "0\n",
      "28\n",
      "46\n",
      "66\n",
      "78\n",
      "100\n",
      "109\n",
      "143\n",
      "177\n",
      "229\n",
      "258\n",
      "41\n",
      "66\n",
      "100\n",
      "146\n",
      "182\n",
      "224\n",
      "248\n",
      "276\n",
      "0\n",
      "0\n",
      "33\n",
      "65\n",
      "101\n",
      "116\n",
      "132\n",
      "162\n",
      "206\n",
      "224\n",
      "248\n",
      "288"
     ]
    }
   ],
   "source": [
    "for docs_key in docs_title.keys():\n",
    "    try :\n",
    "        docs_folder = titles_folder+docs_key+\"/\"\n",
    "        text = merge_articles(docs_folder)\n",
    "        summary = summarize(text,query,d2v_model, fc_model, limit = 250)\n",
    "\n",
    "        summary = \" \".join(summary.split()[:250])\n",
    "\n",
    "        with open(system_folder+docs_key,'w') as f :\n",
    "            f.write(summary)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'History of Israel Babylonian rule (586\\u2013538 BCE)'"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### pyrouge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:47,742 [MainThread  ] [INFO ]  Writing summaries.\n",
      "INFO:global:Writing summaries.\n",
      "2016-08-28 07:51:47,743 [MainThread  ] [INFO ]  Processing summaries. Saving system files to /tmp/tmpx8vS_G/system and model files to /tmp/tmpx8vS_G/model.\n",
      "INFO:global:Processing summaries. Saving system files to /tmp/tmpx8vS_G/system and model files to /tmp/tmpx8vS_G/model.\n",
      "2016-08-28 07:51:47,744 [MainThread  ] [INFO ]  Processing files in /home/ubuntu/summarization_query_oriented/DUC/duc2005_summary_system/first_try.\n",
      "INFO:global:Processing files in /home/ubuntu/summarization_query_oriented/DUC/duc2005_summary_system/first_try.\n",
      "2016-08-28 07:51:47,745 [MainThread  ] [INFO ]  Processing d413a.\n",
      "INFO:global:Processing d413a.\n",
      "2016-08-28 07:51:47,746 [MainThread  ] [INFO ]  Processing d345j.\n",
      "INFO:global:Processing d345j.\n",
      "2016-08-28 07:51:47,747 [MainThread  ] [INFO ]  Processing d654f.\n",
      "INFO:global:Processing d654f.\n",
      "2016-08-28 07:51:47,748 [MainThread  ] [INFO ]  Processing d376e.\n",
      "INFO:global:Processing d376e.\n",
      "2016-08-28 07:51:47,749 [MainThread  ] [INFO ]  Processing d391h.\n",
      "INFO:global:Processing d391h.\n",
      "2016-08-28 07:51:47,750 [MainThread  ] [INFO ]  Processing d446j.\n",
      "INFO:global:Processing d446j.\n",
      "2016-08-28 07:51:47,751 [MainThread  ] [INFO ]  Processing d383j.\n",
      "INFO:global:Processing d383j.\n",
      "2016-08-28 07:51:47,752 [MainThread  ] [INFO ]  Processing d370i.\n",
      "INFO:global:Processing d370i.\n",
      "2016-08-28 07:51:47,753 [MainThread  ] [INFO ]  Processing d633g.\n",
      "INFO:global:Processing d633g.\n",
      "2016-08-28 07:51:47,754 [MainThread  ] [INFO ]  Processing d332h.\n",
      "INFO:global:Processing d332h.\n",
      "2016-08-28 07:51:47,756 [MainThread  ] [INFO ]  Processing d354c.\n",
      "INFO:global:Processing d354c.\n",
      "2016-08-28 07:51:47,757 [MainThread  ] [INFO ]  Processing d400b.\n",
      "INFO:global:Processing d400b.\n",
      "2016-08-28 07:51:47,758 [MainThread  ] [INFO ]  Processing d435f.\n",
      "INFO:global:Processing d435f.\n",
      "2016-08-28 07:51:47,759 [MainThread  ] [INFO ]  Processing d389h.\n",
      "INFO:global:Processing d389h.\n",
      "2016-08-28 07:51:47,760 [MainThread  ] [INFO ]  Processing d360f.\n",
      "INFO:global:Processing d360f.\n",
      "2016-08-28 07:51:47,761 [MainThread  ] [INFO ]  Processing d331f.\n",
      "INFO:global:Processing d331f.\n",
      "2016-08-28 07:51:47,762 [MainThread  ] [INFO ]  Processing d436j.\n",
      "INFO:global:Processing d436j.\n",
      "2016-08-28 07:51:47,763 [MainThread  ] [INFO ]  Processing d357i.\n",
      "INFO:global:Processing d357i.\n",
      "2016-08-28 07:51:47,764 [MainThread  ] [INFO ]  Processing d346h.\n",
      "INFO:global:Processing d346h.\n",
      "2016-08-28 07:51:47,765 [MainThread  ] [INFO ]  Processing d426a.\n",
      "INFO:global:Processing d426a.\n",
      "2016-08-28 07:51:47,766 [MainThread  ] [INFO ]  Processing d422c.\n",
      "INFO:global:Processing d422c.\n",
      "2016-08-28 07:51:47,767 [MainThread  ] [INFO ]  Processing d632i.\n",
      "INFO:global:Processing d632i.\n",
      "2016-08-28 07:51:47,768 [MainThread  ] [INFO ]  Processing d401e.\n",
      "INFO:global:Processing d401e.\n",
      "2016-08-28 07:51:47,769 [MainThread  ] [INFO ]  Processing d385g.\n",
      "INFO:global:Processing d385g.\n",
      "2016-08-28 07:51:47,770 [MainThread  ] [INFO ]  Processing d694j.\n",
      "INFO:global:Processing d694j.\n",
      "2016-08-28 07:51:47,771 [MainThread  ] [INFO ]  Processing d350a.\n",
      "INFO:global:Processing d350a.\n",
      "2016-08-28 07:51:47,772 [MainThread  ] [INFO ]  Processing d407b.\n",
      "INFO:global:Processing d407b.\n",
      "2016-08-28 07:51:47,773 [MainThread  ] [INFO ]  Processing d311i.\n",
      "INFO:global:Processing d311i.\n",
      "2016-08-28 07:51:47,774 [MainThread  ] [INFO ]  Processing d431h.\n",
      "INFO:global:Processing d431h.\n",
      "2016-08-28 07:51:47,775 [MainThread  ] [INFO ]  Processing d699a.\n",
      "INFO:global:Processing d699a.\n",
      "2016-08-28 07:51:47,776 [MainThread  ] [INFO ]  Processing d347b.\n",
      "INFO:global:Processing d347b.\n",
      "2016-08-28 07:51:47,777 [MainThread  ] [INFO ]  Processing d438g.\n",
      "INFO:global:Processing d438g.\n",
      "2016-08-28 07:51:47,778 [MainThread  ] [INFO ]  Processing d374a.\n",
      "INFO:global:Processing d374a.\n",
      "2016-08-28 07:51:47,779 [MainThread  ] [INFO ]  Processing d428e.\n",
      "INFO:global:Processing d428e.\n",
      "2016-08-28 07:51:47,780 [MainThread  ] [INFO ]  Processing d695c.\n",
      "INFO:global:Processing d695c.\n",
      "2016-08-28 07:51:47,781 [MainThread  ] [INFO ]  Processing d404g.\n",
      "INFO:global:Processing d404g.\n",
      "2016-08-28 07:51:47,782 [MainThread  ] [INFO ]  Processing d307b.\n",
      "INFO:global:Processing d307b.\n",
      "2016-08-28 07:51:47,783 [MainThread  ] [INFO ]  Processing d313e.\n",
      "INFO:global:Processing d313e.\n",
      "2016-08-28 07:51:47,785 [MainThread  ] [INFO ]  Processing d301i.\n",
      "INFO:global:Processing d301i.\n",
      "2016-08-28 07:51:47,786 [MainThread  ] [INFO ]  Processing d324e.\n",
      "INFO:global:Processing d324e.\n",
      "2016-08-28 07:51:47,787 [MainThread  ] [INFO ]  Processing d683j.\n",
      "INFO:global:Processing d683j.\n",
      "2016-08-28 07:51:47,788 [MainThread  ] [INFO ]  Processing d321f.\n",
      "INFO:global:Processing d321f.\n",
      "2016-08-28 07:51:47,789 [MainThread  ] [INFO ]  Processing d343c.\n",
      "INFO:global:Processing d343c.\n",
      "2016-08-28 07:51:47,790 [MainThread  ] [INFO ]  Processing d366i.\n",
      "INFO:global:Processing d366i.\n",
      "2016-08-28 07:51:47,791 [MainThread  ] [INFO ]  Processing d398e.\n",
      "INFO:global:Processing d398e.\n",
      "2016-08-28 07:51:47,792 [MainThread  ] [INFO ]  Processing d434b.\n",
      "INFO:global:Processing d434b.\n",
      "2016-08-28 07:51:47,793 [MainThread  ] [INFO ]  Processing d393f.\n",
      "INFO:global:Processing d393f.\n",
      "2016-08-28 07:51:47,794 [MainThread  ] [INFO ]  Saved processed files to /tmp/tmpx8vS_G/system.\n",
      "INFO:global:Saved processed files to /tmp/tmpx8vS_G/system.\n",
      "2016-08-28 07:51:47,795 [MainThread  ] [INFO ]  Processing files in /home/ubuntu/summarization_query_oriented/DUC/duc2005_summary_model.\n",
      "INFO:global:Processing files in /home/ubuntu/summarization_query_oriented/DUC/duc2005_summary_model.\n",
      "2016-08-28 07:51:47,796 [MainThread  ] [INFO ]  Processing D346.M.250.H.A.\n",
      "INFO:global:Processing D346.M.250.H.A.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d413a\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d345j\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d654f\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d376e\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d391h\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d446j\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d383j\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d370i\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d633g\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d332h\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d354c\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d400b\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d435f\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d389h\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d360f\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d331f\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d436j\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d357i\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d346h\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d426a\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d422c\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d632i\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d401e\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d385g\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d694j\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d350a\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d407b\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d311i\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d431h\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d699a\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d347b\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d438g\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d374a\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d428e\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d695c\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d404g\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d307b\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d313e\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d301i\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d324e\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d683j\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d321f\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d343c\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d366i\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d398e\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d434b\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "d393f\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:47,798 [MainThread  ] [INFO ]  Processing D426.M.250.A.C.\n",
      "INFO:global:Processing D426.M.250.A.C.\n",
      "2016-08-28 07:51:47,799 [MainThread  ] [INFO ]  Processing D301.M.250.I.A.\n",
      "INFO:global:Processing D301.M.250.I.A.\n",
      "2016-08-28 07:51:47,800 [MainThread  ] [INFO ]  Processing D407.M.250.B.G.\n",
      "INFO:global:Processing D407.M.250.B.G.\n",
      "2016-08-28 07:51:47,801 [MainThread  ] [INFO ]  Processing D350.M.250.A.B.\n",
      "INFO:global:Processing D350.M.250.A.B.\n",
      "2016-08-28 07:51:47,802 [MainThread  ] [INFO ]  Processing D370.M.250.I.D.\n",
      "INFO:global:Processing D370.M.250.I.D.\n",
      "2016-08-28 07:51:47,803 [MainThread  ] [INFO ]  Processing D426.M.250.A.H.\n",
      "INFO:global:Processing D426.M.250.A.H.\n",
      "2016-08-28 07:51:47,804 [MainThread  ] [INFO ]  Processing D632.M.250.I.B.\n",
      "INFO:global:Processing D632.M.250.I.B.\n",
      "2016-08-28 07:51:47,805 [MainThread  ] [INFO ]  Processing D428.M.250.E.G.\n",
      "INFO:global:Processing D428.M.250.E.G.\n",
      "2016-08-28 07:51:47,806 [MainThread  ] [INFO ]  Processing D311.M.250.I.E.\n",
      "INFO:global:Processing D311.M.250.I.E.\n",
      "2016-08-28 07:51:47,807 [MainThread  ] [INFO ]  Processing D426.M.250.A.B.\n",
      "INFO:global:Processing D426.M.250.A.B.\n",
      "2016-08-28 07:51:47,809 [MainThread  ] [INFO ]  Processing D407.M.250.B.D.\n",
      "INFO:global:Processing D407.M.250.B.D.\n",
      "2016-08-28 07:51:47,810 [MainThread  ] [INFO ]  Processing D354.M.250.C.B.\n",
      "INFO:global:Processing D354.M.250.C.B.\n",
      "2016-08-28 07:51:47,811 [MainThread  ] [INFO ]  Processing D374.M.250.A.F.\n",
      "INFO:global:Processing D374.M.250.A.F.\n",
      "2016-08-28 07:51:47,812 [MainThread  ] [INFO ]  Processing D442.M.250.G.J.\n",
      "INFO:global:Processing D442.M.250.G.J.\n",
      "2016-08-28 07:51:47,813 [MainThread  ] [INFO ]  Processing D426.M.250.A.A.\n",
      "INFO:global:Processing D426.M.250.A.A.\n",
      "2016-08-28 07:51:47,814 [MainThread  ] [INFO ]  Processing D311.M.250.I.I.\n",
      "INFO:global:Processing D311.M.250.I.I.\n",
      "2016-08-28 07:51:47,815 [MainThread  ] [INFO ]  Processing D374.M.250.A.E.\n",
      "INFO:global:Processing D374.M.250.A.E.\n",
      "2016-08-28 07:51:47,816 [MainThread  ] [INFO ]  Processing D632.M.250.I.C.\n",
      "INFO:global:Processing D632.M.250.I.C.\n",
      "2016-08-28 07:51:47,817 [MainThread  ] [INFO ]  Processing D434.M.250.B.G.\n",
      "INFO:global:Processing D434.M.250.B.G.\n",
      "2016-08-28 07:51:47,818 [MainThread  ] [INFO ]  Processing D671.M.250.G.F.\n",
      "INFO:global:Processing D671.M.250.G.F.\n",
      "2016-08-28 07:51:47,820 [MainThread  ] [INFO ]  Processing D632.M.250.I.A.\n",
      "INFO:global:Processing D632.M.250.I.A.\n",
      "2016-08-28 07:51:47,821 [MainThread  ] [INFO ]  Processing D393.M.250.F.F.\n",
      "INFO:global:Processing D393.M.250.F.F.\n",
      "2016-08-28 07:51:47,822 [MainThread  ] [INFO ]  Processing D398.M.250.E.E.\n",
      "INFO:global:Processing D398.M.250.E.E.\n",
      "2016-08-28 07:51:47,823 [MainThread  ] [INFO ]  Processing D654.M.250.F.J.\n",
      "INFO:global:Processing D654.M.250.F.J.\n",
      "2016-08-28 07:51:47,824 [MainThread  ] [INFO ]  Processing D671.M.250.G.C.\n",
      "INFO:global:Processing D671.M.250.G.C.\n",
      "2016-08-28 07:51:47,825 [MainThread  ] [INFO ]  Processing D385.M.250.G.G.\n",
      "INFO:global:Processing D385.M.250.G.G.\n",
      "2016-08-28 07:51:47,826 [MainThread  ] [INFO ]  Processing D435.M.250.F.H.\n",
      "INFO:global:Processing D435.M.250.F.H.\n",
      "2016-08-28 07:51:47,827 [MainThread  ] [INFO ]  Processing D389.M.250.H.E.\n",
      "INFO:global:Processing D389.M.250.H.E.\n",
      "2016-08-28 07:51:47,828 [MainThread  ] [INFO ]  Processing D671.M.250.G.A.\n",
      "INFO:global:Processing D671.M.250.G.A.\n",
      "2016-08-28 07:51:47,829 [MainThread  ] [INFO ]  Processing D311.M.250.I.G.\n",
      "INFO:global:Processing D311.M.250.I.G.\n",
      "2016-08-28 07:51:47,831 [MainThread  ] [INFO ]  Processing D347.M.250.B.A.\n",
      "INFO:global:Processing D347.M.250.B.A.\n",
      "2016-08-28 07:51:47,832 [MainThread  ] [INFO ]  Processing D413.M.250.A.I.\n",
      "INFO:global:Processing D413.M.250.A.I.\n",
      "2016-08-28 07:51:47,833 [MainThread  ] [INFO ]  Processing D391.M.250.H.F.\n",
      "INFO:global:Processing D391.M.250.H.F.\n",
      "2016-08-28 07:51:47,834 [MainThread  ] [INFO ]  Processing D332.M.250.H.B.\n",
      "INFO:global:Processing D332.M.250.H.B.\n",
      "2016-08-28 07:51:47,835 [MainThread  ] [INFO ]  Processing D393.M.250.F.D.\n",
      "INFO:global:Processing D393.M.250.F.D.\n",
      "2016-08-28 07:51:47,836 [MainThread  ] [INFO ]  Processing D383.M.250.J.E.\n",
      "INFO:global:Processing D383.M.250.J.E.\n",
      "2016-08-28 07:51:47,837 [MainThread  ] [INFO ]  Processing D671.M.250.G.E.\n",
      "INFO:global:Processing D671.M.250.G.E.\n",
      "2016-08-28 07:51:47,838 [MainThread  ] [INFO ]  Processing D370.M.250.I.I.\n",
      "INFO:global:Processing D370.M.250.I.I.\n",
      "2016-08-28 07:51:47,839 [MainThread  ] [INFO ]  Processing D311.M.250.I.J.\n",
      "INFO:global:Processing D311.M.250.I.J.\n",
      "2016-08-28 07:51:47,840 [MainThread  ] [INFO ]  Processing D383.M.250.J.J.\n",
      "INFO:global:Processing D383.M.250.J.J.\n",
      "2016-08-28 07:51:47,842 [MainThread  ] [INFO ]  Processing D376.M.250.E.G.\n",
      "INFO:global:Processing D376.M.250.E.G.\n",
      "2016-08-28 07:51:47,843 [MainThread  ] [INFO ]  Processing D324.M.250.E.E.\n",
      "INFO:global:Processing D324.M.250.E.E.\n",
      "2016-08-28 07:51:47,844 [MainThread  ] [INFO ]  Processing D307.M.250.B.C.\n",
      "INFO:global:Processing D307.M.250.B.C.\n",
      "2016-08-28 07:51:47,845 [MainThread  ] [INFO ]  Processing D313.M.250.E.A.\n",
      "INFO:global:Processing D313.M.250.E.A.\n",
      "2016-08-28 07:51:47,846 [MainThread  ] [INFO ]  Processing D345.M.250.J.J.\n",
      "INFO:global:Processing D345.M.250.J.J.\n",
      "2016-08-28 07:51:47,847 [MainThread  ] [INFO ]  Processing D321.M.250.F.F.\n",
      "INFO:global:Processing D321.M.250.F.F.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "D346.M.250.H.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D301.M.250.I.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D350.M.250.A.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D370.M.250.I.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D428.M.250.E.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D354.M.250.C.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D374.M.250.A.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D442.M.250.G.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D374.M.250.A.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D434.M.250.B.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D398.M.250.E.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D385.M.250.G.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D389.M.250.H.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D347.M.250.B.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D332.M.250.H.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D383.M.250.J.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D370.M.250.I.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D383.M.250.J.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D307.M.250.B.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D313.M.250.E.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:47,849 [MainThread  ] [INFO ]  Processing D413.M.250.A.F.\n",
      "INFO:global:Processing D413.M.250.A.F.\n",
      "2016-08-28 07:51:47,850 [MainThread  ] [INFO ]  Processing D376.M.250.E.A.\n",
      "INFO:global:Processing D376.M.250.E.A.\n",
      "2016-08-28 07:51:47,851 [MainThread  ] [INFO ]  Processing D350.M.250.A.A.\n",
      "INFO:global:Processing D350.M.250.A.A.\n",
      "2016-08-28 07:51:47,852 [MainThread  ] [INFO ]  Processing D407.M.250.B.E.\n",
      "INFO:global:Processing D407.M.250.B.E.\n",
      "2016-08-28 07:51:47,853 [MainThread  ] [INFO ]  Processing D313.M.250.E.C.\n",
      "INFO:global:Processing D313.M.250.E.C.\n",
      "2016-08-28 07:51:47,854 [MainThread  ] [INFO ]  Processing D422.M.250.C.J.\n",
      "INFO:global:Processing D422.M.250.C.J.\n",
      "2016-08-28 07:51:47,856 [MainThread  ] [INFO ]  Processing D407.M.250.B.C.\n",
      "INFO:global:Processing D407.M.250.B.C.\n",
      "2016-08-28 07:51:47,857 [MainThread  ] [INFO ]  Processing D431.M.250.H.A.\n",
      "INFO:global:Processing D431.M.250.H.A.\n",
      "2016-08-28 07:51:47,858 [MainThread  ] [INFO ]  Processing D422.M.250.C.C.\n",
      "INFO:global:Processing D422.M.250.C.C.\n",
      "2016-08-28 07:51:47,859 [MainThread  ] [INFO ]  Processing D434.M.250.B.B.\n",
      "INFO:global:Processing D434.M.250.B.B.\n",
      "2016-08-28 07:51:47,860 [MainThread  ] [INFO ]  Processing D699.M.250.A.H.\n",
      "INFO:global:Processing D699.M.250.A.H.\n",
      "2016-08-28 07:51:47,861 [MainThread  ] [INFO ]  Processing D633.M.250.G.J.\n",
      "INFO:global:Processing D633.M.250.G.J.\n",
      "2016-08-28 07:51:47,862 [MainThread  ] [INFO ]  Processing D401.M.250.E.F.\n",
      "INFO:global:Processing D401.M.250.E.F.\n",
      "2016-08-28 07:51:47,863 [MainThread  ] [INFO ]  Processing D422.M.250.C.E.\n",
      "INFO:global:Processing D422.M.250.C.E.\n",
      "2016-08-28 07:51:47,864 [MainThread  ] [INFO ]  Processing D393.M.250.F.H.\n",
      "INFO:global:Processing D393.M.250.F.H.\n",
      "2016-08-28 07:51:47,865 [MainThread  ] [INFO ]  Processing D332.M.250.H.H.\n",
      "INFO:global:Processing D332.M.250.H.H.\n",
      "2016-08-28 07:51:47,867 [MainThread  ] [INFO ]  Processing D301.M.250.I.I.\n",
      "INFO:global:Processing D301.M.250.I.I.\n",
      "2016-08-28 07:51:47,868 [MainThread  ] [INFO ]  Processing D324.M.250.E.H.\n",
      "INFO:global:Processing D324.M.250.E.H.\n",
      "2016-08-28 07:51:47,869 [MainThread  ] [INFO ]  Processing D324.M.250.E.D.\n",
      "INFO:global:Processing D324.M.250.E.D.\n",
      "2016-08-28 07:51:47,870 [MainThread  ] [INFO ]  Processing D694.M.250.J.J.\n",
      "INFO:global:Processing D694.M.250.J.J.\n",
      "2016-08-28 07:51:47,871 [MainThread  ] [INFO ]  Processing D446.M.250.J.J.\n",
      "INFO:global:Processing D446.M.250.J.J.\n",
      "2016-08-28 07:51:47,872 [MainThread  ] [INFO ]  Processing D374.M.250.A.A.\n",
      "INFO:global:Processing D374.M.250.A.A.\n",
      "2016-08-28 07:51:47,873 [MainThread  ] [INFO ]  Processing D400.M.250.B.G.\n",
      "INFO:global:Processing D400.M.250.B.G.\n",
      "2016-08-28 07:51:47,874 [MainThread  ] [INFO ]  Processing D422.M.250.C.D.\n",
      "INFO:global:Processing D422.M.250.C.D.\n",
      "2016-08-28 07:51:47,875 [MainThread  ] [INFO ]  Processing D434.M.250.B.I.\n",
      "INFO:global:Processing D434.M.250.B.I.\n",
      "2016-08-28 07:51:47,877 [MainThread  ] [INFO ]  Processing D343.M.250.C.D.\n",
      "INFO:global:Processing D343.M.250.C.D.\n",
      "2016-08-28 07:51:47,878 [MainThread  ] [INFO ]  Processing D694.M.250.J.I.\n",
      "INFO:global:Processing D694.M.250.J.I.\n",
      "2016-08-28 07:51:47,879 [MainThread  ] [INFO ]  Processing D431.M.250.H.H.\n",
      "INFO:global:Processing D431.M.250.H.H.\n",
      "2016-08-28 07:51:47,880 [MainThread  ] [INFO ]  Processing D683.M.250.J.E.\n",
      "INFO:global:Processing D683.M.250.J.E.\n",
      "2016-08-28 07:51:47,881 [MainThread  ] [INFO ]  Processing D428.M.250.E.I.\n",
      "INFO:global:Processing D428.M.250.E.I.\n",
      "2016-08-28 07:51:47,882 [MainThread  ] [INFO ]  Processing D654.M.250.F.G.\n",
      "INFO:global:Processing D654.M.250.F.G.\n",
      "2016-08-28 07:51:47,883 [MainThread  ] [INFO ]  Processing D654.M.250.F.D.\n",
      "INFO:global:Processing D654.M.250.F.D.\n",
      "2016-08-28 07:51:47,884 [MainThread  ] [INFO ]  Processing D354.M.250.C.D.\n",
      "INFO:global:Processing D354.M.250.C.D.\n",
      "2016-08-28 07:51:47,885 [MainThread  ] [INFO ]  Processing D331.M.250.F.F.\n",
      "INFO:global:Processing D331.M.250.F.F.\n",
      "2016-08-28 07:51:47,886 [MainThread  ] [INFO ]  Processing D401.M.250.E.J.\n",
      "INFO:global:Processing D401.M.250.E.J.\n",
      "2016-08-28 07:51:47,888 [MainThread  ] [INFO ]  Processing D413.M.250.A.B.\n",
      "INFO:global:Processing D413.M.250.A.B.\n",
      "2016-08-28 07:51:47,889 [MainThread  ] [INFO ]  Processing D345.M.250.J.D.\n",
      "INFO:global:Processing D345.M.250.J.D.\n",
      "2016-08-28 07:51:47,890 [MainThread  ] [INFO ]  Processing D385.M.250.G.F.\n",
      "INFO:global:Processing D385.M.250.G.F.\n",
      "2016-08-28 07:51:47,891 [MainThread  ] [INFO ]  Processing D407.M.250.B.F.\n",
      "INFO:global:Processing D407.M.250.B.F.\n",
      "2016-08-28 07:51:47,892 [MainThread  ] [INFO ]  Processing D632.M.250.I.J.\n",
      "INFO:global:Processing D632.M.250.I.J.\n",
      "2016-08-28 07:51:47,893 [MainThread  ] [INFO ]  Processing D428.M.250.E.J.\n",
      "INFO:global:Processing D428.M.250.E.J.\n",
      "2016-08-28 07:51:47,894 [MainThread  ] [INFO ]  Processing D385.M.250.G.E.\n",
      "INFO:global:Processing D385.M.250.G.E.\n",
      "2016-08-28 07:51:47,895 [MainThread  ] [INFO ]  Processing D422.M.250.C.I.\n",
      "INFO:global:Processing D422.M.250.C.I.\n",
      "2016-08-28 07:51:47,896 [MainThread  ] [INFO ]  Processing D370.M.250.I.F.\n",
      "INFO:global:Processing D370.M.250.I.F.\n",
      "2016-08-28 07:51:47,897 [MainThread  ] [INFO ]  Processing D431.M.250.H.J.\n",
      "INFO:global:Processing D431.M.250.H.J.\n",
      "2016-08-28 07:51:47,899 [MainThread  ] [INFO ]  Processing D391.M.250.H.C.\n",
      "INFO:global:Processing D391.M.250.H.C.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "D321.M.250.F.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D350.M.250.A.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D313.M.250.E.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D434.M.250.B.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D699.M.250.A.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D401.M.250.E.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D332.M.250.H.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D301.M.250.I.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D694.M.250.J.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D446.M.250.J.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D374.M.250.A.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D434.M.250.B.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D343.M.250.C.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D694.M.250.J.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D428.M.250.E.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D354.M.250.C.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D331.M.250.F.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D401.M.250.E.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D385.M.250.G.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D428.M.250.E.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D385.M.250.G.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D370.M.250.I.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:47,900 [MainThread  ] [INFO ]  Processing D400.M.250.B.A.\n",
      "INFO:global:Processing D400.M.250.B.A.\n",
      "2016-08-28 07:51:47,901 [MainThread  ] [INFO ]  Processing D671.M.250.G.D.\n",
      "INFO:global:Processing D671.M.250.G.D.\n",
      "2016-08-28 07:51:47,903 [MainThread  ] [INFO ]  Processing D360.M.250.F.G.\n",
      "INFO:global:Processing D360.M.250.F.G.\n",
      "2016-08-28 07:51:47,904 [MainThread  ] [INFO ]  Processing D683.M.250.J.F.\n",
      "INFO:global:Processing D683.M.250.J.F.\n",
      "2016-08-28 07:51:47,905 [MainThread  ] [INFO ]  Processing D324.M.250.E.J.\n",
      "INFO:global:Processing D324.M.250.E.J.\n",
      "2016-08-28 07:51:47,906 [MainThread  ] [INFO ]  Processing D311.M.250.I.H.\n",
      "INFO:global:Processing D311.M.250.I.H.\n",
      "2016-08-28 07:51:47,907 [MainThread  ] [INFO ]  Processing D683.M.250.J.J.\n",
      "INFO:global:Processing D683.M.250.J.J.\n",
      "2016-08-28 07:51:47,908 [MainThread  ] [INFO ]  Processing D632.M.250.I.H.\n",
      "INFO:global:Processing D632.M.250.I.H.\n",
      "2016-08-28 07:51:47,909 [MainThread  ] [INFO ]  Processing D400.M.250.B.E.\n",
      "INFO:global:Processing D400.M.250.B.E.\n",
      "2016-08-28 07:51:47,910 [MainThread  ] [INFO ]  Processing D391.M.250.H.E.\n",
      "INFO:global:Processing D391.M.250.H.E.\n",
      "2016-08-28 07:51:47,911 [MainThread  ] [INFO ]  Processing D632.M.250.I.D.\n",
      "INFO:global:Processing D632.M.250.I.D.\n",
      "2016-08-28 07:51:47,912 [MainThread  ] [INFO ]  Processing D366.M.250.I.C.\n",
      "INFO:global:Processing D366.M.250.I.C.\n",
      "2016-08-28 07:51:47,914 [MainThread  ] [INFO ]  Processing D357.M.250.I.I.\n",
      "INFO:global:Processing D357.M.250.I.I.\n",
      "2016-08-28 07:51:47,915 [MainThread  ] [INFO ]  Processing D374.M.250.A.D.\n",
      "INFO:global:Processing D374.M.250.A.D.\n",
      "2016-08-28 07:51:47,916 [MainThread  ] [INFO ]  Processing D366.M.250.I.F.\n",
      "INFO:global:Processing D366.M.250.I.F.\n",
      "2016-08-28 07:51:47,917 [MainThread  ] [INFO ]  Processing D376.M.250.E.H.\n",
      "INFO:global:Processing D376.M.250.E.H.\n",
      "2016-08-28 07:51:47,918 [MainThread  ] [INFO ]  Processing D683.M.250.J.D.\n",
      "INFO:global:Processing D683.M.250.J.D.\n",
      "2016-08-28 07:51:47,919 [MainThread  ] [INFO ]  Processing D360.M.250.F.E.\n",
      "INFO:global:Processing D360.M.250.F.E.\n",
      "2016-08-28 07:51:47,920 [MainThread  ] [INFO ]  Processing D391.M.250.H.A.\n",
      "INFO:global:Processing D391.M.250.H.A.\n",
      "2016-08-28 07:51:47,921 [MainThread  ] [INFO ]  Processing D633.M.250.G.B.\n",
      "INFO:global:Processing D633.M.250.G.B.\n",
      "2016-08-28 07:51:47,922 [MainThread  ] [INFO ]  Processing D346.M.250.H.H.\n",
      "INFO:global:Processing D346.M.250.H.H.\n",
      "2016-08-28 07:51:47,924 [MainThread  ] [INFO ]  Processing D422.M.250.C.F.\n",
      "INFO:global:Processing D422.M.250.C.F.\n",
      "2016-08-28 07:51:47,925 [MainThread  ] [INFO ]  Processing D683.M.250.J.G.\n",
      "INFO:global:Processing D683.M.250.J.G.\n",
      "2016-08-28 07:51:47,926 [MainThread  ] [INFO ]  Processing D401.M.250.E.G.\n",
      "INFO:global:Processing D401.M.250.E.G.\n",
      "2016-08-28 07:51:47,927 [MainThread  ] [INFO ]  Processing D683.M.250.J.B.\n",
      "INFO:global:Processing D683.M.250.J.B.\n",
      "2016-08-28 07:51:47,928 [MainThread  ] [INFO ]  Processing D413.M.250.A.E.\n",
      "INFO:global:Processing D413.M.250.A.E.\n",
      "2016-08-28 07:51:47,929 [MainThread  ] [INFO ]  Processing D695.M.250.C.A.\n",
      "INFO:global:Processing D695.M.250.C.A.\n",
      "2016-08-28 07:51:47,930 [MainThread  ] [INFO ]  Processing D324.M.250.E.F.\n",
      "INFO:global:Processing D324.M.250.E.F.\n",
      "2016-08-28 07:51:47,931 [MainThread  ] [INFO ]  Processing D632.M.250.I.E.\n",
      "INFO:global:Processing D632.M.250.I.E.\n",
      "2016-08-28 07:51:47,932 [MainThread  ] [INFO ]  Processing D393.M.250.F.G.\n",
      "INFO:global:Processing D393.M.250.F.G.\n",
      "2016-08-28 07:51:47,933 [MainThread  ] [INFO ]  Processing D633.M.250.G.H.\n",
      "INFO:global:Processing D633.M.250.G.H.\n",
      "2016-08-28 07:51:47,935 [MainThread  ] [INFO ]  Processing D311.M.250.I.B.\n",
      "INFO:global:Processing D311.M.250.I.B.\n",
      "2016-08-28 07:51:47,936 [MainThread  ] [INFO ]  Processing D401.M.250.E.E.\n",
      "INFO:global:Processing D401.M.250.E.E.\n",
      "2016-08-28 07:51:47,937 [MainThread  ] [INFO ]  Processing D383.M.250.J.D.\n",
      "INFO:global:Processing D383.M.250.J.D.\n",
      "2016-08-28 07:51:47,938 [MainThread  ] [INFO ]  Processing D654.M.250.F.B.\n",
      "INFO:global:Processing D654.M.250.F.B.\n",
      "2016-08-28 07:51:47,939 [MainThread  ] [INFO ]  Processing D376.M.250.E.E.\n",
      "INFO:global:Processing D376.M.250.E.E.\n",
      "2016-08-28 07:51:47,940 [MainThread  ] [INFO ]  Processing D376.M.250.E.C.\n",
      "INFO:global:Processing D376.M.250.E.C.\n",
      "2016-08-28 07:51:47,941 [MainThread  ] [INFO ]  Processing D391.M.250.H.B.\n",
      "INFO:global:Processing D391.M.250.H.B.\n",
      "2016-08-28 07:51:47,942 [MainThread  ] [INFO ]  Processing D431.M.250.H.C.\n",
      "INFO:global:Processing D431.M.250.H.C.\n",
      "2016-08-28 07:51:47,943 [MainThread  ] [INFO ]  Processing D426.M.250.A.G.\n",
      "INFO:global:Processing D426.M.250.A.G.\n",
      "2016-08-28 07:51:47,944 [MainThread  ] [INFO ]  Processing D695.M.250.C.B.\n",
      "INFO:global:Processing D695.M.250.C.B.\n",
      "2016-08-28 07:51:47,946 [MainThread  ] [INFO ]  Processing D376.M.250.E.I.\n",
      "INFO:global:Processing D376.M.250.E.I.\n",
      "2016-08-28 07:51:47,947 [MainThread  ] [INFO ]  Processing D391.M.250.H.I.\n",
      "INFO:global:Processing D391.M.250.H.I.\n",
      "2016-08-28 07:51:47,948 [MainThread  ] [INFO ]  Processing D400.M.250.B.I.\n",
      "INFO:global:Processing D400.M.250.B.I.\n",
      "2016-08-28 07:51:47,949 [MainThread  ] [INFO ]  Processing D695.M.250.C.C.\n",
      "INFO:global:Processing D695.M.250.C.C.\n",
      "2016-08-28 07:51:47,950 [MainThread  ] [INFO ]  Processing D446.M.250.J.I.\n",
      "INFO:global:Processing D446.M.250.J.I.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "D391.M.250.H.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D360.M.250.F.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D357.M.250.I.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D374.M.250.A.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D360.M.250.F.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D346.M.250.H.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D401.M.250.E.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D401.M.250.E.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D383.M.250.J.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:47,952 [MainThread  ] [INFO ]  Processing D350.M.250.A.C.\n",
      "INFO:global:Processing D350.M.250.A.C.\n",
      "2016-08-28 07:51:47,953 [MainThread  ] [INFO ]  Processing D446.M.250.J.H.\n",
      "INFO:global:Processing D446.M.250.J.H.\n",
      "2016-08-28 07:51:47,954 [MainThread  ] [INFO ]  Processing D391.M.250.H.H.\n",
      "INFO:global:Processing D391.M.250.H.H.\n",
      "2016-08-28 07:51:47,955 [MainThread  ] [INFO ]  Processing D671.M.250.G.G.\n",
      "INFO:global:Processing D671.M.250.G.G.\n",
      "2016-08-28 07:51:47,956 [MainThread  ] [INFO ]  Processing D695.M.250.C.D.\n",
      "INFO:global:Processing D695.M.250.C.D.\n",
      "2016-08-28 07:51:47,957 [MainThread  ] [INFO ]  Processing D671.M.250.G.B.\n",
      "INFO:global:Processing D671.M.250.G.B.\n",
      "2016-08-28 07:51:47,958 [MainThread  ] [INFO ]  Processing D345.M.250.J.I.\n",
      "INFO:global:Processing D345.M.250.J.I.\n",
      "2016-08-28 07:51:47,959 [MainThread  ] [INFO ]  Processing D393.M.250.F.B.\n",
      "INFO:global:Processing D393.M.250.F.B.\n",
      "2016-08-28 07:51:47,961 [MainThread  ] [INFO ]  Processing D446.M.250.J.G.\n",
      "INFO:global:Processing D446.M.250.J.G.\n",
      "2016-08-28 07:51:47,962 [MainThread  ] [INFO ]  Processing D357.M.250.I.D.\n",
      "INFO:global:Processing D357.M.250.I.D.\n",
      "2016-08-28 07:51:47,963 [MainThread  ] [INFO ]  Processing D407.M.250.B.B.\n",
      "INFO:global:Processing D407.M.250.B.B.\n",
      "2016-08-28 07:51:47,964 [MainThread  ] [INFO ]  Processing D654.M.250.F.F.\n",
      "INFO:global:Processing D654.M.250.F.F.\n",
      "2016-08-28 07:51:47,965 [MainThread  ] [INFO ]  Processing D400.M.250.B.J.\n",
      "INFO:global:Processing D400.M.250.B.J.\n",
      "2016-08-28 07:51:47,966 [MainThread  ] [INFO ]  Processing D442.M.250.G.I.\n",
      "INFO:global:Processing D442.M.250.G.I.\n",
      "2016-08-28 07:51:47,967 [MainThread  ] [INFO ]  Processing D393.M.250.F.J.\n",
      "INFO:global:Processing D393.M.250.F.J.\n",
      "2016-08-28 07:51:47,968 [MainThread  ] [INFO ]  Processing D393.M.250.F.I.\n",
      "INFO:global:Processing D393.M.250.F.I.\n",
      "2016-08-28 07:51:47,969 [MainThread  ] [INFO ]  Processing D370.M.250.I.E.\n",
      "INFO:global:Processing D370.M.250.I.E.\n",
      "2016-08-28 07:51:47,970 [MainThread  ] [INFO ]  Processing D431.M.250.H.F.\n",
      "INFO:global:Processing D431.M.250.H.F.\n",
      "2016-08-28 07:51:47,972 [MainThread  ] [INFO ]  Processing D391.M.250.H.J.\n",
      "INFO:global:Processing D391.M.250.H.J.\n",
      "2016-08-28 07:51:47,973 [MainThread  ] [INFO ]  Processing D699.M.250.A.I.\n",
      "INFO:global:Processing D699.M.250.A.I.\n",
      "2016-08-28 07:51:47,974 [MainThread  ] [INFO ]  Processing D343.M.250.C.B.\n",
      "INFO:global:Processing D343.M.250.C.B.\n",
      "2016-08-28 07:51:47,975 [MainThread  ] [INFO ]  Processing D332.M.250.H.A.\n",
      "INFO:global:Processing D332.M.250.H.A.\n",
      "2016-08-28 07:51:47,976 [MainThread  ] [INFO ]  Processing D383.M.250.J.F.\n",
      "INFO:global:Processing D383.M.250.J.F.\n",
      "2016-08-28 07:51:47,977 [MainThread  ] [INFO ]  Processing D400.M.250.B.B.\n",
      "INFO:global:Processing D400.M.250.B.B.\n",
      "2016-08-28 07:51:47,978 [MainThread  ] [INFO ]  Processing D366.M.250.I.E.\n",
      "INFO:global:Processing D366.M.250.I.E.\n",
      "2016-08-28 07:51:47,979 [MainThread  ] [INFO ]  Processing D431.M.250.H.G.\n",
      "INFO:global:Processing D431.M.250.H.G.\n",
      "2016-08-28 07:51:47,980 [MainThread  ] [INFO ]  Processing D357.M.250.I.E.\n",
      "INFO:global:Processing D357.M.250.I.E.\n",
      "2016-08-28 07:51:47,982 [MainThread  ] [INFO ]  Processing D435.M.250.F.J.\n",
      "INFO:global:Processing D435.M.250.F.J.\n",
      "2016-08-28 07:51:47,983 [MainThread  ] [INFO ]  Processing D345.M.250.J.G.\n",
      "INFO:global:Processing D345.M.250.J.G.\n",
      "2016-08-28 07:51:47,984 [MainThread  ] [INFO ]  Processing D683.M.250.J.H.\n",
      "INFO:global:Processing D683.M.250.J.H.\n",
      "2016-08-28 07:51:47,985 [MainThread  ] [INFO ]  Processing D426.M.250.A.F.\n",
      "INFO:global:Processing D426.M.250.A.F.\n",
      "2016-08-28 07:51:47,986 [MainThread  ] [INFO ]  Processing D389.M.250.H.D.\n",
      "INFO:global:Processing D389.M.250.H.D.\n",
      "2016-08-28 07:51:47,987 [MainThread  ] [INFO ]  Processing D398.M.250.E.F.\n",
      "INFO:global:Processing D398.M.250.E.F.\n",
      "2016-08-28 07:51:47,988 [MainThread  ] [INFO ]  Processing D331.M.250.F.A.\n",
      "INFO:global:Processing D331.M.250.F.A.\n",
      "2016-08-28 07:51:47,989 [MainThread  ] [INFO ]  Processing D431.M.250.H.D.\n",
      "INFO:global:Processing D431.M.250.H.D.\n",
      "2016-08-28 07:51:47,991 [MainThread  ] [INFO ]  Processing D407.M.250.B.A.\n",
      "INFO:global:Processing D407.M.250.B.A.\n",
      "2016-08-28 07:51:47,992 [MainThread  ] [INFO ]  Processing D654.M.250.F.H.\n",
      "INFO:global:Processing D654.M.250.F.H.\n",
      "2016-08-28 07:51:47,993 [MainThread  ] [INFO ]  Processing D633.M.250.G.G.\n",
      "INFO:global:Processing D633.M.250.G.G.\n",
      "2016-08-28 07:51:47,994 [MainThread  ] [INFO ]  Processing D400.M.250.B.C.\n",
      "INFO:global:Processing D400.M.250.B.C.\n",
      "2016-08-28 07:51:47,995 [MainThread  ] [INFO ]  Processing D389.M.250.H.H.\n",
      "INFO:global:Processing D389.M.250.H.H.\n",
      "2016-08-28 07:51:47,996 [MainThread  ] [INFO ]  Processing D313.M.250.E.B.\n",
      "INFO:global:Processing D313.M.250.E.B.\n",
      "2016-08-28 07:51:47,997 [MainThread  ] [INFO ]  Processing D694.M.250.J.G.\n",
      "INFO:global:Processing D694.M.250.J.G.\n",
      "2016-08-28 07:51:47,998 [MainThread  ] [INFO ]  Processing D311.M.250.I.F.\n",
      "INFO:global:Processing D311.M.250.I.F.\n",
      "2016-08-28 07:51:47,999 [MainThread  ] [INFO ]  Processing D366.M.250.I.G.\n",
      "INFO:global:Processing D366.M.250.I.G.\n",
      "2016-08-28 07:51:48,001 [MainThread  ] [INFO ]  Processing D376.M.250.E.B.\n",
      "INFO:global:Processing D376.M.250.E.B.\n",
      "2016-08-28 07:51:48,002 [MainThread  ] [INFO ]  Processing D413.M.250.A.J.\n",
      "INFO:global:Processing D413.M.250.A.J.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "D446.M.250.J.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D350.M.250.A.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D446.M.250.J.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D446.M.250.J.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D357.M.250.I.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D442.M.250.G.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D370.M.250.I.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D699.M.250.A.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D343.M.250.C.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D332.M.250.H.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D383.M.250.J.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D357.M.250.I.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D389.M.250.H.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D398.M.250.E.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D331.M.250.F.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D389.M.250.H.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D313.M.250.E.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D694.M.250.J.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:48,003 [MainThread  ] [INFO ]  Processing D345.M.250.J.B.\n",
      "INFO:global:Processing D345.M.250.J.B.\n",
      "2016-08-28 07:51:48,004 [MainThread  ] [INFO ]  Processing D391.M.250.H.G.\n",
      "INFO:global:Processing D391.M.250.H.G.\n",
      "2016-08-28 07:51:48,006 [MainThread  ] [INFO ]  Processing D695.M.250.C.J.\n",
      "INFO:global:Processing D695.M.250.C.J.\n",
      "2016-08-28 07:51:48,007 [MainThread  ] [INFO ]  Processing D343.M.250.C.A.\n",
      "INFO:global:Processing D343.M.250.C.A.\n",
      "2016-08-28 07:51:48,008 [MainThread  ] [INFO ]  Processing D321.M.250.F.B.\n",
      "INFO:global:Processing D321.M.250.F.B.\n",
      "2016-08-28 07:51:48,009 [MainThread  ] [INFO ]  Processing D376.M.250.E.D.\n",
      "INFO:global:Processing D376.M.250.E.D.\n",
      "2016-08-28 07:51:48,010 [MainThread  ] [INFO ]  Processing D683.M.250.J.A.\n",
      "INFO:global:Processing D683.M.250.J.A.\n",
      "2016-08-28 07:51:48,011 [MainThread  ] [INFO ]  Processing D408.M.250.C.C.\n",
      "INFO:global:Processing D408.M.250.C.C.\n",
      "2016-08-28 07:51:48,012 [MainThread  ] [INFO ]  Processing D435.M.250.F.F.\n",
      "INFO:global:Processing D435.M.250.F.F.\n",
      "2016-08-28 07:51:48,013 [MainThread  ] [INFO ]  Processing D438.M.250.G.G.\n",
      "INFO:global:Processing D438.M.250.G.G.\n",
      "2016-08-28 07:51:48,014 [MainThread  ] [INFO ]  Processing D436.M.250.J.J.\n",
      "INFO:global:Processing D436.M.250.J.J.\n",
      "2016-08-28 07:51:48,016 [MainThread  ] [INFO ]  Processing D376.M.250.E.J.\n",
      "INFO:global:Processing D376.M.250.E.J.\n",
      "2016-08-28 07:51:48,017 [MainThread  ] [INFO ]  Processing D404.M.250.G.H.\n",
      "INFO:global:Processing D404.M.250.G.H.\n",
      "2016-08-28 07:51:48,018 [MainThread  ] [INFO ]  Processing D654.M.250.F.C.\n",
      "INFO:global:Processing D654.M.250.F.C.\n",
      "2016-08-28 07:51:48,019 [MainThread  ] [INFO ]  Processing D633.M.250.G.C.\n",
      "INFO:global:Processing D633.M.250.G.C.\n",
      "2016-08-28 07:51:48,020 [MainThread  ] [INFO ]  Processing D345.M.250.J.E.\n",
      "INFO:global:Processing D345.M.250.J.E.\n",
      "2016-08-28 07:51:48,021 [MainThread  ] [INFO ]  Processing D438.M.250.G.I.\n",
      "INFO:global:Processing D438.M.250.G.I.\n",
      "2016-08-28 07:51:48,022 [MainThread  ] [INFO ]  Processing D404.M.250.G.G.\n",
      "INFO:global:Processing D404.M.250.G.G.\n",
      "2016-08-28 07:51:48,023 [MainThread  ] [INFO ]  Processing D301.M.250.I.B.\n",
      "INFO:global:Processing D301.M.250.I.B.\n",
      "2016-08-28 07:51:48,024 [MainThread  ] [INFO ]  Processing D398.M.250.E.G.\n",
      "INFO:global:Processing D398.M.250.E.G.\n",
      "2016-08-28 07:51:48,025 [MainThread  ] [INFO ]  Processing D699.M.250.A.A.\n",
      "INFO:global:Processing D699.M.250.A.A.\n",
      "2016-08-28 07:51:48,027 [MainThread  ] [INFO ]  Processing D398.M.250.E.H.\n",
      "INFO:global:Processing D398.M.250.E.H.\n",
      "2016-08-28 07:51:48,028 [MainThread  ] [INFO ]  Processing D413.M.250.A.D.\n",
      "INFO:global:Processing D413.M.250.A.D.\n",
      "2016-08-28 07:51:48,029 [MainThread  ] [INFO ]  Processing D346.M.250.H.B.\n",
      "INFO:global:Processing D346.M.250.H.B.\n",
      "2016-08-28 07:51:48,030 [MainThread  ] [INFO ]  Processing D321.M.250.F.A.\n",
      "INFO:global:Processing D321.M.250.F.A.\n",
      "2016-08-28 07:51:48,031 [MainThread  ] [INFO ]  Processing D431.M.250.H.B.\n",
      "INFO:global:Processing D431.M.250.H.B.\n",
      "2016-08-28 07:51:48,032 [MainThread  ] [INFO ]  Processing D413.M.250.A.A.\n",
      "INFO:global:Processing D413.M.250.A.A.\n",
      "2016-08-28 07:51:48,033 [MainThread  ] [INFO ]  Processing D321.M.250.F.C.\n",
      "INFO:global:Processing D321.M.250.F.C.\n",
      "2016-08-28 07:51:48,034 [MainThread  ] [INFO ]  Processing D389.M.250.H.F.\n",
      "INFO:global:Processing D389.M.250.H.F.\n",
      "2016-08-28 07:51:48,035 [MainThread  ] [INFO ]  Processing D695.M.250.C.E.\n",
      "INFO:global:Processing D695.M.250.C.E.\n",
      "2016-08-28 07:51:48,036 [MainThread  ] [INFO ]  Processing D393.M.250.F.A.\n",
      "INFO:global:Processing D393.M.250.F.A.\n",
      "2016-08-28 07:51:48,038 [MainThread  ] [INFO ]  Processing D345.M.250.J.A.\n",
      "INFO:global:Processing D345.M.250.J.A.\n",
      "2016-08-28 07:51:48,039 [MainThread  ] [INFO ]  Processing D385.M.250.G.D.\n",
      "INFO:global:Processing D385.M.250.G.D.\n",
      "2016-08-28 07:51:48,040 [MainThread  ] [INFO ]  Processing D360.M.250.F.D.\n",
      "INFO:global:Processing D360.M.250.F.D.\n",
      "2016-08-28 07:51:48,041 [MainThread  ] [INFO ]  Processing D331.M.250.F.B.\n",
      "INFO:global:Processing D331.M.250.F.B.\n",
      "2016-08-28 07:51:48,042 [MainThread  ] [INFO ]  Processing D435.M.250.F.A.\n",
      "INFO:global:Processing D435.M.250.F.A.\n",
      "2016-08-28 07:51:48,043 [MainThread  ] [INFO ]  Processing D435.M.250.F.C.\n",
      "INFO:global:Processing D435.M.250.F.C.\n",
      "2016-08-28 07:51:48,044 [MainThread  ] [INFO ]  Processing D435.M.250.F.E.\n",
      "INFO:global:Processing D435.M.250.F.E.\n",
      "2016-08-28 07:51:48,045 [MainThread  ] [INFO ]  Processing D366.M.250.I.B.\n",
      "INFO:global:Processing D366.M.250.I.B.\n",
      "2016-08-28 07:51:48,046 [MainThread  ] [INFO ]  Processing D632.M.250.I.G.\n",
      "INFO:global:Processing D632.M.250.I.G.\n",
      "2016-08-28 07:51:48,047 [MainThread  ] [INFO ]  Processing D345.M.250.J.H.\n",
      "INFO:global:Processing D345.M.250.J.H.\n",
      "2016-08-28 07:51:48,049 [MainThread  ] [INFO ]  Processing D408.M.250.C.H.\n",
      "INFO:global:Processing D408.M.250.C.H.\n",
      "2016-08-28 07:51:48,050 [MainThread  ] [INFO ]  Processing D307.M.250.B.B.\n",
      "INFO:global:Processing D307.M.250.B.B.\n",
      "2016-08-28 07:51:48,051 [MainThread  ] [INFO ]  Processing D366.M.250.I.D.\n",
      "INFO:global:Processing D366.M.250.I.D.\n",
      "2016-08-28 07:51:48,052 [MainThread  ] [INFO ]  Processing D422.M.250.C.A.\n",
      "INFO:global:Processing D422.M.250.C.A.\n",
      "2016-08-28 07:51:48,053 [MainThread  ] [INFO ]  Processing D354.M.250.C.C.\n",
      "INFO:global:Processing D354.M.250.C.C.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "D413.M.250.A.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D391.M.250.H.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D343.M.250.C.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D321.M.250.F.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D408.M.250.C.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D438.M.250.G.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D436.M.250.J.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D376.M.250.E.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D404.M.250.G.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D438.M.250.G.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D404.M.250.G.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D301.M.250.I.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D398.M.250.E.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D699.M.250.A.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D398.M.250.E.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D346.M.250.H.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D321.M.250.F.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D321.M.250.F.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D389.M.250.H.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D385.M.250.G.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D360.M.250.F.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D331.M.250.F.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D408.M.250.C.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D307.M.250.B.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:48,055 [MainThread  ] [INFO ]  Processing D654.M.250.F.A.\n",
      "INFO:global:Processing D654.M.250.F.A.\n",
      "2016-08-28 07:51:48,056 [MainThread  ] [INFO ]  Processing D347.M.250.B.D.\n",
      "INFO:global:Processing D347.M.250.B.D.\n",
      "2016-08-28 07:51:48,057 [MainThread  ] [INFO ]  Processing D354.M.250.C.E.\n",
      "INFO:global:Processing D354.M.250.C.E.\n",
      "2016-08-28 07:51:48,058 [MainThread  ] [INFO ]  Processing D683.M.250.J.C.\n",
      "INFO:global:Processing D683.M.250.J.C.\n",
      "2016-08-28 07:51:48,059 [MainThread  ] [INFO ]  Processing D426.M.250.A.D.\n",
      "INFO:global:Processing D426.M.250.A.D.\n",
      "2016-08-28 07:51:48,060 [MainThread  ] [INFO ]  Processing D436.M.250.J.I.\n",
      "INFO:global:Processing D436.M.250.J.I.\n",
      "2016-08-28 07:51:48,061 [MainThread  ] [INFO ]  Processing D408.M.250.C.J.\n",
      "INFO:global:Processing D408.M.250.C.J.\n",
      "2016-08-28 07:51:48,062 [MainThread  ] [INFO ]  Processing D407.M.250.B.I.\n",
      "INFO:global:Processing D407.M.250.B.I.\n",
      "2016-08-28 07:51:48,064 [MainThread  ] [INFO ]  Processing D408.M.250.C.I.\n",
      "INFO:global:Processing D408.M.250.C.I.\n",
      "2016-08-28 07:51:48,065 [MainThread  ] [INFO ]  Processing D442.M.250.G.H.\n",
      "INFO:global:Processing D442.M.250.G.H.\n",
      "2016-08-28 07:51:48,066 [MainThread  ] [INFO ]  Processing D393.M.250.F.C.\n",
      "INFO:global:Processing D393.M.250.F.C.\n",
      "2016-08-28 07:51:48,067 [MainThread  ] [INFO ]  Processing D633.M.250.G.A.\n",
      "INFO:global:Processing D633.M.250.G.A.\n",
      "2016-08-28 07:51:48,068 [MainThread  ] [INFO ]  Processing D671.M.250.G.H.\n",
      "INFO:global:Processing D671.M.250.G.H.\n",
      "2016-08-28 07:51:48,069 [MainThread  ] [INFO ]  Processing D632.M.250.I.F.\n",
      "INFO:global:Processing D632.M.250.I.F.\n",
      "2016-08-28 07:51:48,070 [MainThread  ] [INFO ]  Processing D654.M.250.F.E.\n",
      "INFO:global:Processing D654.M.250.F.E.\n",
      "2016-08-28 07:51:48,071 [MainThread  ] [INFO ]  Processing D366.M.250.I.I.\n",
      "INFO:global:Processing D366.M.250.I.I.\n",
      "2016-08-28 07:51:48,073 [MainThread  ] [INFO ]  Processing D307.M.250.B.A.\n",
      "INFO:global:Processing D307.M.250.B.A.\n",
      "2016-08-28 07:51:48,074 [MainThread  ] [INFO ]  Processing D422.M.250.C.H.\n",
      "INFO:global:Processing D422.M.250.C.H.\n",
      "2016-08-28 07:51:48,075 [MainThread  ] [INFO ]  Processing D695.M.250.C.F.\n",
      "INFO:global:Processing D695.M.250.C.F.\n",
      "2016-08-28 07:51:48,076 [MainThread  ] [INFO ]  Processing D404.M.250.G.F.\n",
      "INFO:global:Processing D404.M.250.G.F.\n",
      "2016-08-28 07:51:48,077 [MainThread  ] [INFO ]  Processing D413.M.250.A.G.\n",
      "INFO:global:Processing D413.M.250.A.G.\n",
      "2016-08-28 07:51:48,078 [MainThread  ] [INFO ]  Processing D428.M.250.E.E.\n",
      "INFO:global:Processing D428.M.250.E.E.\n",
      "2016-08-28 07:51:48,079 [MainThread  ] [INFO ]  Processing D434.M.250.B.J.\n",
      "INFO:global:Processing D434.M.250.B.J.\n",
      "2016-08-28 07:51:48,080 [MainThread  ] [INFO ]  Processing D313.M.250.E.E.\n",
      "INFO:global:Processing D313.M.250.E.E.\n",
      "2016-08-28 07:51:48,081 [MainThread  ] [INFO ]  Processing D324.M.250.E.G.\n",
      "INFO:global:Processing D324.M.250.E.G.\n",
      "2016-08-28 07:51:48,082 [MainThread  ] [INFO ]  Processing D350.M.250.A.D.\n",
      "INFO:global:Processing D350.M.250.A.D.\n",
      "2016-08-28 07:51:48,083 [MainThread  ] [INFO ]  Processing D435.M.250.F.G.\n",
      "INFO:global:Processing D435.M.250.F.G.\n",
      "2016-08-28 07:51:48,085 [MainThread  ] [INFO ]  Processing D324.M.250.E.I.\n",
      "INFO:global:Processing D324.M.250.E.I.\n",
      "2016-08-28 07:51:48,086 [MainThread  ] [INFO ]  Processing D422.M.250.C.B.\n",
      "INFO:global:Processing D422.M.250.C.B.\n",
      "2016-08-28 07:51:48,087 [MainThread  ] [INFO ]  Processing D324.M.250.E.A.\n",
      "INFO:global:Processing D324.M.250.E.A.\n",
      "2016-08-28 07:51:48,088 [MainThread  ] [INFO ]  Processing D695.M.250.C.H.\n",
      "INFO:global:Processing D695.M.250.C.H.\n",
      "2016-08-28 07:51:48,089 [MainThread  ] [INFO ]  Processing D413.M.250.A.C.\n",
      "INFO:global:Processing D413.M.250.A.C.\n",
      "2016-08-28 07:51:48,090 [MainThread  ] [INFO ]  Processing D307.M.250.B.D.\n",
      "INFO:global:Processing D307.M.250.B.D.\n",
      "2016-08-28 07:51:48,091 [MainThread  ] [INFO ]  Processing D311.M.250.I.C.\n",
      "INFO:global:Processing D311.M.250.I.C.\n",
      "2016-08-28 07:51:48,092 [MainThread  ] [INFO ]  Processing D311.M.250.I.D.\n",
      "INFO:global:Processing D311.M.250.I.D.\n",
      "2016-08-28 07:51:48,093 [MainThread  ] [INFO ]  Processing D436.M.250.J.G.\n",
      "INFO:global:Processing D436.M.250.J.G.\n",
      "2016-08-28 07:51:48,095 [MainThread  ] [INFO ]  Processing D431.M.250.H.E.\n",
      "INFO:global:Processing D431.M.250.H.E.\n",
      "2016-08-28 07:51:48,096 [MainThread  ] [INFO ]  Processing D695.M.250.C.G.\n",
      "INFO:global:Processing D695.M.250.C.G.\n",
      "2016-08-28 07:51:48,097 [MainThread  ] [INFO ]  Processing D346.M.250.H.C.\n",
      "INFO:global:Processing D346.M.250.H.C.\n",
      "2016-08-28 07:51:48,098 [MainThread  ] [INFO ]  Processing D331.M.250.F.C.\n",
      "INFO:global:Processing D331.M.250.F.C.\n",
      "2016-08-28 07:51:48,099 [MainThread  ] [INFO ]  Processing D404.M.250.G.J.\n",
      "INFO:global:Processing D404.M.250.G.J.\n",
      "2016-08-28 07:51:48,100 [MainThread  ] [INFO ]  Processing D699.M.250.A.J.\n",
      "INFO:global:Processing D699.M.250.A.J.\n",
      "2016-08-28 07:51:48,101 [MainThread  ] [INFO ]  Processing D426.M.250.A.J.\n",
      "INFO:global:Processing D426.M.250.A.J.\n",
      "2016-08-28 07:51:48,102 [MainThread  ] [INFO ]  Processing D357.M.250.I.F.\n",
      "INFO:global:Processing D357.M.250.I.F.\n",
      "2016-08-28 07:51:48,103 [MainThread  ] [INFO ]  Processing D435.M.250.F.D.\n",
      "INFO:global:Processing D435.M.250.F.D.\n",
      "2016-08-28 07:51:48,104 [MainThread  ] [INFO ]  Processing D438.M.250.G.H.\n",
      "INFO:global:Processing D438.M.250.G.H.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "D354.M.250.C.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D347.M.250.B.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D354.M.250.C.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D683.M.250.J.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D436.M.250.J.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D408.M.250.C.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D408.M.250.C.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D442.M.250.G.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D393.M.250.F.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D632.M.250.I.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D654.M.250.F.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D307.M.250.B.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D404.M.250.G.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D428.M.250.E.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D434.M.250.B.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D313.M.250.E.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D350.M.250.A.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.I\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D422.M.250.C.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D413.M.250.A.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D307.M.250.B.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D311.M.250.I.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D436.M.250.J.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D431.M.250.H.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D695.M.250.C.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D346.M.250.H.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D331.M.250.F.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D404.M.250.G.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D699.M.250.A.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D357.M.250.I.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2016-08-28 07:51:48,106 [MainThread  ] [INFO ]  Processing D347.M.250.B.C.\n",
      "INFO:global:Processing D347.M.250.B.C.\n",
      "2016-08-28 07:51:48,107 [MainThread  ] [INFO ]  Processing D633.M.250.G.E.\n",
      "INFO:global:Processing D633.M.250.G.E.\n",
      "2016-08-28 07:51:48,108 [MainThread  ] [INFO ]  Processing D671.M.250.G.J.\n",
      "INFO:global:Processing D671.M.250.G.J.\n",
      "2016-08-28 07:51:48,109 [MainThread  ] [INFO ]  Processing D343.M.250.C.C.\n",
      "INFO:global:Processing D343.M.250.C.C.\n",
      "2016-08-28 07:51:48,111 [MainThread  ] [INFO ]  Processing D366.M.250.I.H.\n",
      "INFO:global:Processing D366.M.250.I.H.\n",
      "2016-08-28 07:51:48,112 [MainThread  ] [INFO ]  Processing D633.M.250.G.D.\n",
      "INFO:global:Processing D633.M.250.G.D.\n",
      "2016-08-28 07:51:48,113 [MainThread  ] [INFO ]  Processing D407.M.250.B.J.\n",
      "INFO:global:Processing D407.M.250.B.J.\n",
      "2016-08-28 07:51:48,114 [MainThread  ] [INFO ]  Processing D400.M.250.B.D.\n",
      "INFO:global:Processing D400.M.250.B.D.\n",
      "2016-08-28 07:51:48,115 [MainThread  ] [INFO ]  Processing D360.M.250.F.F.\n",
      "INFO:global:Processing D360.M.250.F.F.\n",
      "2016-08-28 07:51:48,116 [MainThread  ] [INFO ]  Processing D332.M.250.H.C.\n",
      "INFO:global:Processing D332.M.250.H.C.\n",
      "2016-08-28 07:51:48,117 [MainThread  ] [INFO ]  Processing D426.M.250.A.E.\n",
      "INFO:global:Processing D426.M.250.A.E.\n",
      "2016-08-28 07:51:48,118 [MainThread  ] [INFO ]  Processing D324.M.250.E.C.\n",
      "INFO:global:Processing D324.M.250.E.C.\n",
      "2016-08-28 07:51:48,119 [MainThread  ] [INFO ]  Processing D435.M.250.F.B.\n",
      "INFO:global:Processing D435.M.250.F.B.\n",
      "2016-08-28 07:51:48,121 [MainThread  ] [INFO ]  Processing D442.M.250.G.G.\n",
      "INFO:global:Processing D442.M.250.G.G.\n",
      "2016-08-28 07:51:48,122 [MainThread  ] [INFO ]  Processing D400.M.250.B.H.\n",
      "INFO:global:Processing D400.M.250.B.H.\n",
      "2016-08-28 07:51:48,123 [MainThread  ] [INFO ]  Processing D345.M.250.J.F.\n",
      "INFO:global:Processing D345.M.250.J.F.\n",
      "2016-08-28 07:51:48,124 [MainThread  ] [INFO ]  Processing D301.M.250.I.C.\n",
      "INFO:global:Processing D301.M.250.I.C.\n",
      "2016-08-28 07:51:48,125 [MainThread  ] [INFO ]  Processing D694.M.250.J.H.\n",
      "INFO:global:Processing D694.M.250.J.H.\n",
      "2016-08-28 07:51:48,126 [MainThread  ] [INFO ]  Processing D633.M.250.G.F.\n",
      "INFO:global:Processing D633.M.250.G.F.\n",
      "2016-08-28 07:51:48,127 [MainThread  ] [INFO ]  Processing D366.M.250.I.A.\n",
      "INFO:global:Processing D366.M.250.I.A.\n",
      "2016-08-28 07:51:48,128 [MainThread  ] [INFO ]  Processing D438.M.250.G.J.\n",
      "INFO:global:Processing D438.M.250.G.J.\n",
      "2016-08-28 07:51:48,129 [MainThread  ] [INFO ]  Processing D347.M.250.B.B.\n",
      "INFO:global:Processing D347.M.250.B.B.\n",
      "2016-08-28 07:51:48,130 [MainThread  ] [INFO ]  Processing D436.M.250.J.H.\n",
      "INFO:global:Processing D436.M.250.J.H.\n",
      "2016-08-28 07:51:48,132 [MainThread  ] [INFO ]  Saved processed files to /tmp/tmpx8vS_G/model.\n",
      "INFO:global:Saved processed files to /tmp/tmpx8vS_G/model.\n",
      "2016-08-28 07:51:48,157 [MainThread  ] [INFO ]  Written ROUGE configuration to /tmp/tmptSRRHr/rouge_conf.xml\n",
      "INFO:global:Written ROUGE configuration to /tmp/tmptSRRHr/rouge_conf.xml\n",
      "2016-08-28 07:51:48,158 [MainThread  ] [INFO ]  Running ROUGE with command /home/ubuntu/ROUGE/RELEASE-1.5.5/ROUGE-1.5.5.pl -e /home/ubuntu/ROUGE/RELEASE-1.5.5/data -c 95 -2 -1 -U -r 1000 -n 4 -w 1.2 -a -m /tmp/tmptSRRHr/rouge_conf.xml\n",
      "INFO:global:Running ROUGE with command /home/ubuntu/ROUGE/RELEASE-1.5.5/ROUGE-1.5.5.pl -e /home/ubuntu/ROUGE/RELEASE-1.5.5/data -c 95 -2 -1 -U -r 1000 -n 4 -w 1.2 -a -m /tmp/tmptSRRHr/rouge_conf.xml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "D438.M.250.G.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D347.M.250.B.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D671.M.250.G.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D343.M.250.C.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D407.M.250.B.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.D\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D360.M.250.F.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D332.M.250.H.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D426.M.250.A.E\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D324.M.250.E.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D435.M.250.F.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D442.M.250.G.G\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D400.M.250.B.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D345.M.250.J.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D301.M.250.I.C\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D694.M.250.J.H\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D633.M.250.G.F\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D366.M.250.I.A\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D438.M.250.G.J\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D347.M.250.B.B\n",
      "/home/ubuntu/anaconda2/lib/python2.7/site-packages/pyrouge/utils/file_utils.py ligne 28:\n",
      "D436.M.250.J.H\n",
      "---------------------------------------------\n",
      "1 ROUGE-1 Average_R: 0.19080 (95%-conf.int. 0.16669 - 0.21293)\n",
      "1 ROUGE-1 Average_P: 0.52241 (95%-conf.int. 0.44795 - 0.60512)\n",
      "1 ROUGE-1 Average_F: 0.23933 (95%-conf.int. 0.21542 - 0.26184)\n",
      "---------------------------------------------\n",
      "1 ROUGE-2 Average_R: 0.06422 (95%-conf.int. 0.06094 - 0.06762)\n",
      "1 ROUGE-2 Average_P: 0.32166 (95%-conf.int. 0.22056 - 0.43138)\n",
      "1 ROUGE-2 Average_F: 0.08629 (95%-conf.int. 0.08302 - 0.08957)\n",
      "---------------------------------------------\n",
      "1 ROUGE-3 Average_R: 0.04781 (95%-conf.int. 0.04662 - 0.04919)\n",
      "1 ROUGE-3 Average_P: 0.28577 (95%-conf.int. 0.18275 - 0.39223)\n",
      "1 ROUGE-3 Average_F: 0.06576 (95%-conf.int. 0.06219 - 0.06973)\n",
      "---------------------------------------------\n",
      "1 ROUGE-4 Average_R: 0.04177 (95%-conf.int. 0.04101 - 0.04260)\n",
      "1 ROUGE-4 Average_P: 0.26452 (95%-conf.int. 0.16587 - 0.36532)\n",
      "1 ROUGE-4 Average_F: 0.05769 (95%-conf.int. 0.05431 - 0.06124)\n",
      "---------------------------------------------\n",
      "1 ROUGE-L Average_R: 0.16630 (95%-conf.int. 0.14615 - 0.18481)\n",
      "1 ROUGE-L Average_P: 0.48533 (95%-conf.int. 0.40339 - 0.57492)\n",
      "1 ROUGE-L Average_F: 0.20985 (95%-conf.int. 0.19119 - 0.22772)\n",
      "---------------------------------------------\n",
      "1 ROUGE-W-1.2 Average_R: 0.04654 (95%-conf.int. 0.04199 - 0.05074)\n",
      "1 ROUGE-W-1.2 Average_P: 0.31518 (95%-conf.int. 0.24441 - 0.39200)\n",
      "1 ROUGE-W-1.2 Average_F: 0.07203 (95%-conf.int. 0.06653 - 0.07739)\n",
      "---------------------------------------------\n",
      "1 ROUGE-S* Average_R: 0.04380 (95%-conf.int. 0.03628 - 0.05096)\n",
      "1 ROUGE-S* Average_P: 0.35415 (95%-conf.int. 0.25334 - 0.46630)\n",
      "1 ROUGE-S* Average_F: 0.06141 (95%-conf.int. 0.05152 - 0.07099)\n",
      "---------------------------------------------\n",
      "1 ROUGE-SU* Average_R: 0.04448 (95%-conf.int. 0.03687 - 0.05173)\n",
      "1 ROUGE-SU* Average_P: 0.35534 (95%-conf.int. 0.25477 - 0.46730)\n",
      "1 ROUGE-SU* Average_F: 0.06235 (95%-conf.int. 0.05239 - 0.07202)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyrouge import Rouge155\n",
    "\n",
    "r = Rouge155()\n",
    "r.system_dir = '/home/ubuntu/summarization_query_oriented/DUC/duc2005_summary_system/first_try'\n",
    "r.model_dir = '/home/ubuntu/summarization_query_oriented/DUC/duc2005_summary_model'\n",
    "r.system_filename_pattern = 'd(\\d+)[a-z]'\n",
    "r.model_filename_pattern = 'D#ID#.M.250.[A-Z].[A-Z]'\n",
    "\n",
    "output = r.convert_and_evaluate()\n",
    "print(output)\n",
    "output_dict = r.output_to_dict(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'rouge_3_f_score_cb': 0.06219, u'rouge_3_f_score_ce': 0.06973, u'rouge_1_precision': 0.52241, u'rouge_l_precision_cb': 0.40339, u'rouge_l_f_score_cb': 0.19119, u'rouge_l_precision_ce': 0.57492, u'rouge_3_recall': 0.04781, u'rouge_3_precision_ce': 0.39223, u'rouge_w_1.2_f_score_ce': 0.07739, u'rouge_s*_f_score_ce': 0.07099, u'rouge_2_precision_ce': 0.43138, u'rouge_2_precision_cb': 0.22056, u'rouge_2_recall': 0.06422, u'rouge_w_1.2_f_score_cb': 0.06653, u'rouge_3_precision_cb': 0.18275, u'rouge_4_f_score_ce': 0.06124, u'rouge_2_precision': 0.32166, u'rouge_1_recall_cb': 0.16669, u'rouge_1_recall_ce': 0.21293, u'rouge_s*_precision': 0.35415, u'rouge_4_f_score_cb': 0.05431, u'rouge_s*_recall_cb': 0.03628, u'rouge_su*_recall_ce': 0.05173, u'rouge_w_1.2_f_score': 0.07203, u'rouge_w_1.2_recall': 0.04654, u'rouge_su*_recall_cb': 0.03687, u'rouge_l_precision': 0.48533, u'rouge_2_recall_cb': 0.06094, u'rouge_s*_precision_ce': 0.4663, u'rouge_2_recall_ce': 0.06762, u'rouge_4_precision_cb': 0.16587, u'rouge_4_precision_ce': 0.36532, u'rouge_1_f_score': 0.23933, u'rouge_s*_recall_ce': 0.05096, u'rouge_4_recall_ce': 0.0426, u'rouge_su*_precision': 0.35534, u'rouge_1_recall': 0.1908, u'rouge_4_recall_cb': 0.04101, u'rouge_l_recall': 0.1663, u'rouge_l_f_score_ce': 0.22772, u'rouge_4_recall': 0.04177, u'rouge_su*_precision_ce': 0.4673, u'rouge_3_recall_cb': 0.04662, u'rouge_su*_recall': 0.04448, u'rouge_3_recall_ce': 0.04919, u'rouge_4_precision': 0.26452, u'rouge_4_f_score': 0.05769, u'rouge_su*_f_score_cb': 0.05239, u'rouge_3_f_score': 0.06576, u'rouge_2_f_score_cb': 0.08302, u'rouge_s*_precision_cb': 0.25334, u'rouge_3_precision': 0.28577, u'rouge_w_1.2_recall_ce': 0.05074, u'rouge_2_f_score_ce': 0.08957, u'rouge_su*_f_score': 0.06235, u'rouge_l_recall_ce': 0.18481, u'rouge_l_recall_cb': 0.14615, u'rouge_s*_recall': 0.0438, u'rouge_su*_precision_cb': 0.25477, u'rouge_w_1.2_precision': 0.31518, u'rouge_w_1.2_precision_cb': 0.24441, u'rouge_w_1.2_precision_ce': 0.392, u'rouge_s*_f_score': 0.06141, u'rouge_1_precision_cb': 0.44795, u'rouge_su*_f_score_ce': 0.07202, u'rouge_1_precision_ce': 0.60512, u'rouge_l_f_score': 0.20985, u'rouge_2_f_score': 0.08629, u'rouge_1_f_score_ce': 0.26184, u'rouge_1_f_score_cb': 0.21542, u'rouge_w_1.2_recall_cb': 0.04199, u'rouge_s*_f_score_cb': 0.05152}\n"
     ]
    }
   ],
   "source": [
    "print output_dict"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}