Script API Azure.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PRENDRE EN COMPTE LES ERREURS D UPLOAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "from wordcloud import WordCloud\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from PIL import Image\n",
    "from collections import OrderedDict\n",
    "import wordcloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Creation du blob de stockage des fichiers traites\n",
    "# !az storage container create --name done --account-name vivatechstorage --account-key Mif4MhreZR9F0NkcIqAa9XQOUohHvb5JHTzlRhSEiLn4C44Xb21Orvpzs6pr+w/D+SivvMvJHNF2sCJKGyUD+g=="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 - Fonctions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_files_list():\n",
    "    files = !az storage blob list --account-name vivatechstorage --account-key Mif4MhreZR9F0NkcIqAa9XQOUohHvb5JHTzlRhSEiLn4C44Xb21Orvpzs6pr+w/D+SivvMvJHNF2sCJKGyUD+g== -c video \n",
    "    files_json = json.loads(files.s)\n",
    "    files_list = []\n",
    "    for count, value in enumerate(files_json):\n",
    "        files_list.append(value['name'])\n",
    "    return files_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def launch_job(iteration, file_name):\n",
    "    #cree asset\n",
    "    asset_name = file_name.partition(\".\")[0]\n",
    "    #!az ams asset delete -n $asset_name -a vivatechams -g vivatech\n",
    "    !az ams asset create -n $asset_name -a vivatechams -g vivatech\n",
    "    print(\"#### asset \" + asset_name + \" created ####\")\n",
    "    \n",
    "    #cree job\n",
    "    job_name = \"job_timestamp_\" + str(iteration)\n",
    "    file_url = \"https://vivatechstorage.blob.core.windows.net/video/\" + file_name\n",
    "    #!az ams job delete --name $job_name --transform-name audioAnalyzerTransform  -a vivatechams -g vivatech \n",
    "    !az ams job start --name $job_name --transform-name audioAnalyzerTransform --files $file_url --output-asset-names $asset_name -a vivatechams -g vivatech --priority High\n",
    "    print(\"####\" + job_name + \" launched ####\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_job_status(files_list):\n",
    "    unfinished_jobs = len(files_list)\n",
    "    #Waiting for jobs to finish\n",
    "    while unfinished_jobs != 0:\n",
    "        try:\n",
    "            jobs = !az ams job list --account-name vivatechams -g vivatech --transform-name audioAnalyzerTransform\n",
    "            jobs_json = json.loads(jobs.s)\n",
    "            finished_jobs = 0\n",
    "            scheduled_jobs = 0\n",
    "            error_jobs = 0\n",
    "            processing_jobs = 0\n",
    "            queued_jobs = 0\n",
    "            status = []\n",
    "            for count, job in enumerate(jobs_json):\n",
    "                if jobs_json[count]['outputs'][0]['state'] == \"Finished\":finished_jobs += 1\n",
    "                status.append((jobs_json[count][\"input\"]['files'][0].partition(\"video/\")[-1], \":\", jobs_json[count][\"outputs\"][0]['progress'], '%', jobs_json[count][\"outputs\"][0]['state']))\n",
    "            print(status , end=\"\\r\")\n",
    "        except:print(\"Error catching status. Please wait...\")\n",
    "        if (jobs_json[count ]['outputs'][0]['state']=='Error') or (jobs_json[count ]['outputs'][0]['state']=='Canceled'):\n",
    "            error_jobs+= 1\n",
    "        if error_jobs > 0 :\n",
    "            break\n",
    "        unfinished_jobs = len(files_list) - finished_jobs\n",
    "        #time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_files():\n",
    "    assets = !az ams asset list --account-name vivatechams -g vivatech\n",
    "    assets_json = json.loads(assets.s)\n",
    "    for count, asset in enumerate(assets_json):\n",
    "        asset_id = \"asset-\" + assets_json[count][\"assetId\"]\n",
    "        asset_name = assets_json[count][\"name\"] + \".json\"\n",
    "        !az storage blob download -n insights.json -f $asset_name -c $asset_id --account-name vivatechstorage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def delete_jobs():\n",
    "    jobs = !az ams job list --account-name vivatechams -g vivatech --transform-name audioAnalyzerTransform\n",
    "    jobs_json = json.loads(jobs.s)\n",
    "\n",
    "    for count, job in enumerate(jobs_json):\n",
    "        job_name = jobs_json[count]['name']\n",
    "        !az ams job delete --name $job_name --transform-name audioAnalyzerTransform  -a vivatechams -g vivatech\n",
    "        print(jobs_json[count]['name'], \" deleted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def delete_assets():\n",
    "    assets = !az ams asset list --account-name vivatechams -g vivatech\n",
    "    assets_json = json.loads(assets.s)\n",
    "    for count, job in enumerate(assets_json):\n",
    "        asset_name = assets_json[count]['name']\n",
    "        !az ams asset delete --name $asset_name -a vivatechams -g vivatech\n",
    "        print(assets_json[count]['name'], \" deleted\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_jsons_per_speaker(files_list):\n",
    "    text_files = []\n",
    "    text = \"\"\n",
    "    for file_count, file in enumerate(files_list):\n",
    "        with open(file.partition(\".\")[0] + \".json\") as f:\n",
    "            data = json.load(f)\n",
    "            \n",
    "        print(file)\n",
    "        for count, sentence in enumerate(data['transcript']):\n",
    "            text  = text + \" \" + data['transcript'][count]['text']\n",
    "\n",
    "        text_files.append(text)\n",
    "    \n",
    "    return text_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_jsons(files_list):\n",
    "    text_files = []\n",
    "    text = \"\"\n",
    "    for file_count, file in enumerate(files_list):\n",
    "        with open(file.partition(\".\")[0] + \".json\") as f:\n",
    "            data = json.load(f)\n",
    "\n",
    "        for count, sentence in enumerate(data['transcript']):\n",
    "            text  = text + \" \" + data['transcript'][count]['text']\n",
    "\n",
    "        #on sauvegarde dans la liste seulement si le fichier suivant correspond a une autre conference\n",
    "        try:\n",
    "            #if files_list[file_count][:-5] != files_list[file_count+1][:-5]:\n",
    "            if files_list[file_count].partition(\"speaker\")[0] != files_list[file_count+1].partition(\"speaker\")[0]:\n",
    "                text_files.append(text)\n",
    "                text = \"\"\n",
    "        except:text_files.append(text)\n",
    "    \n",
    "    return text_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_words_count(text, file_name, path, granularity):\n",
    "    tokenizer = RegexpTokenizer(r'\\b[^\\d\\W]+\\b')\n",
    "    text_no_punc = tokenizer.tokenize(text)\n",
    "    word_count_dict = Counter(w.lower() for w in text_no_punc if w.lower() not in stopwords.words('english'))\n",
    "    #word_count_dict = word_count_dict.most_common()\n",
    "    word_count_dict = dict(word_count_dict)\n",
    "    word_count_df = pd.DataFrame.from_dict(word_count_dict, orient='index').reset_index()\n",
    "    word_count_df.columns = ['word', 'count']\n",
    "    word_count_df.sort_values(axis=0, by='count', ascending=False, inplace=True)\n",
    "    word_count_df.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    half_day_indicator = ''\n",
    "    #save\n",
    "    if \"_am_\" in file_name: half_day_indicator = '_am'\n",
    "    else: half_day_indicator = '_pm'\n",
    "    \n",
    "    word_count_df.to_excel(\"wordscount/\"+ path + half_day_indicator +  '/wordscount_'+ granularity + '/' + file_name +\".xlsx\")\n",
    "    \n",
    "    return word_count_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SimpleGroupedColorFunc(object):\n",
    "    \"\"\"Create a color function object which assigns EXACT colors\n",
    "       to certain words based on the color to words mapping\n",
    "       Parameters\n",
    "       ----------\n",
    "       color_to_words : dict(str -> list(str))\n",
    "         A dictionary that maps a color to the list of words.\n",
    "       default_color : str\n",
    "         Color that will be assigned to a word that's not a member\n",
    "         of any value from color_to_words.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, color_to_words, default_color):\n",
    "        self.word_to_color = {word: color\n",
    "                              for (color, words) in color_to_words.items()\n",
    "                              for word in words}\n",
    "\n",
    "        self.default_color = default_color\n",
    "\n",
    "    def __call__(self, word, **kwargs):\n",
    "        return self.word_to_color.get(word, self.default_color)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_words_cloud(word_count_dict, file_name, mask, path):\n",
    "    mask = np.array(Image.open(\"masks/\" + mask))\n",
    "    wc = WordCloud(background_color=\"rgba(255, 255, 255, 0)\", mode=\"RGBA\", mask=mask, \n",
    "                      font_path = \"C:/Users/maxbauch/Box Sync/Vivatech/Tests nuages de mots/monserrat_font/Montserrat-Regular.otf\", max_words=100) #, width=3000,height=2500 \n",
    "\n",
    "    wc.generate_from_frequencies(word_count_dict)\n",
    "    \n",
    "    color_to_words = {\n",
    "        '#FFFFFF': list(word_count_dict.keys())\n",
    "    }\n",
    "    default_color = 'white'\n",
    "\n",
    "    # Create a color function with single tone\n",
    "    grouped_color_func = SimpleGroupedColorFunc(color_to_words, default_color)\n",
    "\n",
    "    wc.recolor(color_func=grouped_color_func)\n",
    "    \n",
    "    half_day_indicator = ''\n",
    "    #save\n",
    "    if \"_am_\" in file_name: half_day_indicator = '_am'\n",
    "    else: half_day_indicator = '_pm'\n",
    "    \n",
    "    \n",
    "    wc.to_file(\"wordclouds/\" + path + half_day_indicator + '/' + file_name + '.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_set_files_name(files_list): return list(OrderedDict.fromkeys([x.partition(\"speaker\")[0] for x in files_list]))\n",
    "#     set_files_name = []\n",
    "#     for count, file in enumerate(files_list):\n",
    "#         set_files_name.append(files_list[count].partition(\"speaker\")[0])\n",
    "    \n",
    "    #return list(set(set_files_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def copy_clean_video_container():\n",
    "    key = \"Mif4MhreZR9F0NkcIqAa9XQOUohHvb5JHTzlRhSEiLn4C44Xb21Orvpzs6pr+w/D+SivvMvJHNF2sCJKGyUD+g==\"\n",
    "\n",
    "    #copy\n",
    "    !AzCopy /Source:https://vivatechstorage.blob.core.windows.net/video /Dest:https://vivatechstorage.blob.core.windows.net/done /SourceKey:$key /DestKey:$key /S\n",
    "\n",
    "    #remove and create video container\n",
    "    !az storage container delete --name video --account-name vivatechstorage --account-key Mif4MhreZR9F0NkcIqAa9XQOUohHvb5JHTzlRhSEiLn4C44Xb21Orvpzs6pr+w/D+SivvMvJHNF2sCJKGyUD+g==\n",
    "    !az storage container create --name video --account-name vivatechstorage --account-key Mif4MhreZR9F0NkcIqAa9XQOUohHvb5JHTzlRhSEiLn4C44Xb21Orvpzs6pr+w/D+SivvMvJHNF2sCJKGyUD+g=="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 - Lancement des jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"additionalProperties\": {},\n",
      "  \"alternateId\": null,\n",
      "  \"assetId\": \"b497430f-44b5-4008-b2d2-79a73a5bdad6\",\n",
      "  \"container\": null,\n",
      "  \"created\": \"2018-05-25T16:47:52.807000+00:00\",\n",
      "  \"description\": null,\n",
      "  \"id\": \"/subscriptions/1cb459f7-4b9d-4edf-8577-ee0e24fdf838/resourceGroups/vivatech/providers/Microsoft.Media/mediaservices/vivatechams/assets/25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin\",\n",
      "  \"lastModified\": \"2018-05-25T16:47:52.807000+00:00\",\n",
      "  \"name\": \"25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin\",\n",
      "  \"resourceGroup\": \"vivatech\",\n",
      "  \"storageAccountName\": \"vivatechstorage\",\n",
      "  \"storageEncryptionFormat\": \"None\",\n",
      "  \"type\": \"Microsoft.Media/mediaservices/assets\"\n",
      "}\n",
      "#### asset 25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin created ####\n",
      "{\n",
      "  \"additionalProperties\": {},\n",
      "  \"created\": \"2018-05-25T16:48:01.555718+00:00\",\n",
      "  \"description\": null,\n",
      "  \"id\": \"/subscriptions/1cb459f7-4b9d-4edf-8577-ee0e24fdf838/resourceGroups/vivatech/providers/Microsoft.Media/mediaservices/vivatechams/transforms/audioAnalyzerTransform/jobs/job_timestamp_0\",\n",
      "  \"input\": {\n",
      "    \"additionalProperties\": {},\n",
      "    \"baseUri\": null,\n",
      "    \"files\": [\n",
      "      \"https://vivatechstorage.blob.core.windows.net/video/25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin.mp3\"\n",
      "    ],\n",
      "    \"label\": null,\n",
      "    \"odatatype\": \"#Microsoft.Media.JobInputHttp\"\n",
      "  },\n",
      "  \"lastModified\": \"2018-05-25T16:48:01.555718+00:00\",\n",
      "  \"name\": \"job_timestamp_0\",\n",
      "  \"outputs\": [\n",
      "    {\n",
      "      \"additionalProperties\": {},\n",
      "      \"assetName\": \"25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin\",\n",
      "      \"error\": null,\n",
      "      \"odatatype\": \"#Microsoft.Media.JobOutputAsset\",\n",
      "      \"progress\": 0,\n",
      "      \"state\": \"Queued\"\n",
      "    }\n",
      "  ],\n",
      "  \"priority\": \"High\",\n",
      "  \"resourceGroup\": \"vivatech\",\n",
      "  \"state\": \"Queued\",\n",
      "  \"type\": \"Microsoft.Media/mediaservices/transforms/jobs\"\n",
      "}\n",
      "####job_timestamp_0 launched ####\n",
      "{\n",
      "  \"additionalProperties\": {},\n",
      "  \"alternateId\": null,\n",
      "  \"assetId\": \"6fbf61c8-f8b3-4232-88de-f23f75854819\",\n",
      "  \"container\": null,\n",
      "  \"created\": \"2018-05-25T16:48:09.973000+00:00\",\n",
      "  \"description\": null,\n",
      "  \"id\": \"/subscriptions/1cb459f7-4b9d-4edf-8577-ee0e24fdf838/resourceGroups/vivatech/providers/Microsoft.Media/mediaservices/vivatechams/assets/25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin\",\n",
      "  \"lastModified\": \"2018-05-25T16:48:09.973000+00:00\",\n",
      "  \"name\": \"25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin\",\n",
      "  \"resourceGroup\": \"vivatech\",\n",
      "  \"storageAccountName\": \"vivatechstorage\",\n",
      "  \"storageEncryptionFormat\": \"None\",\n",
      "  \"type\": \"Microsoft.Media/mediaservices/assets\"\n",
      "}\n",
      "#### asset 25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin created ####\n",
      "{\n",
      "  \"additionalProperties\": {},\n",
      "  \"created\": \"2018-05-25T16:48:19.209837+00:00\",\n",
      "  \"description\": null,\n",
      "  \"id\": \"/subscriptions/1cb459f7-4b9d-4edf-8577-ee0e24fdf838/resourceGroups/vivatech/providers/Microsoft.Media/mediaservices/vivatechams/transforms/audioAnalyzerTransform/jobs/job_timestamp_1\",\n",
      "  \"input\": {\n",
      "    \"additionalProperties\": {},\n",
      "    \"baseUri\": null,\n",
      "    \"files\": [\n",
      "      \"https://vivatechstorage.blob.core.windows.net/video/25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin.mp3\"\n",
      "    ],\n",
      "    \"label\": null,\n",
      "    \"odatatype\": \"#Microsoft.Media.JobInputHttp\"\n",
      "  },\n",
      "  \"lastModified\": \"2018-05-25T16:48:19.209837+00:00\",\n",
      "  \"name\": \"job_timestamp_1\",\n",
      "  \"outputs\": [\n",
      "    {\n",
      "      \"additionalProperties\": {},\n",
      "      \"assetName\": \"25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin\",\n",
      "      \"error\": null,\n",
      "      \"odatatype\": \"#Microsoft.Media.JobOutputAsset\",\n",
      "      \"progress\": 0,\n",
      "      \"state\": \"Queued\"\n",
      "    }\n",
      "  ],\n",
      "  \"priority\": \"High\",\n",
      "  \"resourceGroup\": \"vivatech\",\n",
      "  \"state\": \"Queued\",\n",
      "  \"type\": \"Microsoft.Media/mediaservices/transforms/jobs\"\n",
      "}\n",
      "####job_timestamp_1 launched ####\n",
      "{\n",
      "  \"additionalProperties\": {},\n",
      "  \"alternateId\": null,\n",
      "  \"assetId\": \"183939cd-6f01-46f3-8da0-202aad4e754a\",\n",
      "  \"container\": null,\n",
      "  \"created\": \"2018-05-25T16:48:27.423000+00:00\",\n",
      "  \"description\": null,\n",
      "  \"id\": \"/subscriptions/1cb459f7-4b9d-4edf-8577-ee0e24fdf838/resourceGroups/vivatech/providers/Microsoft.Media/mediaservices/vivatechams/assets/25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi\",\n",
      "  \"lastModified\": \"2018-05-25T16:48:27.423000+00:00\",\n",
      "  \"name\": \"25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi\",\n",
      "  \"resourceGroup\": \"vivatech\",\n",
      "  \"storageAccountName\": \"vivatechstorage\",\n",
      "  \"storageEncryptionFormat\": \"None\",\n",
      "  \"type\": \"Microsoft.Media/mediaservices/assets\"\n",
      "}\n",
      "#### asset 25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi created ####\n",
      "{\n",
      "  \"additionalProperties\": {},\n",
      "  \"created\": \"2018-05-25T16:48:35.945527+00:00\",\n",
      "  \"description\": null,\n",
      "  \"id\": \"/subscriptions/1cb459f7-4b9d-4edf-8577-ee0e24fdf838/resourceGroups/vivatech/providers/Microsoft.Media/mediaservices/vivatechams/transforms/audioAnalyzerTransform/jobs/job_timestamp_2\",\n",
      "  \"input\": {\n",
      "    \"additionalProperties\": {},\n",
      "    \"baseUri\": null,\n",
      "    \"files\": [\n",
      "      \"https://vivatechstorage.blob.core.windows.net/video/25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi.mp3\"\n",
      "    ],\n",
      "    \"label\": null,\n",
      "    \"odatatype\": \"#Microsoft.Media.JobInputHttp\"\n",
      "  },\n",
      "  \"lastModified\": \"2018-05-25T16:48:35.945527+00:00\",\n",
      "  \"name\": \"job_timestamp_2\",\n",
      "  \"outputs\": [\n",
      "    {\n",
      "      \"additionalProperties\": {},\n",
      "      \"assetName\": \"25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi\",\n",
      "      \"error\": null,\n",
      "      \"odatatype\": \"#Microsoft.Media.JobOutputAsset\",\n",
      "      \"progress\": 0,\n",
      "      \"state\": \"Queued\"\n",
      "    }\n",
      "  ],\n",
      "  \"priority\": \"High\",\n",
      "  \"resourceGroup\": \"vivatech\",\n",
      "  \"state\": \"Queued\",\n",
      "  \"type\": \"Microsoft.Media/mediaservices/transforms/jobs\"\n",
      "}\n",
      "####job_timestamp_2 launched ####\n"
     ]
    }
   ],
   "source": [
    "#obtenir la liste des fichiers\n",
    "files_list = get_files_list()\n",
    "#delete_jobs()\n",
    "#delete_assets()\n",
    "#Launch jobs\n",
    "for count, file in enumerate(files_list):\n",
    "    #print(\"#### LAUNCHING JOB \" + str(count) + \" ####\")\n",
    "    launch_job(count, files_list[count])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin.mp3', ':', 100, '%', 'Finished'), ('25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin.mp3', ':', 100, '%', 'Finished'), ('25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi.mp3', ':', 100, '%', 'Finished')]]]]\r"
     ]
    }
   ],
   "source": [
    "#print job status\n",
    "print_job_status(files_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"content\": null,\n",
      "  \"deleted\": false,\n",
      "  \"metadata\": {},\n",
      "  \"name\": \"insights.json\",\n",
      "  \"properties\": {\n",
      "    \"appendBlobCommittedBlockCount\": null,\n",
      "    \"blobTier\": null,\n",
      "    \"blobTierChangeTime\": null,\n",
      "    \"blobTierInferred\": false,\n",
      "    \"blobType\": \"BlockBlob\",\n",
      "    \"contentLength\": 50510,\n",
      "    \"contentRange\": \"bytes 0-50509/50510\",\n",
      "    \"contentSettings\": {\n",
      "      \"cacheControl\": null,\n",
      "      \"contentDisposition\": null,\n",
      "      \"contentEncoding\": null,\n",
      "      \"contentLanguage\": null,\n",
      "      \"contentType\": \"application/json\"\n",
      "    },\n",
      "    \"copy\": {\n",
      "      \"completionTime\": null,\n",
      "      \"id\": null,\n",
      "      \"progress\": null,\n",
      "      \"source\": null,\n",
      "      \"status\": null,\n",
      "      \"statusDescription\": null\n",
      "    },\n",
      "    \"deletedTime\": null,\n",
      "    \"etag\": \"\\\"0x8D5C260571E64B1\\\"\",\n",
      "    \"lastModified\": \"2018-05-25T16:55:37+00:00\",\n",
      "    \"lease\": {\n",
      "      \"duration\": null,\n",
      "      \"state\": \"available\",\n",
      "      \"status\": \"unlocked\"\n",
      "    },\n",
      "    \"pageBlobSequenceNumber\": null,\n",
      "    \"remainingRetentionDays\": null,\n",
      "    \"serverEncrypted\": true\n",
      "  },\n",
      "  \"snapshot\": null\n",
      "}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Alive[################################################################]  100.0000%\n",
      "Finished[#############################################################]  100.0000%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"content\": null,\n",
      "  \"deleted\": false,\n",
      "  \"metadata\": {},\n",
      "  \"name\": \"insights.json\",\n",
      "  \"properties\": {\n",
      "    \"appendBlobCommittedBlockCount\": null,\n",
      "    \"blobTier\": null,\n",
      "    \"blobTierChangeTime\": null,\n",
      "    \"blobTierInferred\": false,\n",
      "    \"blobType\": \"BlockBlob\",\n",
      "    \"contentLength\": 124434,\n",
      "    \"contentRange\": \"bytes 0-124433/124434\",\n",
      "    \"contentSettings\": {\n",
      "      \"cacheControl\": null,\n",
      "      \"contentDisposition\": null,\n",
      "      \"contentEncoding\": null,\n",
      "      \"contentLanguage\": null,\n",
      "      \"contentType\": \"application/json\"\n",
      "    },\n",
      "    \"copy\": {\n",
      "      \"completionTime\": null,\n",
      "      \"id\": null,\n",
      "      \"progress\": null,\n",
      "      \"source\": null,\n",
      "      \"status\": null,\n",
      "      \"statusDescription\": null\n",
      "    },\n",
      "    \"deletedTime\": null,\n",
      "    \"etag\": \"\\\"0x8D5C260CD80551D\\\"\",\n",
      "    \"lastModified\": \"2018-05-25T16:58:56+00:00\",\n",
      "    \"lease\": {\n",
      "      \"duration\": null,\n",
      "      \"state\": \"available\",\n",
      "      \"status\": \"unlocked\"\n",
      "    },\n",
      "    \"pageBlobSequenceNumber\": null,\n",
      "    \"remainingRetentionDays\": null,\n",
      "    \"serverEncrypted\": true\n",
      "  },\n",
      "  \"snapshot\": null\n",
      "}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Alive[################################################################]  100.0000%\n",
      "Finished[#############################################################]  100.0000%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"content\": null,\n",
      "  \"deleted\": false,\n",
      "  \"metadata\": {},\n",
      "  \"name\": \"insights.json\",\n",
      "  \"properties\": {\n",
      "    \"appendBlobCommittedBlockCount\": null,\n",
      "    \"blobTier\": null,\n",
      "    \"blobTierChangeTime\": null,\n",
      "    \"blobTierInferred\": false,\n",
      "    \"blobType\": \"BlockBlob\",\n",
      "    \"contentLength\": 87884,\n",
      "    \"contentRange\": \"bytes 0-87883/87884\",\n",
      "    \"contentSettings\": {\n",
      "      \"cacheControl\": null,\n",
      "      \"contentDisposition\": null,\n",
      "      \"contentEncoding\": null,\n",
      "      \"contentLanguage\": null,\n",
      "      \"contentType\": \"application/json\"\n",
      "    },\n",
      "    \"copy\": {\n",
      "      \"completionTime\": null,\n",
      "      \"id\": null,\n",
      "      \"progress\": null,\n",
      "      \"source\": null,\n",
      "      \"status\": null,\n",
      "      \"statusDescription\": null\n",
      "    },\n",
      "    \"deletedTime\": null,\n",
      "    \"etag\": \"\\\"0x8D5C26129CB4C47\\\"\",\n",
      "    \"lastModified\": \"2018-05-25T17:01:31+00:00\",\n",
      "    \"lease\": {\n",
      "      \"duration\": null,\n",
      "      \"state\": \"available\",\n",
      "      \"status\": \"unlocked\"\n",
      "    },\n",
      "    \"pageBlobSequenceNumber\": null,\n",
      "    \"remainingRetentionDays\": null,\n",
      "    \"serverEncrypted\": true\n",
      "  },\n",
      "  \"snapshot\": null\n",
      "}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Alive[################################################################]  100.0000%\n",
      "Finished[#############################################################]  100.0000%\n"
     ]
    }
   ],
   "source": [
    "#download files\n",
    "download_files()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "job_timestamp_0  deleted\n",
      "job_timestamp_1  deleted\n",
      "job_timestamp_2  deleted\n"
     ]
    }
   ],
   "source": [
    "#delete jobs\n",
    "delete_jobs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin  deleted\n",
      "25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin  deleted\n",
      "25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi  deleted\n"
     ]
    }
   ],
   "source": [
    "#delete assets\n",
    "delete_assets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#copy_clean_video_container()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# files_list = ['24_am_1240_1240_stage_one_the_rise_of_the_inte_speaker_bill_mcdermott.mp3', '24_pm_1300_1400_stage_one_the_new_hospitality_speaker_sebastien_missoffe']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 - Génération des viz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extraction des données contenues dans les json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.A - Bilan par speaker"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pas de nuages pour le bilan par speaker. Seulement un words count !"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### changer le path par demi-journee (24_am, 24_pm, 25_am, 25_pm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "half_day_path = \"25\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# files_list.remove('25_pm_1620_1645_startup_stage_the_lure_of_the_usa_speaker_xavier _menchez.mp3')\n",
    "# files_list.remove('25_pm_1510_1535_startup_stage_injecting_a_startup_speaker_chloe_bonnet.mp3')\n",
    "# files_list.remove('25_pm_1400_1430_ceo_forum_a_digital_and_human_speaker_patricia_hollar.mp3')\n",
    "# # # files_list.remove('24_pm_1700_1730_ceo_forum_pioneering_europes_speaker_herve_le_gavrian.mp3')\n",
    "# # # files_list.remove('24_pm_1700_1730_ceo_forum_pioneering_europes_speaker_juliette_jarry.mp3')\n",
    "# # # files_list.remove('24_pm_1735_1800_startup_stage_africas_horizon_eye_speaker_rania_belkahia.mp3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin.mp3',\n",
       " '25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin.mp3',\n",
       " '25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi.mp3']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin.mp3\n",
      "25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin.mp3\n",
      "25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi.mp3\n"
     ]
    }
   ],
   "source": [
    "text_per_speaker = extract_jsons_per_speaker(files_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#obtenir le words count des conférences\n",
    "word_count_dicts_per_speaker = []\n",
    "for count, text in enumerate(text_per_speaker):\n",
    "    word_count_dicts_per_speaker.append(get_words_count(text, files_list[count], half_day_path, \"par_speaker\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.B - Bilan par conférence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#extract jsons\n",
    "texts = extract_jsons(files_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Préparer les masks de nuages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['25_pm_1630_1700_stage_one_ai_meets_gdpr_speaker_isabelle_falque_pierrotin.mp3',\n",
       " '25_pm_1645_1710_startup_stage_startup_survival_man_speaker_fleur_pellerin.mp3',\n",
       " '25_pm_1700_1745_stage_one_startups_french_tech_speaker_mounir_mahjoubi.mp3']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "set_files_name = get_set_files_name(files_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['25_pm_1630_1700_stage_one_ai_meets_gdpr_',\n",
       " '25_pm_1645_1710_startup_stage_startup_survival_man_',\n",
       " '25_pm_1700_1745_stage_one_startups_french_tech_']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set_files_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# masks = [\"ball.png\", \"ball.png\"] #africa\n",
    "# len(set_files_name)==len(masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "masks = []\n",
    "for file in set_files_name:\n",
    "    masks.append(\"ball.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Générer les words count et les nuages "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#obtenir le words count des conférences\n",
    "word_count_dicts = []\n",
    "for count, text in enumerate(texts):\n",
    "    word_count_dicts.append(get_words_count(text, set_files_name[count], half_day_path, \"par_conference\"))\n",
    "    generate_words_cloud(word_count_dicts[count], set_files_name[count],  masks[count], half_day_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.C - Bilan de demi-journee"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### changer le path par demi-journee (24_am, 24_pm, 25_am, 25_pm) et le mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "half_day_path = \"25_pm\"\n",
    "half_day_mask = \"ball.png\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1315_1400_stage_one_roundtable_equality_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1400_1430_ceo_forum_a_digital_and_human_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1405_1425_stage_x_tech_to_the_rescue_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1405_1425_startup_stage_cryptocurrencies_and_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1425_1435_stage_x_biased_algorithms_le_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1425_1450_startup_stage_right_person_right_j_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1430_1500_ceo_forum_the_startup_sector_i_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1435_1500_stage_x_cleaning_up_our_mess_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1450_1530_stage_one_from_startups_to_sca_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1500_1510_stage_x_transition_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1500_1530_ceo_forum_navigating_your_digi_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1510_1535_startup_stage_injecting_a_startup_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1510_1540_stage_x_hacking_the_food_ind_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1530_1550_stage_one_five_mistakes_your_c_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1530_1600_ceo_forum_a_digitally_enhanced_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1535_1545_startup_stage_founder_story_an_unc_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1540_1615_stage_x_tackling_inequality_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1545_1610_startup_stage_no_place_like_home_s_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1550_1630_stage_one_investing_in_unchart_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1600_1630_ceo_forum_bridging_the_gap_for_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1610_1620_startup_stage_transition_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1615_1625_stage_x_digital_creativity_f_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1620_1645_startup_stage_the_lure_of_the_usa_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1625_1635_stage_x_transition_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1630_1700_stage_one_ai_meets_gdpr_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1635_1705_stage_x_ai_for_good_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1645_1710_startup_stage_startup_survival_man_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1700_1745_stage_one_startups_french_tech_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1705_1715_stage_x_crowdsourcing_soluti_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1710_1725_startup_stage_key_takeaways_from_a_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_1715_1735_stage_x_augmented_schools_wh_.xlsx\n",
      "C:\\Users\\maxbauch\\Box Sync\\Vivatech\\script_prod/wordscount/25_pm/wordscount_par_conference/25_pm_goodbye_speech_stage_x_.xlsx\n"
     ]
    }
   ],
   "source": [
    "data = pd.DataFrame()\n",
    "for file in os.listdir(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference'):\n",
    "    print(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference/' + file)\n",
    "    data = data.append(pd.read_excel(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference/' + file))\n",
    "\n",
    "#generation fichier words count\n",
    "data = data.groupby('word', as_index=False)['count'].sum()\n",
    "data.sort_values(axis=0, by='count', ascending=False, inplace=True)\n",
    "data.reset_index(drop=True, inplace=True)\n",
    "half_day_file_name = half_day_path + '_bilan'\n",
    "data.to_excel(\"wordscount/\"+ half_day_path + '/' + half_day_file_name + \".xlsx\")\n",
    "\n",
    "#generation du nuage de mots\n",
    "data_to_dict = data.set_index('word').to_dict()['count']\n",
    "generate_words_cloud(data_to_dict, half_day_file_name,  half_day_mask, half_day_path[:-3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.D - Bilan de journée"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### changer le path par demi-journee (24_am, 24_pm, 25_am, 25_pm) et le mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "half_day_path = \"25_am\"\n",
    "half_day_mask = \"ball.png\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame()\n",
    "for file in os.listdir(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference'):\n",
    "    #print(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference/' + file)\n",
    "    data = data.append(pd.read_excel(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference/' + file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "half_day_path = \"25_pm\"\n",
    "half_day_mask = \"ball.png\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for file in os.listdir(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference'):\n",
    "    #print(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference/' + file)\n",
    "    data = data.append(pd.read_excel(os.getcwd() + \"/wordscount/\" + half_day_path + '/wordscount_par_conference/' + file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#generation fichier words count\n",
    "data = data.groupby('word', as_index=False)['count'].sum()\n",
    "data.sort_values(axis=0, by='count', ascending=False, inplace=True)\n",
    "data.reset_index(drop=True, inplace=True)\n",
    "half_day_file_name = half_day_path + '_bilan_journee'\n",
    "data.to_excel(\"wordscount/\"+ half_day_path + '/' + half_day_file_name + \".xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#generation du nuage de mots\n",
    "data_to_dict = data.set_index('word').to_dict()['count']\n",
    "generate_words_cloud(data_to_dict, half_day_file_name,  half_day_mask, half_day_path[:-3])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}