Abstractive summarization using PEGASUS_XSUM & large model.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Abstractive summarization using PEGASUS_XSUM & large model.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyMVUKVs7CVmRnb3fm6gg3Fj"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":35,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tjtqsJbMX7tX","executionInfo":{"status":"ok","timestamp":1648798330294,"user_tz":-360,"elapsed":5364,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"849dece7-cd5f-401c-d370-0d290c64ebfb"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in links: https://download.pytorch.org/whl/lts/1.8/torch_lts.html\n","Requirement already satisfied: torch==1.8.2+cu111 in /usr/local/lib/python3.7/dist-packages (1.8.2+cu111)\n","Requirement already satisfied: torchvision==0.9.2+cu111 in /usr/local/lib/python3.7/dist-packages (0.9.2+cu111)\n","Requirement already satisfied: torchaudio===0.8.2 in /usr/local/lib/python3.7/dist-packages (0.8.2)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.8.2+cu111) (3.10.0.2)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch==1.8.2+cu111) (1.21.5)\n","Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from torchvision==0.9.2+cu111) (7.1.2)\n"]}],"source":["##Install Pytorch\n","!pip3 install torch==1.8.2+cu111 torchvision==0.9.2+cu111 torchaudio===0.8.2 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html"]},{"cell_type":"code","execution_count":36,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"H6cU3n5sotJ-","executionInfo":{"status":"ok","timestamp":1648798334727,"user_tz":-360,"elapsed":4443,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"f34fd0f6-b1a7-4ad1-b12f-e9abccc03020"},"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.96)\n"]}],"source":["#Install sentencepiece for Pegasus Tokenizer\n","!pip3 install sentencepiece"]},{"cell_type":"code","source":["#Install Transformmers for using Hugging face Pegasus-xsum model\n","!pip3 install transformers"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"KpCpFC-gp9iA","executionInfo":{"status":"ok","timestamp":1648798338599,"user_tz":-360,"elapsed":3880,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"5a3993f3-fd4a-4ec0-8473-e204683c6662"},"execution_count":37,"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.17.0)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n","Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n","Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.49)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.4.0)\n","Requirement already satisfied: tokenizers!=0.11.3,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.12.0)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n"]}]},{"cell_type":"markdown","source":["##Scrape Scientific Article "],"metadata":{"id":"3UhKtDxvINEC"}},{"cell_type":"code","source":["##Import BeautifulSoup libraries\n","from bs4 import BeautifulSoup\n","import requests"],"metadata":{"id":"TyjR4JVSBUBy","executionInfo":{"status":"ok","timestamp":1648798361021,"user_tz":-360,"elapsed":13,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":41,"outputs":[]},{"cell_type":"code","source":["url='https://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html'\n","response=requests.get(url)"],"metadata":{"id":"NCU0mZLOBizV","executionInfo":{"status":"ok","timestamp":1648798991812,"user_tz":-360,"elapsed":410,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":54,"outputs":[]},{"cell_type":"code","source":["#Perse content\n","soup=BeautifulSoup(response.text, 'html')\n","print(soup.prettify())"],"metadata":{"id":"J6X9qsDfBi8N","executionInfo":{"status":"ok","timestamp":1648799001738,"user_tz":-360,"elapsed":874,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"1aaa79e4-6178-4d81-c961-62e297285990"},"execution_count":55,"outputs":[{"output_type":"stream","name":"stdout","text":["<!DOCTYPE html>\n","<html class=\"v2 detail-page\" dir=\"ltr\" itemscope=\"\" itemtype=\"http://schema.org/Blog\" lang=\"en\" xmlns=\"http://www.w3.org/1999/xhtml\" xmlns:b=\"http://www.google.com/2005/gml/b\" xmlns:data=\"http://www.google.com/2005/gml/data\" xmlns:expr=\"http://www.google.com/2005/gml/expr\">\n"," <head>\n","  <link href=\"https://www.blogger.com/static/v1/widgets/1529571102-css_bundle_v2.css\" rel=\"stylesheet\" type=\"text/css\"/>\n","  <title>\n","   Google AI Blog: Auto-generated Summaries in Google Docs\n","  </title>\n","  <meta content=\"width=device-width, height=device-height, minimum-scale=1.0, initial-scale=1.0, user-scalable=0\" name=\"viewport\"/>\n","  <meta content=\"IE=Edge\" http-equiv=\"X-UA-Compatible\"/>\n","  <meta content=\"article\" property=\"og:type\"/>\n","  <meta content=\"Auto-generated Summaries in Google Docs\" property=\"og:title\"/>\n","  <meta content=\"http://2.bp.blogspot.com/-qRz-hnwUdY4/WulXSQ6Rv4I/AAAAAAAATvQ/shk7KsphA0c3E3nUMsDVASqYaH0PhLPNwCK4BGAYYCw/s1600/GoogleAI_logo_horizontal_color_rgb.png\" property=\"og:image\"/>\n","  <meta content=\"en_US\" property=\"og:locale\"/>\n","  <meta content=\"http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html\" property=\"og:url\"/>\n","  <meta content=\"Google AI Blog\" property=\"og:site_name\"/>\n","  <!-- Twitter Card properties -->\n","  <meta content=\"Google AI Blog\" property=\"twitter:site\"/>\n","  <meta content=\"Auto-generated Summaries in Google Docs\" property=\"twitter:title\"/>\n","  <meta content=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s72-c/Auto-generated%20summaries%20.gif\" property=\"twitter:image\"/>\n","  <meta content=\"summary\" name=\"twitter:card\"/>\n","  <meta content=\"@googleai\" name=\"twitter:creator\"/>\n","  <link href=\"https://fonts.googleapis.com/css?family=Roboto:400italic,400,500,500italic,700,700italic\" rel=\"stylesheet\" type=\"text/css\"/>\n","  <link href=\"https://fonts.googleapis.com/icon?family=Material+Icons\" rel=\"stylesheet\"/>\n","  <script src=\"https://ajax.googleapis.com/ajax/libs/jquery/1.11.3/jquery.min.js\" type=\"text/javascript\">\n","  </script>\n","  <!-- End -->\n","  <style id=\"page-skin-1\" type=\"text/css\">\n","   <!--\n","/*\n","<Group description=\"Header Color\" selector=\"header\">\n","<Variable name=\"header.background.color\" description=\"Header Background\"\n","type=\"color\" default=\"#ffffff\"/>\n","</Group>\n","*/\n",".header-outer {\n","border-bottom: 1px solid #e0e0e0;\n","background: #ffffff;\n","}\n","html, .Label h2, #sidebar .rss a, .BlogArchive h2, .FollowByEmail h2.title, .widget .post h2 {\n","font-family: Roboto, sans-serif;\n","}\n",".plusfollowers h2.title, .post h2.title, .widget h2.title {\n","font-family: Roboto, sans-serif;\n","}\n",".widget-item-control {\n","height: 100%;\n","}\n",".widget.Header, #header {\n","position: relative;\n","height: 100%;\n","width: 100%;\n","}\n","}\n",".widget.Header .header-logo1 {\n","float: left;\n","margin-right: 15px;\n","padding-right: 15px;\n","border-right: 1px solid #ddd;\n","}\n",".header-title h1 {\n","color: rgba(39, 37, 37, 0.69);\n","display: inline-block;\n","font-size: 36px;\n","font-family: Roboto, sans-serif;\n","font-weight: normal;\n","line-height: 50px;\n","vertical-align: top;\n","margin-left: -23px;\n","}\n",".header-inner {\n","background-repeat: no-repeat;\n","background-position: right 0px;\n","}\n",".post-author,\n",".byline-author {\n","font-size: 14px;\n","font-weight: normal;\n","color: #757575;\n","color: rgba(0,0,0,.54);\n","}\n",".post-content .img-border {\n","border: 1px solid rgb(235, 235, 235);\n","padding: 4px;\n","}\n",".header-title a {\n","text-decoration: none !important;\n","}\n","pre {\n","border: 1px solid #bbbbbb;\n","margin-top: 1em 0 0 0;\n","padding: 0.99em;\n","overflow-x: auto;\n","overflow-y: auto;\n","}\n","pre, code {\n","font-size: 9pt;\n","background-color: #fafafa;\n","line-height: 125%;\n","font-family: monospace;\n","}\n","pre, code {\n","color: #060;\n","font: 13px/1.54 \"courier new\",courier,monospace;\n","}\n",".header-left .header-logo1 {\n","width: 128px !important;\n","}\n",".header-desc {\n","line-height: 20px;\n","margin-top: -22px;\n","margin-left: 17px;\n","}\n",".fb-custom img, .twitter-custom img, .gplus-share img {\n","cursor: pointer;\n","opacity: 0.54;\n","}\n",".fb-custom img:hover, .twitter-custom img:hover, .gplus-share img:hover {\n","opacity: 0.87;\n","}\n",".fb-like {\n","width: 80px;\n","}\n",".post .share {\n","float: right;\n","}\n","#twitter-share{\n","border: #CCC solid 1px;\n","border-radius: 3px;\n","background-image: -webkit-linear-gradient(top,#ffffff,#dedede);\n","}\n",".twitter-follow {\n","background: url(//3.bp.blogspot.com/-M7uPAxKEeh4/WKrvV1ujKCI/AAAAAAAATZE/cdHhTldtvk4q4ad1Me1XDIgQD9Aul09CACK4B/s1600/twitter-bird.png) no-repeat left center;\n","padding-left: 18px;\n","font: normal normal normal 11px/18px 'Helvetica Neue',Arial,sans-serif;\n","font-weight: bold;\n","text-shadow: 0 1px 0 rgba(255,255,255,.5);\n","cursor: pointer;\n","margin-bottom: 10px;\n","}\n",".twitter-fb {\n","padding-top: 2px;\n","}\n",".fb-follow-button  {\n","background: -webkit-linear-gradient(#4c69ba, #3b55a0);\n","background: -moz-linear-gradient(#4c69ba, #3b55a0);\n","background: linear-gradient(#4c69ba, #3b55a0);\n","border-radius: 2px;\n","height: 18px;\n","padding: 4px 0 0 3px;\n","width: 57px;\n","border: #4c69ba solid 1px;\n","}\n",".fb-follow-button a {\n","text-decoration: none !important;\n","text-shadow: 0 -1px 0 #354c8c;\n","text-align: center;\n","white-space: nowrap;\n","font-size: 11px;\n","color: white;\n","vertical-align: top;\n","}\n",".fb-follow-button a:visited {\n","color: white;\n","}\n",".fb-follow {\n","padding: 0px 5px 3px 0px;\n","width: 14px;\n","vertical-align: bottom;\n","}\n",".gplus-wrapper {\n","margin-top: 3px;\n","display: inline-block;\n","vertical-align: top;\n","}\n",".twitter-custom, .gplus-share {\n","margin-right: 12px;\n","}\n",".fb-follow-button{\n","margin: 10px auto;\n","}\n","sub, sup {\n","line-height: 0;\n","}\n","/** CUSTOM CODE **/\n",".post-content td {\n","width: inherit;\n","}\n","-->\n","  </style>\n","  <style id=\"template-skin-1\" type=\"text/css\">\n","   <!--\n",".header-outer {\n","clear: both;\n","}\n",".header-inner {\n","margin: auto;\n","padding: 0px;\n","}\n",".footer-outer {\n","background: #f5f5f5;\n","clear: both;\n","margin: 0;\n","}\n",".footer-inner {\n","margin: auto;\n","padding: 0px;\n","}\n",".footer-inner-2 {\n","/* Account for right hand column elasticity. */\n","max-width: calc(100% - 248px);\n","}\n",".google-footer-outer {\n","clear: both;\n","}\n",".cols-wrapper, .google-footer-outer, .footer-inner, .header-inner {\n","max-width: 978px;\n","margin-left: auto;\n","margin-right: auto;\n","}\n",".cols-wrapper {\n","margin: auto;\n","clear: both;\n","margin-top: 60px;\n","margin-bottom: 60px;\n","overflow: hidden;\n","}\n",".col-main-wrapper {\n","float: left;\n","width: 100%;\n","}\n",".col-main {\n","margin-right: 278px;\n","max-width: 660px;\n","}\n",".col-right {\n","float: right;\n","width: 248px;\n","margin-left: -278px;\n","}\n","/* Tweaks for layout mode. */\n","body#layout .google-footer-outer {\n","display: none;\n","}\n","body#layout .header-outer, body#layout .footer-outer {\n","background: none;\n","}\n","body#layout .header-inner {\n","height: initial;\n","}\n","body#layout .cols-wrapper {\n","margin-top: initial;\n","margin-bottom: initial;\n","}\n","-->\n","  </style>\n","  <!-- start all head -->\n","  <meta content=\"text/html; charset=utf-8\" http-equiv=\"Content-Type\"/>\n","  <meta content=\"blogger\" name=\"generator\"/>\n","  <link href=\"//ai.googleblog.com/favicon.ico\" rel=\"icon\" type=\"image/x-icon\"/>\n","  <link href=\"http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html\" rel=\"canonical\"/>\n","  <link href=\"http://ai.googleblog.com/feeds/posts/default\" rel=\"alternate\" title=\"Google AI Blog - Atom\" type=\"application/atom+xml\"/>\n","  <link href=\"http://ai.googleblog.com/feeds/posts/default?alt=rss\" rel=\"alternate\" title=\"Google AI Blog - RSS\" type=\"application/rss+xml\"/>\n","  <link href=\"https://www.blogger.com/feeds/8474926331452026626/posts/default\" rel=\"service.post\" title=\"Google AI Blog - Atom\" type=\"application/atom+xml\"/>\n","  <link href=\"http://ai.googleblog.com/feeds/1455778591516274568/comments/default\" rel=\"alternate\" title=\"Google AI Blog - Atom\" type=\"application/atom+xml\"/>\n","  <!--[if IE]><script type=\"text/javascript\" src=\"https://www.blogger.com/static/v1/jsbin/2287435483-ieretrofit.js\"></script>\n","<![endif]-->\n","  <link href=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s16000/Auto-generated%20summaries%20.gif\" rel=\"image_src\"/>\n","  <meta content=\"http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html\" property=\"og:url\"/>\n","  <meta content=\"Auto-generated Summaries in Google Docs\" property=\"og:title\"/>\n","  <meta content=\"Posted by Mohammad Saleh, Software Engineer, Google Research, Brain Team and Anjuli Kannan, Software Engineer, Google Docs    For many of us...\" property=\"og:description\"/>\n","  <meta content=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/w1200-h630-p-k-no-nu/Auto-generated%20summaries%20.gif\" property=\"og:image\"/>\n","  <!--[if IE]> <script> (function() { var html5 = (\"abbr,article,aside,audio,canvas,datalist,details,\" + \"figure,footer,header,hgroup,mark,menu,meter,nav,output,\" + \"progress,section,time,video\").split(','); for (var i = 0; i < html5.length; i++) { document.createElement(html5[i]); } try { document.execCommand('BackgroundImageCache', false, true); } catch(e) {} })(); </script> <![endif]-->\n","  <!-- end all head -->\n","  <base target=\"_self\"/>\n","  <style>\n","   html {\n","        font-family: Roboto, sans-serif;\n","        -moz-osx-font-smoothing: grayscale;\n","        -webkit-font-smoothing: antialiased;\n","      }\n","      body {\n","        padding: 0;\n","        /* This ensures that the scroll bar is always present, which is needed */\n","        /* because content render happens after page load; otherwise the header */\n","        /* would \"bounce\" in-between states. */\n","        min-height: 150%;\n","      }\n","      h2 {\n","        font-size: 16px;\n","      }\n","      h1, h2, h3, h4, h5 {\n","        line-height: 2em;\n","      }\n","      html, h4, h5, h6 {\n","        font-size: 14px;\n","      }\n","      a, a:visited {\n","        color: #4184F3;\n","        text-decoration: none;\n","      }\n","      a:focus, a:hover, a:active {\n","        text-decoration: none;\n","      }\n","      .Header {\n","        margin-top: 15px;\n","      }\n","      /*.Header h1 {\n","        font-size: 32px;\n","        font-weight: 300;\n","        line-height: 32px;\n","        height: 42px;\n","      }*/\n","      .header-inner .Header .titlewrapper {\n","        padding: 0;\n","        margin-top: 30px;\n","      }\n","      .header-inner .Header .descriptionwrapper {\n","        padding: 0;\n","        margin: 0;\n","      }\n","      .cols-wrapper {\n","        margin-top: 56px;\n","      }\n","      .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer {\n","        padding: 0 60px;\n","      }\n","      .header-inner {\n","        height: 256px;\n","        position: relative;\n","      }\n","      html, .header-inner a {\n","        color: #212121;\n","        color: rgba(0,0,0,.87);\n","      }\n","      .header-inner .google-logo {\n","        display: inline-block;\n","        background-size: contain;\n","        z-index: 1;\n","        height: 100px;\n","        overflow: hidden;\n","        margin-top: -23px;\n","        margin-right: 8px;\n","      }\n","      .header-left {\n","        position: absolute;\n","        top: 50%;\n","        -webkit-transform: translateY(-50%);\n","        transform: translateY(-50%);\n","        margin-top: 2px;\n","        width: 100%;\n","        margin-left: -15px;\n","      }\n","      .google-logo {\n","        margin-left: -4px;\n","      }\n","      .google-logo img{\n","        height: 250px;\n","    \tmargin-top: -79px;\n","      }\n","      #google-footer {\n","        position: relative;\n","        font-size: 13px;\n","        list-style: none;\n","        text-align: right;\n","      }\n","      #google-footer a {\n","        color: #444;\n","      }\n","      #google-footer ul {\n","        margin: 0;\n","        padding: 0;\n","        height: 144px;\n","        line-height: 144px;\n","      }\n","      #google-footer ul li {\n","        display: inline;\n","      }\n","      #google-footer ul li:before {\n","        color: #999;\n","        content: \"\\00b7\";\n","        font-weight: bold;\n","        margin: 5px;\n","      }\n","      #google-footer ul li:first-child:before {\n","        content: '';\n","      }\n","      #google-footer .google-logo-dark {\n","        left: 0;\n","        margin-top: -16px;\n","        position: absolute;\n","        top: 50%;\n","      }\n","      /** Sitemap links. **/\n","      .footer-inner-2 {\n","        font-size: 14px;\n","        padding-top: 42px;\n","        padding-bottom: 74px;\n","      }\n","      .footer-inner-2 .HTML h2 {\n","        color: #212121;\n","        color: rgba(0,0,0,.87);\n","        font-size: 14px;\n","        font-weight: 500;\n","        padding-left: 0;\n","        margin: 10px 0;\n","      }\n","      .footer-inner-2 .HTML ul {\n","        font-weight: normal;\n","        list-style: none;\n","        padding-left: 0;\n","      }\n","      .footer-inner-2 .HTML li {\n","        line-height: 24px;\n","        padding: 0;\n","      }\n","      .footer-inner-2 li a {\n","        color: rgba(65,132,243,.87);\n","      }\n","      /** Archive widget. **/\n","      .BlogArchive {\n","        font-size: 13px;\n","        font-weight: normal;\n","      }\n","      .BlogArchive .widget-content {\n","        display: none;\n","      }\n","      .BlogArchive h2, .Label h2 {\n","        color: #4184F3;\n","        text-decoration: none;\n","      }\n","      .BlogArchive .hierarchy li {\n","        display: inline-block;\n","      }\n","      /* Specificity needed here to override widget CSS defaults. */\n","      .BlogArchive #ArchiveList ul li, .BlogArchive #ArchiveList ul ul li {\n","        margin: 0;\n","        padding-left: 0;\n","        text-indent: 0;\n","      }\n","      .BlogArchive .intervalToggle {\n","        cursor: pointer;\n","      }\n","      .BlogArchive .expanded .intervalToggle .new-toggle {\n","        -ms-transform: rotate(180deg);\n","        transform: rotate(180deg);\n","      }\n","      .BlogArchive .new-toggle {\n","        float: right;\n","        padding-top: 3px;\n","        opacity: 0.87;\n","      }\n","      #ArchiveList {\n","        text-transform: uppercase;\n","      }\n","      #ArchiveList .expanded > ul:last-child {\n","        margin-bottom: 16px;\n","      }\n","      #ArchiveList .archivedate {\n","        width: 100%;\n","      }\n","      /* Months */\n","      .BlogArchive .items {\n","        max-width: 150px;\n","        margin-left: -4px;\n","      }\n","      .BlogArchive .expanded .items {\n","        margin-bottom: 10px;\n","        overflow: hidden;\n","      }\n","      .BlogArchive .items > ul {\n","        float: left;\n","        height: 32px;\n","      }\n","      .BlogArchive .items a {\n","        padding: 0 4px;\n","      }\n","      .Label {\n","        font-size: 13px;\n","        font-weight: normal;\n","      }\n","      .sidebar-icon {\n","        display: inline-block;\n","        width: 24px;\n","        height: 24px;\n","        vertical-align: middle;\n","        margin-right: 12px;\n","        margin-top: -1px\n","      }\n","      .Label a {\n","        margin-right: 4px;\n","      }\n","      .Label .widget-content {\n","        display: none;\n","      }\n","      .FollowByEmail {\n","        font-size: 13px;\n","        font-weight: normal;\n","      }\n","      .FollowByEmail h2 {\n","        background: url(\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC\");\n","        background-repeat: no-repeat;\n","        background-position: 0 50%;\n","        text-indent: 30px;\n","      }\n","      .FollowByEmail .widget-content {\n","        display: none;\n","      }\n","      .searchBox input {\n","        border: 1px solid #eee;\n","        color: #212121;\n","        color: rgba(0,0,0,.87);\n","        font-size: 14px;\n","        padding: 8px 8px 8px 40px;\n","        width: 164px;\n","        font-family: Roboto, sans-serif;\n","        background: url(\"https://www.gstatic.com/images/icons/material/system/1x/search_grey600_24dp.png\") 8px center no-repeat;\n","      }\n","      .searchBox ::-webkit-input-placeholder { /* WebKit, Blink, Edge */\n","        color:    rgba(0,0,0,.54);\n","      }\n","      .searchBox :-moz-placeholder { /* Mozilla Firefox 4 to 18 */\n","        color:    #000;\n","        opacity:  0.54;\n","      }\n","      .searchBox ::-moz-placeholder { /* Mozilla Firefox 19+ */\n","        color:    #000;\n","        opacity:  0.54;\n","      }\n","      .searchBox :-ms-input-placeholder { /* Internet Explorer 10-11 */\n","        color:    #757575;\n","      }\n","      .widget-item-control {\n","        margin-top: 0px;\n","      }\n","      .section {\n","        margin: 0;\n","        padding: 0;\n","      }\n","      #sidebar-top {\n","        border: 1px solid #eee;\n","      }\n","      #sidebar-top > div {\n","        margin: 16px 0;\n","      }\n","      .widget ul {\n","        line-height: 1.6;\n","      }\n","      /*main post*/\n","      .post {\n","\t\tdisplay: inline-block;\n","        margin-bottom:30px;\n","      }\n","      #main .post .title {\n","        margin: 0;\n","      }\n","      #main .post .title a {\n","        color: #212121;\n","        color: rgba(0,0,0,.87);\n","        font-weight: normal;\n","        font-size: 24px;\n","      }\n","      #main .post .title a:hover {\n","        text-decoration:none;\n","        color:#4184F3;\n","      }\n","      .message,  #main .post .post-header {\n","        margin: 0;\n","        padding: 0;\n","      }\n","      #main .post .post-header .caption, #main .post .post-header .labels-caption,  #main .post .post-footer .caption, #main .post .post-footer .labels-caption {\n","        color: #444;\n","        font-weight: 500;\n","      }\n","      #main .tr-caption-container td {\n","        text-align: center;\n","      }\n","      #main .post .tr-caption {\n","        color: #757575;\n","        color: rgba(0,0,0,.54);\n","        display: block;\n","        padding-bottom: 20px;\n","\t\tline-height: 1.5;\n","      }\n","      #main .post .tr-caption-container {\n","        line-height: 24px;\n","        padding: 4px 0;\n","        text-align: center;\n","      }\n","      #main .post .post-header .published{\n","        font-size:11px;\n","        font-weight:bold;\n","      }\n","      .post-header .publishdate {\n","        font-size: 17px;\n","        font-weight:normal;\n","        color: #757575;\n","        color: rgba(0,0,0,.54);\n","      }\n","      #main .post .post-footer{\n","        font-size:12px;\n","        padding-bottom: 21px;\n","      }\n","      .label-footer {\n","        margin-bottom: 12px;\n","        margin-top: 12px;\n","      }\n","      .comment-img {\n","        margin-right: 16px;\n","        opacity: 0.54;\n","        vertical-align: middle;\n","      }\n","      #main .post .post-header .published {\n","        margin-bottom: 10px;\n","        margin-top: -2px;\n","      }\n","      .post .post-content {\n","        color: #212121;\n","        color: rgba(0,0,0,.87);\n","        font-size: 15px;\n","        margin: 5px 0 36px 0;\n","        line-height: 20px;\n","      }\n","      .post-body .post-content ul, .post-body .post-content ol {\n","        margin: 16px 0;\n","        padding: 0 48px;\n","      }\n","      .post-summary {\n","        display: none;\n","      }\n","      /* Another old-style caption. */\n","      .post-content div i, .post-content div + i {\n","        font-size: 14px;\n","        font-style: normal;\n","        color: #757575;\n","        color: rgba(0,0,0,.54);\n","        display: block;\n","        line-height: 24px;\n","        margin-bottom: 16px;\n","        text-align: left;\n","      }\n","      /* Another old-style caption (with link) */\n","      .post-content a > i {\n","        color: #4184F3 !important;\n","      }\n","      /* Old-style captions for images. */\n","      .post-content .separator + div:not(.separator) {\n","        margin-top: -16px;\n","      }\n","      /* Capture section headers. */\n","      .post-content br + br + b, .post-content .space + .space + b, .post-content .separator + b {\n","        display: inline-block;\n","      }\n","      /*.post-content li {\n","        line-height: 1.5;\n","      }*/\n","      /* Override all post images/videos to left align. */\n","      .post-content .separator, .post-content > div {\n","        text-align: center;\n","      }\n","      .post-content .separator > a, .post-content .separator > span {\n","        margin-left: 0 !important;\n","      }\n","      .post-content img {\n","        max-width: 100%;\n","        height: auto;\n","        width: auto;\n","      }\n","      .post-content .tr-caption-container img {\n","        margin-bottom: 12px;\n","      }\n","      .post-content iframe, .post-content embed {\n","        max-width: 100%;\n","      }\n","      .post-content .carousel-container {\n","        margin-bottom: 48px;\n","      }\n","      #main .post-content b {\n","        font-weight: 500;\n","      }\n","      /* These are the main paragraph spacing tweaks. */\n","      #main .post-content br {\n","        /*content: ' ';*/\n","        display: block;\n","        padding: 4px;\n","      }\n","      .post-content .space {\n","        display: block;\n","        height: 8px;\n","      }\n","      .post-content iframe + .space, .post-content iframe + br {\n","        padding: 0 !important;\n","      }\n","      #main .post .jump-link {\n","        margin-bottom:10px;\n","      }\n","      .post-content img, .post-content iframe {\n","        margin: 15px 0 20px 0;\n","      }\n","      .post-content > img:first-child, .post-content > iframe:first-child {\n","        margin-top: 0;\n","      }\n","      .col-right .section {\n","        padding: 0 16px;\n","      }\n","      #aside {\n","        background:#fff;\n","        border:1px solid #eee;\n","        border-top: 0;\n","      }\n","      #aside .widget {\n","        margin:0;\n","      }\n","      #aside .widget h2, #ArchiveList .toggle + a.post-count-link {\n","        color: #212121;\n","        color: rgba(0,0,0,.87);\n","        font-weight: 400 !important;\n","        margin: 0;\n","      }\n","      #ArchiveList .toggle {\n","        float: right;\n","      }\n","      #ArchiveList .toggle .material-icons {\n","        padding-top: 4px;\n","      }\n","      #sidebar .tab {\n","        cursor: pointer;\n","      }\n","      #sidebar .tab .arrow {\n","        display: inline-block;\n","        float: right;\n","      }\n","      #sidebar .tab .icon {\n","        display: inline-block;\n","        vertical-align: top;\n","        height: 24px;\n","        width: 24px;\n","        margin-right: 13px;\n","        margin-left: -1px;\n","        margin-top: 1px;\n","        color: #757575;\n","        color: rgba(0,0,0,.54);\n","      }\n","      #sidebar .widget-content > :first-child {\n","        padding-top: 8px;\n","      }\n","      #sidebar .active .tab .arrow {\n","        -ms-transform: rotate(180deg);\n","        transform: rotate(180deg);\n","      }\n","      #sidebar .arrow {\n","        color: #757575;\n","        color: rgba(0,0,0,.54);\n","      }\n","      #sidebar .widget h2 {\n","        font-size: 14px;\n","        line-height: 24px;\n","        display: inline-block;\n","      }\n","      #sidebar .widget .BlogArchive {\n","        padding-bottom: 8px;\n","      }\n","      #sidebar .widget {\n","        border-bottom: 1px solid #eee;\n","        box-shadow: 0px 1px 0 white;\n","        margin-bottom: 0;\n","        padding: 14px 0;\n","        min-height: 20px;\n","      }\n","      #sidebar .widget:last-child {\n","        border-bottom: none;\n","        box-shadow: none;\n","        margin-bottom: 0;\n","      }\n","      #sidebar ul {\n","        margin: 0;\n","        padding: 0;\n","      }\n","      #sidebar ul li {\n","        list-style:none;\n","        padding:0;\n","      }\n","      #sidebar ul li a {\n","        line-height: 32px;\n","      }\n","      #sidebar .archive {\n","        background-image: url(\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAYCAYAAADzoH0MAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAE1JREFUeNpiNDY23s9AAWBioBCwYBM8c+YMVsUmJibEGYBNMS5DaeMFfDYSZQA2v9I3FrB5AZeriI4FmnrBccCT8mhmGs1MwyAzAQQYAKEWG9zm9QFEAAAAAElFTkSuQmCC\");\n","        height: 24px;\n","        line-height: 24px;\n","        padding-left: 30px;\n","      }\n","      #sidebar .labels {\n","        background-image: url(\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAARCAYAAAA7bUf6AAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUxJREFUeNpiNDY23s9AAMycOfM7UF05kHkZmzwTMkdSUhKrIcXFxZy3bt3qBjIN8RrS09PDsHnzZjCNDr58+cKQlpbGDjSoHcg1w2oIyAUODg5gARCNzUVIBrUCuVYYhjx//pzhwIEDYAEQDeJjA1CDWIAGNQK59jBxRuSABbkAlwHIgIeHh2HWrFn/1NTU2oDcvSgBS4wBSC5iArqoCsj1YGIgEyAZVMoEchqlBjEB/cZAiUHg2AEGznpKDAImxOeM////B4VLKtBvEUCngZ1ILKivr3/u6+ubBzJAGZQ9gC5aQoqLgAY8BhkAZL4BuQQkxgXE34A4BuiiZEIuAhrwEGhAEZD5DpzYoIaA2UAM4kQADUrHZRDUgAIg8wO2XAwzbQXQa5OweQ1owB10AyA6gS7BgX1u3ry5397eHow3bdo0EyjGi00tQIABANPgyAH1q1eaAAAAAElFTkSuQmCC\");\n","        height: 20px;\n","        line-height: 20px;\n","        padding-left: 30px;\n","      }\n","      #sidebar .rss a {\n","        background-image: url(\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABIAAAASCAYAAABWzo5XAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAX5JREFUeNqsVDGSgkAQHL2rIiIikohIc/EBRkbwAIwuwgfwAXiAD9AHSI7kEkECRCb6AIyINDLx7K0aa6kT7uq0q7YYtnZ7umdnt7darXbr9Zpegeu61DNNc0dvwCcH4/GYJpMJnc9nOhwOVJbl/4hAAokMECZJQtvt9k+kH7qufyEYDAakqqqYxFdRFBqNRmTbNg2HQ0rTlK7XayvR0xqBdDqdkuM4dE/0ULhYLOh4PHYrknG5XGi/31MYhuL/nkwonM1mlGUZ1XXdrsiyLGEDhY7juJEZ1u5tIixDGdYhmYw+B7CAzPP5nDabjdgIAgCksMX1832/3drtdqPT6SQWapomiGEFNkDEdpDMMAzK81ys/7XYy+XyoQgq2WoURSIJ2iIIgp/WZCCTvFm2wgeAU31aI3Q2GhIDMeB53qPYPIcm5VrxXIOIOxsDMStjVawAc1VViRgN22lNBiuQN3GR+SY07hpOoStmFQAKXRRFY93bnpG+fONfedi+BRgAbkS8Fxp7QQIAAAAASUVORK5CYII=\");\n","      }\n","      #sidebar .subscription a {\n","        background-image: url(\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABEAAAALCAYAAACZIGYHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAUBJREFUeNrMkSGLAlEUhb+ZB4JFi8mx2cz+ApvhRUGTcUCrNqNJDYIi+DO0GUwmQXDK2DSIoGgZcSaIjDrzwrK4ssvChj1w0733O+fdp+m6PozH4yQSCfb7Pa7r8pOi0SjJZBLP8zgej4gAIMvlMuPxmADIYrHger1+C6lUKmo+NJ/NZojb7SZDWiwWo1qtks1msW2bw+HwZdkwDHq9HvV6nel0SqvVYrvdIh6Ph3Qch+VyqRYLhQJSSjRNw7IsfN9XgGKxSLfbJZfL0e/3aTabrFYr7vc7IujLcOh8PqunrNdr0uk0pVKJVCpFJBJRgEajweVyod1uMxgM2O12BAGUgRbU8DV2JpOhVquRz+cRQii3+XxOp9NRN3jVR5LPOp1OjEYjlSL8hclkgmmabDabt4d+m+S30vkD/R/IU4ABAPTZgnZdmG/PAAAAAElFTkSuQmCC\");\n","      }\n","      #sidebar-bottom {\n","        background: #f5f5f5;\n","        border-top:1px solid #eee;\n","      }\n","      #sidebar-bottom .widget {\n","        border-bottom: 1px solid #e0e0e0;\n","        padding: 15px 0;\n","        text-align: center;\n","      }\n","      #sidebar-bottom > div:last-child {\n","        border-bottom: 0;\n","      }\n","      #sidebar-bottom .text {\n","        line-height: 20px;\n","      }\n","      /* Home, forward, and backward pagination. */\n","      .blog-pager {\n","        border-top : 1px #e0e0e0 solid;\n","        padding-top: 10px;\n","        margin-top: 15px;\n","        text-align: right !important;\n","      }\n","      #blog-pager {\n","        margin-botom: 0;\n","        margin-top: -14px;\n","        padding: 16px 0 0 0;\n","      }\n","      #blog-pager a {\n","        display: inline-block;\n","      }\n","      .blog-pager i.disabled {\n","        opacity: 0.2 !important;\n","      }\n","      .blog-pager i {\n","        color: black;\n","        margin-left: 16px;\n","        opacity: 0.54;\n","      }\n","      .blog-pager i:hover, .blog-pager i:active {\n","        opacity: 0.87;\n","      }\n","      #blog-pager-older-link, #blog-pager-newer-link {\n","        float: none;\n","      }\n","      .gplus-profile {\n","        background-color: #fafafa;\n","        border: 1px solid #eee;\n","        overflow: hidden;\n","        width: 212px;\n","      }\n","      .gplus-profile-inner {\n","        margin-left: -1px;\n","        margin-top: -1px;\n","      }\n","      /* Sidebar follow buttons. */\n","      .followgooglewrapper {\n","        padding: 12px 0 0 0;\n","      }\n","      .loading {\n","        visibility: hidden;\n","      }\n","      .detail-page .post-footer .cmt_iframe_holder {\n","        padding-top: 40px !important;\n","      }\n","      /** Desktop **/\n","      @media (max-width: 900px) {\n","        .col-right {\n","          display: none;\n","        }\n","        .col-main {\n","          margin-right: 0;\n","          min-width: initial;\n","        }\n","        .footer-outer {\n","          display: none;\n","        }\n","        .cols-wrapper {\n","          min-width: initial;\n","        }\n","        .google-footer-outer {\n","          background-color: #f5f5f5;\n","        }\n","      }\n","      /** Tablet **/\n","      @media (max-width: 712px) {\n","        .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer {\n","          padding: 0 40px;\n","        }\n","      }\n","      /* An extra breakpoint accommodating for long blog titles. */\n","      @media (max-width: 600px) {\n","        .header-left {\n","          height: 100%;\n","          top: inherit;\n","          margin-top: 0;\n","          -webkit-transform: initial;\n","          transform: initial;\n","        }\n","        .header-title {\n","          margin-top: 88px;\n","        }\n","        .header-inner .google-logo {\n","          height: 40px;\n","          margin-top: 3px;\n","        }\n","        .header-inner .google-logo img {\n","          height: 200px;\n","        }\n","        .header-title h2 {\n","          font-size: 32px;\n","          line-height: 40px;\n","        }\n","\t\t.header-title h1{\n","          font-size: 32px;\n","          line-height: 50px;\n","\t\t}\n","        .header-desc {\n","          bottom: 94px;\n","          position: absolute;\n","          margin-left: 14px;\n","        }\n","      }\n","      /** Mobile/small desktop window; also landscape. **/\n","      @media (max-width: 480px), (max-height: 480px) {\n","        .header-outer, .cols-wrapper, .footer-outer, .google-footer-outer {\n","          padding: 0 16px;\n","        }\n","        .cols-wrapper {\n","          margin-top: 0;\n","        }\n","        .post-header .publishdate, .post .post-content {\n","          font-size: 16px;\n","        }\n","        .post .post-content {\n","          line-height: 28px;\n","          margin-bottom: 30px;\n","        }\n","        .post {\n","          margin-top: 30px;\n","        }\n","        .byline-author {\n","          display: block;\n","          font-size: 12px;\n","          line-height: 24px;\n","          margin-top: 6px;\n","        }\n","        #main .post .title a {\n","          font-weight: 500;\n","          color: #4c4c4c;\n","          color: rgba(0,0,0,.70);\n","        }\n","        #main .post .post-header {\n","          padding-bottom: 12px;\n","        }\n","        #main .post .post-header .published {\n","          margin-bottom: -8px;\n","          margin-top: 3px;\n","        }\n","        .post .read-more {\n","          display: block;\n","          margin-top: 14px;\n","        }\n","        .post .tr-caption {\n","          font-size: 12px;\n","        }\n","        #main .post .title a {\n","          font-size: 20px;\n","          line-height: 30px;\n","        }\n","        .post-content iframe {\n","          /* iframe won't keep aspect ratio when scaled down. */\n","          max-height: 240px;\n","        }\n","        .post-content .separator img, .post-content .tr-caption-container img, .post-content iframe {\n","          max-width: inherit;\n","          width: calc(100% + 32px);\n","        }\n","        /*.post-content table, .post-content td {\n","          width: 100%;\n","        }*/\n","        #blog-pager {\n","          margin: 0;\n","          padding: 16px 0;\n","        }\n","        /** List page tweaks. **/\n","        .list-page .post-original {\n","          display: none;\n","        }\n","        .list-page .post-summary {\n","          display: block;\n","        }\n","        .list-page .comment-container {\n","          display: none;\n","        }\n","        .list-page #blog-pager {\n","          padding-top: 0;\n","          border: 0;\n","          margin-top: -8px;\n","        }\n","        .list-page .label-footer {\n","          display: none;\n","        }\n","        .list-page #main .post .post-footer {\n","          border-bottom: 1px solid #eee;\n","          margin: -16px 0 0 0;\n","          padding: 0 0 20px 0;\n","        }\n","        .list-page .post .share {\n","          display: none;\n","        }\n","        /** Detail page tweaks. **/\n","        .detail-page .post-footer .cmt_iframe_holder {\n","          padding-top: 32px !important;\n","        }\n","        .detail-page .label-footer {\n","          margin-bottom: 0;\n","        }\n","        .detail-page #main .post .post-footer {\n","          padding-bottom: 0;\n","        }\n","        .detail-page #comments {\n","          display: none;\n","        }\n","      }\n","      [data-about-pullquote], [data-is-preview], [data-about-syndication] {\n","        display: none;\n","      }\n","  </style>\n","  <noscript>\n","   <style>\n","    .loading { visibility: visible }\n","   </style>\n","  </noscript>\n","  <script type=\"text/javascript\">\n","   (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){\n","      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),\n","      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)\n","      })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');\n","      ga('create', 'UA-961555-69', 'auto', 'blogger');\n","      ga('blogger.send', 'pageview');\n","  </script>\n","  <link href=\"https://www.blogger.com/dyn-css/authorization.css?targetBlogID=8474926331452026626&amp;zx=1169a2c3-7112-4a1d-aaa1-07e218194c74\" media=\"none\" onload=\"if(media!='all')media='all'\" rel=\"stylesheet\"/>\n","  <noscript>\n","   <link href=\"https://www.blogger.com/dyn-css/authorization.css?targetBlogID=8474926331452026626&amp;zx=1169a2c3-7112-4a1d-aaa1-07e218194c74\" rel=\"stylesheet\"/>\n","  </noscript>\n","  <meta content=\"ca-host-pub-1556223355139109\" name=\"google-adsense-platform-account\"/>\n","  <meta content=\"blogspot.com\" name=\"google-adsense-platform-domain\"/>\n"," </head>\n"," <body>\n","  <script type=\"text/javascript\">\n","   //<![CDATA[\n","      var axel = Math.random() + \"\";\n","      var a = axel * 10000000000000;\n","      document.write('<iframe src=\"https://2542116.fls.doubleclick.net/activityi;src=2542116;type=gblog;cat=googl0;ord=ord=' + a + '?\" width=\"1\" height=\"1\" frameborder=\"0\" style=\"display:none\"></iframe>');\n","      //]]>\n","  </script>\n","  <noscript>\n","   <img alt=\"\" height=\"1\" src=\"https://ad.doubleclick.net/ddm/activity/src=2542116;type=gblog;cat=googl0;ord=1?\" width=\"1\"/>\n","  </noscript>\n","  <!-- Header -->\n","  <div class=\"header-outer\">\n","   <div class=\"header-inner\">\n","    <div class=\"section\" id=\"header\">\n","     <div class=\"widget Header\" data-version=\"1\" id=\"Header1\">\n","      <div class=\"header-left\">\n","       <div class=\"header-title\">\n","        <a class=\"google-logo\" href=\"http://ai.googleblog.com/\">\n","         <img height=\"100\" src=\"//2.bp.blogspot.com/-qRz-hnwUdY4/WulXSQ6Rv4I/AAAAAAAATvQ/shk7KsphA0c3E3nUMsDVASqYaH0PhLPNwCK4BGAYYCw/s1600/GoogleAI_logo_horizontal_color_rgb.png\"/>\n","        </a>\n","        <a href=\"/.\">\n","         <h1>\n","          Blog\n","         </h1>\n","        </a>\n","       </div>\n","       <div class=\"header-desc\">\n","        The latest from Google Research\n","       </div>\n","      </div>\n","     </div>\n","    </div>\n","   </div>\n","  </div>\n","  <!-- all content wrapper start -->\n","  <div class=\"cols-wrapper loading\">\n","   <div class=\"col-main-wrapper\">\n","    <div class=\"col-main\">\n","     <div class=\"section\" id=\"main\">\n","      <div class=\"widget Blog\" data-version=\"1\" id=\"Blog1\">\n","       <div class=\"post\" data-id=\"1455778591516274568\" itemscope=\"\" itemtype=\"http://schema.org/BlogPosting\">\n","        <h2 class=\"title\" itemprop=\"name\">\n","         <a href=\"http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html\" itemprop=\"url\" title=\"Auto-generated Summaries in Google Docs\">\n","          Auto-generated Summaries in Google Docs\n","         </a>\n","        </h2>\n","        <div class=\"post-header\">\n","         <div class=\"published\">\n","          <span class=\"publishdate\" itemprop=\"datePublished\">\n","           Wednesday, March 23, 2022\n","          </span>\n","         </div>\n","        </div>\n","        <div class=\"post-body\">\n","         <div class=\"post-content\" itemprop=\"articleBody\">\n","          <script type=\"text/template\">\n","           <span class=\"byline-author\">Posted by Mohammad Saleh, Software Engineer, Google Research, Brain Team and Anjuli Kannan, Software Engineer, Google Docs\n","</span>\n","\n","<p>\n","For many of us, it can be challenging to keep up with the volume of documents that arrive in our inboxes every day: reports, reviews, briefs, policies and the list goes on. When a new document is received, readers often wish it included a brief summary of the main points in order to effectively prioritize it. However, composing a document summary can be cognitively challenging and time-consuming, especially when a document writer is starting from scratch.\n","</p>\n","\n","<p>\n","To help with this, we recently <a href=\"https://cloud.google.com/blog/products/workspace/delivering-new-innovations-in-google-workspace-with-smart-canvas\">announced</a> that Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available. Today we describe how this was enabled using a machine learning (ML) model that comprehends document text and, when confident, generates a 1-2 sentence natural language description of the document content. However, the document writer maintains full control &#8212; accepting the suggestion as-is, making necessary edits to better capture the document summary or ignoring the suggestion altogether. Readers can also use this section, along with the outline, to understand and navigate the document at a high level. While all users can add summaries, auto-generated suggestions are currently only <a href=\"https://workspaceupdates.googleblog.com/2022/02/new-smart-canvas-features-in-google-docs.html\">available</a> to Google Workspace business customers. Building on <a href=\"https://cloud.google.com/blog/products/g-suite/let-grammar-suggestions-in-google-docs-help-you-write-even-better\">grammar suggestions</a>, <a href=\"https://workspaceupdates.googleblog.com/2020/02/smart-compose-ga.html\">Smart Compose,</a> and <a href=\"https://workspaceupdates.googleblog.com/2020/02/autocorrect-docs-ga.html\">autocorrect</a>, we see this as another valuable step toward improving written communication in the workplace.\n","</p>\n","\n","<table align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto;\"><tbody><tr><td style=\"text-align: center;\"><a href=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s1014/Auto-generated%20summaries%20.gif\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"><img border=\"0\" data-original-height=\"756\" data-original-width=\"1014\" src=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s16000/Auto-generated%20summaries%20.gif\" /></a></td></tr><tr><td class=\"tr-caption\" style=\"text-align: center;\">A blue summary icon appears in the top left corner when a document summary suggestion is available. Document writers can then view, edit, or ignore the suggested document summary.</td></tr></tbody></table>\n","\n","<p>\n","<b>Model Details</b>\n","<br>\n","Automatically generated summaries would not be possible without the tremendous advances in ML for <a href=\"https://en.wikipedia.org/wiki/Natural-language_understanding\">natural language understanding</a> (NLU) and <a href=\"https://en.wikipedia.org/wiki/Natural_language_generation\">natural language generation</a> (NLG) over the past five years, especially with the introduction of <a href=\"https://arxiv.org/abs/1706.03762\">Transformer</a> and <a href=\"https://ai.googleblog.com/2020/06/pegasus-state-of-art-model-for.html\">Pegasus</a>.\n","</p>\n","\n","<p>\n","<a href=\"https://ai.googleblog.com/2016/08/text-summarization-with-tensorflow.html\">Abstractive text summarization</a>, which combines the individually challenging tasks of long document language understanding and generation, has been a long-standing problem in NLU and NLG research. A popular method for combining NLU and NLG is training an ML model using <a href=\"https://arxiv.org/abs/1409.3215\">sequence-to-sequence</a> learning, where the inputs are the document words, and the outputs are the summary words. A neural network then learns to map input tokens to output tokens. <a href=\"https://arxiv.org/abs/1602.06023\">Early applications</a> of the sequence-to-sequence paradigm used <a href=\"https://en.wikipedia.org/wiki/Recurrent_neural_network\">recurrent neural networks</a> (RNNs) for both the encoder and decoder. \n","</p>\n","\n","<p>\n","The introduction of Transformers provided a promising alternative to RNNs because Transformers use <a href=\"https://en.wikipedia.org/wiki/Attention_(machine_learning)\">self-attention</a> to provide better modeling of long input and output dependencies, which is critical in document summarization. Still, these models require large amounts of manually labeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization.\n","</p>\n","\n","<p>\n","The combination of Transformers with self-supervised pre-training (e.g., <a href=\"https://arxiv.org/abs/1810.04805\">BERT</a>, <a href=\"https://openai.com/blog/better-language-models/\">GPT</a>, <a href=\"https://arxiv.org/abs/1910.10683\">T5</a>) led to a major breakthrough in many NLU tasks for which limited labeled data is available. In self-supervised pre-training, a model uses large amounts of unlabeled text to learn general language understanding and generation capabilities. Then, in a subsequent fine-tuning stage, the model learns to apply these abilities on a specific task, such as summarization or question answering.\n","</p>\n","\n","<p>\n","The Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization. In Pegasus pre-training, also called <a href=\"https://arxiv.org/abs/1912.08777\">Gap Sentence Prediction</a> (GSP), full sentences from unlabeled news articles and web documents are masked from the input and the model is required to reconstruct them, conditioned on the remaining unmasked sentences. In particular, GSP attempts to mask sentences that are considered essential to the document through different heuristics. The intuition is to make the pre-training as close as possible to the summarization task. <a href=\"https://ai.googleblog.com/2020/06/pegasus-state-of-art-model-for.html\">Pegasus achieved state-of-the-art results</a> on a varied set of summarization datasets. However, a number of challenges remained to apply this research advancement into a product.\n","</p>\n","\n","<p>\n","<b>Applying Recent Research Advances to Google Docs</b>\n","<br>\n","<ul>\n","\n","  <li><i>Data</i>\n","<p>\n","\n","    Self-supervised pre-training results in an ML model that has general language understanding and generation capabilities, but a subsequent fine-tuning stage is critical for the model to adapt to the application domain. We fine-tuned early versions of our model on a corpus of documents with manually-generated summaries that were consistent with typical use cases.\n","</p>\n","<p>\n","    However, early versions of this corpus suffered from inconsistencies and high variation because they included many types of documents, as well as many ways to write a summary &#8212; e.g., academic abstracts are typically long and detailed, while executive summaries are brief and punchy. This led to a model that was easily confused because it had been trained on so many different types of documents and summaries that it struggled to learn the relationships between any of them.\n","</p>\n","<p>\n","    Fortunately, one of the key findings in the Pegasus work was that an effective pre-training phase required less supervised data in the fine-tuning stage. Some summarization benchmarks required as few as 1,000 fine-tuning examples for Pegasus to match the performance of Transformer baselines that saw 10,000+ supervised examples &#8212; suggesting that one could focus on quality rather than quantity. \n","</p>\n","<p>\n","\n","    We carefully cleaned and filtered the fine-tuning data to contain training examples that were more consistent and represented a coherent definition of summaries. Despite the fact that we reduced the amount of training data, this led to a higher quality model. The key lesson, consistent with recent work in domains like <a href=\"https://ai.googleblog.com/2021/12/training-machine-learning-models-more.html\">dataset distillation</a>, was that it was better to have a smaller, high quality dataset, than a larger, high-variance dataset.</p>\n","    <table align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto;\"><tbody><tr><td style=\"text-align: center;\"><a href=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9Txor_d6GdrEQ5n7fb0Scr1g4Otva3xMxt93QxVKKUss30RRkUoysaVbvKPyvfP40nvdRcr3rPU_x3OJgiG_naIsl3DvCUpxB4rGgUM2IkYWesL8hQqRyaQKJQXDOL3cGGIS8jFtjSp_cU4g8FL2Hgz-Iz85zwl4wLY-TDb3kWuBiXlrPJjRo9E_OVQ/s750/image2.gif\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\"><img border=\"0\" data-original-height=\"750\" data-original-width=\"750\" height=\"400\" src=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9Txor_d6GdrEQ5n7fb0Scr1g4Otva3xMxt93QxVKKUss30RRkUoysaVbvKPyvfP40nvdRcr3rPU_x3OJgiG_naIsl3DvCUpxB4rGgUM2IkYWesL8hQqRyaQKJQXDOL3cGGIS8jFtjSp_cU4g8FL2Hgz-Iz85zwl4wLY-TDb3kWuBiXlrPJjRo9E_OVQ/w400-h400/image2.gif\" width=\"400\" /></a></td></tr></tbody></table>\n","\n","</li>\n","</ul>\n","\n","<ul>\n","  <li><i>Serving</i>\n","<p>\n","    Once we trained the high quality model, we turned to the challenge of serving the model in production. While the Transformer version of the encoder-decoder architecture is the dominant approach to train models for sequence-to-sequence tasks like abstractive summarization, it can be inefficient and impractical to serve in real-world applications. The main inefficiency comes from the Transformer decoder where we generate the output summary token by token through <a href=\"https://en.wikipedia.org/wiki/Autoregressive_model\">autoregressive</a> decoding. The decoding process becomes noticeably slow when summaries get longer since the decoder attends to all previously generated tokens at each step. RNNs are a more efficient architecture for decoding since there is no self-attention with previous tokens as in a Transformer model. \n","</p>\n","\n","<p>\n","    We used <a href=\"https://arxiv.org/abs/1503.02531\">knowledge distillation</a>, which is the process of transferring knowledge from a large model to a smaller more efficient model, to distill the Pegasus model into a <a href=\"https://arxiv.org/abs/1804.09849\">hybrid architecture</a> of a Transformer encoder and an RNN decoder. To improve efficiency we also reduced the number of RNN decoder layers. The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model. To further improve the latency and user experience, we serve the summarization model using <a href=\"https://cloud.google.com/blog/products/ai-machine-learning/cloud-tpu-breaks-scalability-records-for-ai-inference\">TPUs</a>, which provide significant speed ups and allow more requests to be handled by a single machine.\n","</p>\n","</li>\n","</ul>\n","\n","<p>\n","<b>Ongoing Challenges and Next Steps</b>\n","<br>\n","While we are excited by the progress so far, there are a few challenges we are continuing to tackle:\n","</p>\n","<ul>\n","\n","<li><i>Document coverage</i>: Developing a set of documents for the fine-tuning stage was difficult due to the tremendous variety that exists among documents, and the same challenge is true at inference time. Some of the documents our users create (e.g., meeting notes, recipes, lesson plans and resumes) are not suitable for summarization or can be difficult to summarize. Currently, our model only suggests a summary for documents where it is most confident, but we hope to continue broadening this set as our model improves. </li>\n","  <li><i>Evaluation</i>: Abstractive summaries need to capture the essence of a document while being fluent and grammatically correct. A specific document may have many summaries that can be considered correct, and different readers may prefer different ones. This makes it hard to evaluate summaries with automatic metrics only, user feedback and usage statistics will be critical for us to understand and keep improving quality.</li>\n","  <li><i>Long documents</i>: Long documents are some of the toughest documents for the model to summarize because it is harder to capture all the points and abstract them in a single summary, and it can also significantly increase memory usage during training and serving. However, long documents are perhaps most useful for the model to automatically summarize because it can help document writers get a head start on this tedious task. We hope we can apply the latest ML advancements to better address this challenge.\n","</li>\n","</ul>\n","\n","<p>\n","<b>Conclusion</b>\n","<br>\n","Overall, we are thrilled that we can apply recent progress in NLU and NLG to continue assisting users with reading and writing. We hope the automatic suggestions now offered in Google Workspace make it easier for writers to annotate their documents with summaries, and help readers comprehend and navigate documents more easily.</p>\n","\n","<p>\n","<b>Acknowledgements</b>\n","<br>\n","<i>The authors would like to thank the many people across Google that contributed to this work: AJ Motika, Matt Pearson-Beck, Mia Chen, Mahdis Mahdieh, Halit Erdogan, Benjamin Lee, Ali Abdelhadi, Michelle Danoff, Vishnu Sivaji, Sneha Keshav, Aliya Baptista, Karishma Damani, DJ Lick, Yao Zhao, Peter Liu, Aurko Roy, Yonghui Wu, Shubhi Sareen, Andrew Dai, Mekhola Mukherjee, Yinan Wang, Mike Colagrosso, and Behnoosh Hariri.\n","</i>.\n","</p>\n","<span itemprop='author' itemscope='itemscope' itemtype='http://schema.org/Person'>\n","  <meta content='https://plus.google.com/116899029375914044550' itemprop='url'/>\n","</span>\n","          </script>\n","          <noscript>\n","           <span class=\"byline-author\">\n","            Posted by Mohammad Saleh, Software Engineer, Google Research, Brain Team and Anjuli Kannan, Software Engineer, Google Docs\n","           </span>\n","           <p>\n","            For many of us, it can be challenging to keep up with the volume of documents that arrive in our inboxes every day: reports, reviews, briefs, policies and the list goes on. When a new document is received, readers often wish it included a brief summary of the main points in order to effectively prioritize it. However, composing a document summary can be cognitively challenging and time-consuming, especially when a document writer is starting from scratch.\n","           </p>\n","           <p>\n","            To help with this, we recently\n","            <a href=\"https://cloud.google.com/blog/products/workspace/delivering-new-innovations-in-google-workspace-with-smart-canvas\">\n","             announced\n","            </a>\n","            that Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available. Today we describe how this was enabled using a machine learning (ML) model that comprehends document text and, when confident, generates a 1-2 sentence natural language description of the document content. However, the document writer maintains full control — accepting the suggestion as-is, making necessary edits to better capture the document summary or ignoring the suggestion altogether. Readers can also use this section, along with the outline, to understand and navigate the document at a high level. While all users can add summaries, auto-generated suggestions are currently only\n","            <a href=\"https://workspaceupdates.googleblog.com/2022/02/new-smart-canvas-features-in-google-docs.html\">\n","             available\n","            </a>\n","            to Google Workspace business customers. Building on\n","            <a href=\"https://cloud.google.com/blog/products/g-suite/let-grammar-suggestions-in-google-docs-help-you-write-even-better\">\n","             grammar suggestions\n","            </a>\n","            ,\n","            <a href=\"https://workspaceupdates.googleblog.com/2020/02/smart-compose-ga.html\">\n","             Smart Compose,\n","            </a>\n","            and\n","            <a href=\"https://workspaceupdates.googleblog.com/2020/02/autocorrect-docs-ga.html\">\n","             autocorrect\n","            </a>\n","            , we see this as another valuable step toward improving written communication in the workplace.\n","           </p>\n","           <table align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto;\">\n","            <tbody>\n","             <tr>\n","              <td style=\"text-align: center;\">\n","               <a href=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s1014/Auto-generated%20summaries%20.gif\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\">\n","                <img border=\"0\" data-original-height=\"756\" data-original-width=\"1014\" src=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s16000/Auto-generated%20summaries%20.gif\"/>\n","               </a>\n","              </td>\n","             </tr>\n","             <tr>\n","              <td class=\"tr-caption\" style=\"text-align: center;\">\n","               A blue summary icon appears in the top left corner when a document summary suggestion is available. Document writers can then view, edit, or ignore the suggested document summary.\n","              </td>\n","             </tr>\n","            </tbody>\n","           </table>\n","           <p>\n","            <b>\n","             Model Details\n","            </b>\n","            <br/>\n","            Automatically generated summaries would not be possible without the tremendous advances in ML for\n","            <a href=\"https://en.wikipedia.org/wiki/Natural-language_understanding\">\n","             natural language understanding\n","            </a>\n","            (NLU) and\n","            <a href=\"https://en.wikipedia.org/wiki/Natural_language_generation\">\n","             natural language generation\n","            </a>\n","            (NLG) over the past five years, especially with the introduction of\n","            <a href=\"https://arxiv.org/abs/1706.03762\">\n","             Transformer\n","            </a>\n","            and\n","            <a href=\"https://ai.googleblog.com/2020/06/pegasus-state-of-art-model-for.html\">\n","             Pegasus\n","            </a>\n","            .\n","           </p>\n","           <p>\n","            <a href=\"https://ai.googleblog.com/2016/08/text-summarization-with-tensorflow.html\">\n","             Abstractive text summarization\n","            </a>\n","            , which combines the individually challenging tasks of long document language understanding and generation, has been a long-standing problem in NLU and NLG research. A popular method for combining NLU and NLG is training an ML model using\n","            <a href=\"https://arxiv.org/abs/1409.3215\">\n","             sequence-to-sequence\n","            </a>\n","            learning, where the inputs are the document words, and the outputs are the summary words. A neural network then learns to map input tokens to output tokens.\n","            <a href=\"https://arxiv.org/abs/1602.06023\">\n","             Early applications\n","            </a>\n","            of the sequence-to-sequence paradigm used\n","            <a href=\"https://en.wikipedia.org/wiki/Recurrent_neural_network\">\n","             recurrent neural networks\n","            </a>\n","            (RNNs) for both the encoder and decoder.\n","           </p>\n","           <p>\n","            The introduction of Transformers provided a promising alternative to RNNs because Transformers use\n","            <a href=\"https://en.wikipedia.org/wiki/Attention_(machine_learning)\">\n","             self-attention\n","            </a>\n","            to provide better modeling of long input and output dependencies, which is critical in document summarization. Still, these models require large amounts of manually labeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization.\n","           </p>\n","           <p>\n","            The combination of Transformers with self-supervised pre-training (e.g.,\n","            <a href=\"https://arxiv.org/abs/1810.04805\">\n","             BERT\n","            </a>\n","            ,\n","            <a href=\"https://openai.com/blog/better-language-models/\">\n","             GPT\n","            </a>\n","            ,\n","            <a href=\"https://arxiv.org/abs/1910.10683\">\n","             T5\n","            </a>\n","            ) led to a major breakthrough in many NLU tasks for which limited labeled data is available. In self-supervised pre-training, a model uses large amounts of unlabeled text to learn general language understanding and generation capabilities. Then, in a subsequent fine-tuning stage, the model learns to apply these abilities on a specific task, such as summarization or question answering.\n","           </p>\n","           <p>\n","            The Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization. In Pegasus pre-training, also called\n","            <a href=\"https://arxiv.org/abs/1912.08777\">\n","             Gap Sentence Prediction\n","            </a>\n","            (GSP), full sentences from unlabeled news articles and web documents are masked from the input and the model is required to reconstruct them, conditioned on the remaining unmasked sentences. In particular, GSP attempts to mask sentences that are considered essential to the document through different heuristics. The intuition is to make the pre-training as close as possible to the summarization task.\n","            <a href=\"https://ai.googleblog.com/2020/06/pegasus-state-of-art-model-for.html\">\n","             Pegasus achieved state-of-the-art results\n","            </a>\n","            on a varied set of summarization datasets. However, a number of challenges remained to apply this research advancement into a product.\n","           </p>\n","           <p>\n","            <b>\n","             Applying Recent Research Advances to Google Docs\n","            </b>\n","            <br/>\n","           </p>\n","           <ul>\n","            <li>\n","             <i>\n","              Data\n","             </i>\n","             <p>\n","              Self-supervised pre-training results in an ML model that has general language understanding and generation capabilities, but a subsequent fine-tuning stage is critical for the model to adapt to the application domain. We fine-tuned early versions of our model on a corpus of documents with manually-generated summaries that were consistent with typical use cases.\n","             </p>\n","             <p>\n","              However, early versions of this corpus suffered from inconsistencies and high variation because they included many types of documents, as well as many ways to write a summary — e.g., academic abstracts are typically long and detailed, while executive summaries are brief and punchy. This led to a model that was easily confused because it had been trained on so many different types of documents and summaries that it struggled to learn the relationships between any of them.\n","             </p>\n","             <p>\n","              Fortunately, one of the key findings in the Pegasus work was that an effective pre-training phase required less supervised data in the fine-tuning stage. Some summarization benchmarks required as few as 1,000 fine-tuning examples for Pegasus to match the performance of Transformer baselines that saw 10,000+ supervised examples — suggesting that one could focus on quality rather than quantity.\n","             </p>\n","             <p>\n","              We carefully cleaned and filtered the fine-tuning data to contain training examples that were more consistent and represented a coherent definition of summaries. Despite the fact that we reduced the amount of training data, this led to a higher quality model. The key lesson, consistent with recent work in domains like\n","              <a href=\"https://ai.googleblog.com/2021/12/training-machine-learning-models-more.html\">\n","               dataset distillation\n","              </a>\n","              , was that it was better to have a smaller, high quality dataset, than a larger, high-variance dataset.\n","             </p>\n","             <table align=\"center\" cellpadding=\"0\" cellspacing=\"0\" class=\"tr-caption-container\" style=\"margin-left: auto; margin-right: auto;\">\n","              <tbody>\n","               <tr>\n","                <td style=\"text-align: center;\">\n","                 <a href=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9Txor_d6GdrEQ5n7fb0Scr1g4Otva3xMxt93QxVKKUss30RRkUoysaVbvKPyvfP40nvdRcr3rPU_x3OJgiG_naIsl3DvCUpxB4rGgUM2IkYWesL8hQqRyaQKJQXDOL3cGGIS8jFtjSp_cU4g8FL2Hgz-Iz85zwl4wLY-TDb3kWuBiXlrPJjRo9E_OVQ/s750/image2.gif\" imageanchor=\"1\" style=\"margin-left: auto; margin-right: auto;\">\n","                  <img border=\"0\" data-original-height=\"750\" data-original-width=\"750\" height=\"400\" src=\"https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEi9Txor_d6GdrEQ5n7fb0Scr1g4Otva3xMxt93QxVKKUss30RRkUoysaVbvKPyvfP40nvdRcr3rPU_x3OJgiG_naIsl3DvCUpxB4rGgUM2IkYWesL8hQqRyaQKJQXDOL3cGGIS8jFtjSp_cU4g8FL2Hgz-Iz85zwl4wLY-TDb3kWuBiXlrPJjRo9E_OVQ/w400-h400/image2.gif\" width=\"400\"/>\n","                 </a>\n","                </td>\n","               </tr>\n","              </tbody>\n","             </table>\n","            </li>\n","           </ul>\n","           <ul>\n","            <li>\n","             <i>\n","              Serving\n","             </i>\n","             <p>\n","              Once we trained the high quality model, we turned to the challenge of serving the model in production. While the Transformer version of the encoder-decoder architecture is the dominant approach to train models for sequence-to-sequence tasks like abstractive summarization, it can be inefficient and impractical to serve in real-world applications. The main inefficiency comes from the Transformer decoder where we generate the output summary token by token through\n","              <a href=\"https://en.wikipedia.org/wiki/Autoregressive_model\">\n","               autoregressive\n","              </a>\n","              decoding. The decoding process becomes noticeably slow when summaries get longer since the decoder attends to all previously generated tokens at each step. RNNs are a more efficient architecture for decoding since there is no self-attention with previous tokens as in a Transformer model.\n","             </p>\n","             <p>\n","              We used\n","              <a href=\"https://arxiv.org/abs/1503.02531\">\n","               knowledge distillation\n","              </a>\n","              , which is the process of transferring knowledge from a large model to a smaller more efficient model, to distill the Pegasus model into a\n","              <a href=\"https://arxiv.org/abs/1804.09849\">\n","               hybrid architecture\n","              </a>\n","              of a Transformer encoder and an RNN decoder. To improve efficiency we also reduced the number of RNN decoder layers. The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model. To further improve the latency and user experience, we serve the summarization model using\n","              <a href=\"https://cloud.google.com/blog/products/ai-machine-learning/cloud-tpu-breaks-scalability-records-for-ai-inference\">\n","               TPUs\n","              </a>\n","              , which provide significant speed ups and allow more requests to be handled by a single machine.\n","             </p>\n","            </li>\n","           </ul>\n","           <p>\n","            <b>\n","             Ongoing Challenges and Next Steps\n","            </b>\n","            <br/>\n","            While we are excited by the progress so far, there are a few challenges we are continuing to tackle:\n","           </p>\n","           <ul>\n","            <li>\n","             <i>\n","              Document coverage\n","             </i>\n","             : Developing a set of documents for the fine-tuning stage was difficult due to the tremendous variety that exists among documents, and the same challenge is true at inference time. Some of the documents our users create (e.g., meeting notes, recipes, lesson plans and resumes) are not suitable for summarization or can be difficult to summarize. Currently, our model only suggests a summary for documents where it is most confident, but we hope to continue broadening this set as our model improves.\n","            </li>\n","            <li>\n","             <i>\n","              Evaluation\n","             </i>\n","             : Abstractive summaries need to capture the essence of a document while being fluent and grammatically correct. A specific document may have many summaries that can be considered correct, and different readers may prefer different ones. This makes it hard to evaluate summaries with automatic metrics only, user feedback and usage statistics will be critical for us to understand and keep improving quality.\n","            </li>\n","            <li>\n","             <i>\n","              Long documents\n","             </i>\n","             : Long documents are some of the toughest documents for the model to summarize because it is harder to capture all the points and abstract them in a single summary, and it can also significantly increase memory usage during training and serving. However, long documents are perhaps most useful for the model to automatically summarize because it can help document writers get a head start on this tedious task. We hope we can apply the latest ML advancements to better address this challenge.\n","            </li>\n","           </ul>\n","           <p>\n","            <b>\n","             Conclusion\n","            </b>\n","            <br/>\n","            Overall, we are thrilled that we can apply recent progress in NLU and NLG to continue assisting users with reading and writing. We hope the automatic suggestions now offered in Google Workspace make it easier for writers to annotate their documents with summaries, and help readers comprehend and navigate documents more easily.\n","           </p>\n","           <p>\n","            <b>\n","             Acknowledgements\n","            </b>\n","            <br/>\n","            <i>\n","             The authors would like to thank the many people across Google that contributed to this work: AJ Motika, Matt Pearson-Beck, Mia Chen, Mahdis Mahdieh, Halit Erdogan, Benjamin Lee, Ali Abdelhadi, Michelle Danoff, Vishnu Sivaji, Sneha Keshav, Aliya Baptista, Karishma Damani, DJ Lick, Yao Zhao, Peter Liu, Aurko Roy, Yonghui Wu, Shubhi Sareen, Andrew Dai, Mekhola Mukherjee, Yinan Wang, Mike Colagrosso, and Behnoosh Hariri.\n","            </i>\n","            .\n","           </p>\n","           <span itemprop=\"author\" itemscope=\"itemscope\" itemtype=\"http://schema.org/Person\">\n","            <meta content=\"https://plus.google.com/116899029375914044550\" itemprop=\"url\"/>\n","           </span>\n","          </noscript>\n","         </div>\n","        </div>\n","        <div class=\"share\">\n","         <span class=\"twitter-custom social-wrapper\" data-href=\"http://twitter.com/share?text=Google AI Blog:Auto-generated Summaries in Google Docs&amp;url=http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html&amp;via=googleai\">\n","          <img alt=\"Share on Twitter\" height=\"24\" src=\"https://www.gstatic.com/images/icons/material/system/2x/post_twitter_black_24dp.png\" width=\"24\"/>\n","         </span>\n","         <span class=\"fb-custom social-wrapper\" data-href=\"https://www.facebook.com/sharer.php?u=http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html\">\n","          <img alt=\"Share on Facebook\" height=\"24\" src=\"https://www.gstatic.com/images/icons/material/system/2x/post_facebook_black_24dp.png\" width=\"24\"/>\n","         </span>\n","        </div>\n","       </div>\n","       <div class=\"blog-pager\" id=\"blog-pager\">\n","        <a class=\"home-link\" href=\"http://ai.googleblog.com/\">\n","         <i class=\"material-icons\">\n","          \n","         </i>\n","        </a>\n","        <span id=\"blog-pager-newer-link\">\n","         <a class=\"blog-pager-newer-link\" href=\"http://ai.googleblog.com/2022/03/detecting-signs-of-disease-from.html\" id=\"Blog1_blog-pager-newer-link\" title=\"Newer Post\">\n","          <i class=\"material-icons\">\n","           \n","          </i>\n","         </a>\n","        </span>\n","        <span id=\"blog-pager-older-link\">\n","         <a class=\"blog-pager-older-link\" href=\"http://ai.googleblog.com/2022/03/offline-optimization-for-architecting.html\" id=\"Blog1_blog-pager-older-link\" title=\"Older Post\">\n","          <i class=\"material-icons\">\n","           \n","          </i>\n","         </a>\n","        </span>\n","       </div>\n","       <div class=\"clear\">\n","       </div>\n","      </div>\n","     </div>\n","    </div>\n","   </div>\n","   <div class=\"col-right\">\n","    <div class=\"section\" id=\"sidebar-top\">\n","     <div class=\"widget HTML\" data-version=\"1\" id=\"HTML8\">\n","      <div class=\"widget-content\">\n","       <div class=\"searchBox\">\n","        <input placeholder=\"Search blog ...\" title=\"Search This Blog\" type=\"text\"/>\n","       </div>\n","      </div>\n","      <div class=\"clear\">\n","      </div>\n","     </div>\n","    </div>\n","    <div id=\"aside\">\n","     <div class=\"section\" id=\"sidebar\">\n","      <div class=\"widget Label\" data-version=\"1\" id=\"Label1\">\n","       <div class=\"tab\">\n","        <img class=\"sidebar-icon\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAYpJREFUeNrs2aFuwzAQBmAvKRkMKRjZA4QMDJaWFgyMjuzFRg37DIUlA3uFkoGQSaWzJU+tpri5O9+l/zSfdFJlpe59yTmyVedq1PjfcZMZ70NuQnaF8w8htyE/rABtpviXkLcK88c5HhLkMBfgVan43zfFBNGMjHVGT/s55KP2pAvidbGHd+nzKt1RKSLG3rKF1iPFv6UWiPke8i7kEqGdGsI1O+LYVdqJAjgirwkKYD0ytkJBUNbAMvX8V3q9PhUsYvU1sWD8SO/sQvx2ahxOiNoJCSBCoAHYCEQAC4EKICOQASQEOmAS8RcAFxFN5hiIiugpgC3wk9hQAHH/70EBHXUN7IER5EWMiBgo2+nzOKQv9SCAeEM/OQAkhE/ncccFICB87qzQMia5FsJfOui0zMnmRvipU1ormHQuxGTxUsAcCFLxJQBLBLn4UoAFglW8BkATwS5eC6CBEBWvCShBiIvXBkgQRcVbADiI4uKtABSESvGWgB9EzHt3+tNwyO0qa9SoIYtvAQYAqDJhaWWeMecAAAAASUVORK5CYII=\"/>\n","        <h2>\n","         Labels\n","        </h2>\n","        <i class=\"material-icons arrow\">\n","         \n","        </i>\n","       </div>\n","       <div class=\"widget-content list-label-widget-content\">\n","        <ul>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/accessibility\">\n","           accessibility\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ACL\">\n","           ACL\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ACM\">\n","           ACM\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Acoustic%20Modeling\">\n","           Acoustic Modeling\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Adaptive%20Data%20Analysis\">\n","           Adaptive Data Analysis\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ads\">\n","           ads\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/adsense\">\n","           adsense\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/adwords\">\n","           adwords\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Africa\">\n","           Africa\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/AI\">\n","           AI\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/AI%20for%20Social%20Good\">\n","           AI for Social Good\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Algorithms\">\n","           Algorithms\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Android\">\n","           Android\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Android%20Wear\">\n","           Android Wear\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/API\">\n","           API\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/App%20Engine\">\n","           App Engine\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/App%20Inventor\">\n","           App Inventor\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/April%20Fools\">\n","           April Fools\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Art\">\n","           Art\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Audio\">\n","           Audio\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Augmented%20Reality\">\n","           Augmented Reality\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Australia\">\n","           Australia\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Automatic%20Speech%20Recognition\">\n","           Automatic Speech Recognition\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/AutoML\">\n","           AutoML\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Awards\">\n","           Awards\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/BigQuery\">\n","           BigQuery\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Cantonese\">\n","           Cantonese\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Chemistry\">\n","           Chemistry\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/China\">\n","           China\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Chrome\">\n","           Chrome\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Cloud%20Computing\">\n","           Cloud Computing\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Collaboration\">\n","           Collaboration\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Compression\">\n","           Compression\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Computational%20Imaging\">\n","           Computational Imaging\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Computational%20Photography\">\n","           Computational Photography\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Computer%20Science\">\n","           Computer Science\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Computer%20Vision\">\n","           Computer Vision\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/conference\">\n","           conference\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/conferences\">\n","           conferences\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Conservation\">\n","           Conservation\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/correlate\">\n","           correlate\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Course%20Builder\">\n","           Course Builder\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/crowd-sourcing\">\n","           crowd-sourcing\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/CVPR\">\n","           CVPR\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Data%20Center\">\n","           Data Center\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Data%20Discovery\">\n","           Data Discovery\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/data%20science\">\n","           data science\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/datasets\">\n","           datasets\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Deep%20Learning\">\n","           Deep Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/DeepDream\">\n","           DeepDream\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/DeepMind\">\n","           DeepMind\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/distributed%20systems\">\n","           distributed systems\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Diversity\">\n","           Diversity\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Earth%20Engine\">\n","           Earth Engine\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/economics\">\n","           economics\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Education\">\n","           Education\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Electronic%20Commerce%20and%20Algorithms\">\n","           Electronic Commerce and Algorithms\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/electronics\">\n","           electronics\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/EMEA\">\n","           EMEA\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/EMNLP\">\n","           EMNLP\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Encryption\">\n","           Encryption\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/entities\">\n","           entities\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Entity%20Salience\">\n","           Entity Salience\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Environment\">\n","           Environment\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Europe\">\n","           Europe\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Exacycle\">\n","           Exacycle\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Expander\">\n","           Expander\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Faculty%20Institute\">\n","           Faculty Institute\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Faculty%20Summit\">\n","           Faculty Summit\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Flu%20Trends\">\n","           Flu Trends\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Fusion%20Tables\">\n","           Fusion Tables\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/gamification\">\n","           gamification\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Gboard\">\n","           Gboard\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Gmail\">\n","           Gmail\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Accelerated%20Science\">\n","           Google Accelerated Science\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Books\">\n","           Google Books\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Brain\">\n","           Google Brain\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Cloud%20Platform\">\n","           Google Cloud Platform\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Docs\">\n","           Google Docs\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Drive\">\n","           Google Drive\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Genomics\">\n","           Google Genomics\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Maps\">\n","           Google Maps\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Photos\">\n","           Google Photos\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Play%20Apps\">\n","           Google Play Apps\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Science%20Fair\">\n","           Google Science Fair\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Sheets\">\n","           Google Sheets\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Translate\">\n","           Google Translate\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Trips\">\n","           Google Trips\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%20Voice%20Search\">\n","           Google Voice Search\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Google%2B\">\n","           Google+\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Government\">\n","           Government\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/grants\">\n","           grants\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Graph\">\n","           Graph\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Graph%20Mining\">\n","           Graph Mining\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Hardware\">\n","           Hardware\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/HCI\">\n","           HCI\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Health\">\n","           Health\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/High%20Dynamic%20Range%20Imaging\">\n","           High Dynamic Range Imaging\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ICCV\">\n","           ICCV\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ICLR\">\n","           ICLR\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ICML\">\n","           ICML\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ICSE\">\n","           ICSE\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Image%20Annotation\">\n","           Image Annotation\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Image%20Classification\">\n","           Image Classification\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Image%20Processing\">\n","           Image Processing\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Inbox\">\n","           Inbox\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/India\">\n","           India\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Information%20Retrieval\">\n","           Information Retrieval\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/internationalization\">\n","           internationalization\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Internet%20of%20Things\">\n","           Internet of Things\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Interspeech\">\n","           Interspeech\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/IPython\">\n","           IPython\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Journalism\">\n","           Journalism\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/jsm\">\n","           jsm\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/jsm2011\">\n","           jsm2011\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/K-12\">\n","           K-12\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Kaggle\">\n","           Kaggle\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/KDD\">\n","           KDD\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Keyboard%20Input\">\n","           Keyboard Input\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Klingon\">\n","           Klingon\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Korean\">\n","           Korean\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Labs\">\n","           Labs\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Linear%20Optimization\">\n","           Linear Optimization\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/localization\">\n","           localization\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Low-Light%20Photography\">\n","           Low-Light Photography\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Machine%20Hearing\">\n","           Machine Hearing\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Machine%20Intelligence\">\n","           Machine Intelligence\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Machine%20Learning\">\n","           Machine Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Machine%20Perception\">\n","           Machine Perception\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Machine%20Translation\">\n","           Machine Translation\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Magenta\">\n","           Magenta\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/MapReduce\">\n","           MapReduce\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/market%20algorithms\">\n","           market algorithms\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Market%20Research\">\n","           Market Research\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/materials%20science\">\n","           materials science\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Mixed%20Reality\">\n","           Mixed Reality\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ML\">\n","           ML\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ML%20Fairness\">\n","           ML Fairness\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/MOOC\">\n","           MOOC\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Moore%27s%20Law\">\n","           Moore's Law\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Multimodal%20Learning\">\n","           Multimodal Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/NAACL\">\n","           NAACL\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Natural%20Language%20Processing\">\n","           Natural Language Processing\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Natural%20Language%20Understanding\">\n","           Natural Language Understanding\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Network%20Management\">\n","           Network Management\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Networks\">\n","           Networks\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Neural%20Networks\">\n","           Neural Networks\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/NeurIPS\">\n","           NeurIPS\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Nexus\">\n","           Nexus\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Ngram\">\n","           Ngram\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/NIPS\">\n","           NIPS\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/NLP\">\n","           NLP\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/On-device%20Learning\">\n","           On-device Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/open%20source\">\n","           open source\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/operating%20systems\">\n","           operating systems\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Optical%20Character%20Recognition\">\n","           Optical Character Recognition\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/optimization\">\n","           optimization\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/osdi\">\n","           osdi\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/osdi10\">\n","           osdi10\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/patents\">\n","           patents\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Peer%20Review\">\n","           Peer Review\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/ph.d.%20fellowship\">\n","           ph.d. fellowship\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/PhD%20Fellowship\">\n","           PhD Fellowship\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/PhotoScan\">\n","           PhotoScan\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Physics\">\n","           Physics\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/PiLab\">\n","           PiLab\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Pixel\">\n","           Pixel\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Policy\">\n","           Policy\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Professional%20Development\">\n","           Professional Development\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Proposals\">\n","           Proposals\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Public%20Data%20Explorer\">\n","           Public Data Explorer\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/publication\">\n","           publication\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Publications\">\n","           Publications\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Quantum%20AI\">\n","           Quantum AI\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Quantum%20Computing\">\n","           Quantum Computing\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Recommender%20Systems\">\n","           Recommender Systems\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Reinforcement%20Learning\">\n","           Reinforcement Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/renewable%20energy\">\n","           renewable energy\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Research\">\n","           Research\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Research%20Awards\">\n","           Research Awards\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/resource%20optimization\">\n","           resource optimization\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Responsible%20AI\">\n","           Responsible AI\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Robotics\">\n","           Robotics\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/schema.org\">\n","           schema.org\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Search\">\n","           Search\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/search%20ads\">\n","           search ads\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Security%20and%20Privacy\">\n","           Security and Privacy\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Self-Supervised%20Learning\">\n","           Self-Supervised Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Semantic%20Models\">\n","           Semantic Models\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Semi-supervised%20Learning\">\n","           Semi-supervised Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/SIGCOMM\">\n","           SIGCOMM\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/SIGMOD\">\n","           SIGMOD\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Site%20Reliability%20Engineering\">\n","           Site Reliability Engineering\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Social%20Networks\">\n","           Social Networks\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Software\">\n","           Software\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Sound%20Search\">\n","           Sound Search\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Speech\">\n","           Speech\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Speech%20Recognition\">\n","           Speech Recognition\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/statistics\">\n","           statistics\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Structured%20Data\">\n","           Structured Data\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Style%20Transfer\">\n","           Style Transfer\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Supervised%20Learning\">\n","           Supervised Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Systems\">\n","           Systems\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/TensorBoard\">\n","           TensorBoard\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/TensorFlow\">\n","           TensorFlow\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/TPU\">\n","           TPU\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Translate\">\n","           Translate\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/trends\">\n","           trends\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/TTS\">\n","           TTS\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/TV\">\n","           TV\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/UI\">\n","           UI\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/University%20Relations\">\n","           University Relations\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/UNIX\">\n","           UNIX\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Unsupervised%20Learning\">\n","           Unsupervised Learning\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/User%20Experience\">\n","           User Experience\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/video\">\n","           video\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Video%20Analysis\">\n","           Video Analysis\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Virtual%20Reality\">\n","           Virtual Reality\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Vision%20Research\">\n","           Vision Research\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Visiting%20Faculty\">\n","           Visiting Faculty\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Visualization\">\n","           Visualization\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/VLDB\">\n","           VLDB\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Voice%20Search\">\n","           Voice Search\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Wiki\">\n","           Wiki\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/wikipedia\">\n","           wikipedia\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/WWW\">\n","           WWW\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/Year%20in%20Review\">\n","           Year in Review\n","          </a>\n","         </li>\n","         <li>\n","          <a dir=\"ltr\" href=\"http://ai.googleblog.com/search/label/YouTube\">\n","           YouTube\n","          </a>\n","         </li>\n","        </ul>\n","        <div class=\"clear\">\n","        </div>\n","       </div>\n","      </div>\n","      <div class=\"widget BlogArchive\" data-version=\"1\" id=\"BlogArchive1\">\n","       <div class=\"tab\">\n","        <i class=\"material-icons icon\">\n","         \n","        </i>\n","        <h2>\n","         Archive\n","        </h2>\n","        <i class=\"material-icons arrow\">\n","         \n","        </i>\n","       </div>\n","       <div class=\"widget-content\">\n","        <div id=\"ArchiveList\">\n","         <div id=\"BlogArchive1_ArchiveList\">\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate expanded\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy toggle-open\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2022/\">\n","              2022\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate expanded\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2022/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2022/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2022/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/\">\n","              2021\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2021/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/\">\n","              2020\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2020/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/\">\n","              2019\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2019/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/\">\n","              2018\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2018/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/\">\n","              2017\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2017/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/\">\n","              2016\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2016/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/\">\n","              2015\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2015/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/\">\n","              2014\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2014/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/\">\n","              2013\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2013/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/\">\n","              2012\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2012/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/\">\n","              2011\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2011/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/\">\n","              2010\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2010/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/\">\n","              2009\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2009/01/\">\n","                 Jan\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/\">\n","              2008\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/05/\">\n","                 May\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2008/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2007/\">\n","              2007\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2007/10/\">\n","                 Oct\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2007/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2007/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2007/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2007/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2007/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","          <ul class=\"hierarchy\">\n","           <li class=\"archivedate collapsed\">\n","            <div class=\"intervalToggle\">\n","             <span class=\"new-toggle\" href=\"javascript:void(0)\">\n","              <i class=\"material-icons arrow\">\n","               \n","              </i>\n","             </span>\n","             <a class=\"toggle\" href=\"javascript:void(0)\" style=\"display: none\">\n","              <span class=\"zippy\">\n","               <i class=\"material-icons\">\n","                \n","               </i>\n","              </span>\n","             </a>\n","             <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/\">\n","              2006\n","             </a>\n","            </div>\n","            <div class=\"items\">\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/12/\">\n","                 Dec\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/11/\">\n","                 Nov\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/09/\">\n","                 Sep\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/08/\">\n","                 Aug\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/07/\">\n","                 Jul\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/06/\">\n","                 Jun\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/04/\">\n","                 Apr\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/03/\">\n","                 Mar\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","             <ul class=\"hierarchy\">\n","              <li class=\"archivedate collapsed\">\n","               <div class=\"\">\n","                <a class=\"post-count-link\" href=\"http://ai.googleblog.com/2006/02/\">\n","                 Feb\n","                </a>\n","               </div>\n","               <div class=\"items\">\n","               </div>\n","              </li>\n","             </ul>\n","            </div>\n","           </li>\n","          </ul>\n","         </div>\n","        </div>\n","        <div class=\"clear\">\n","        </div>\n","       </div>\n","      </div>\n","      <div class=\"widget HTML\" data-version=\"1\" id=\"HTML6\">\n","       <div class=\"widget-content\">\n","        <a href=\"http://googleaiblog.blogspot.com/atom.xml\">\n","         <img class=\"sidebar-icon\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADAAAAAwCAYAAABXAvmHAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAihJREFUeNrsWa9Pw0AU7viRMDFRBAkzJDMIBIhJJhCzk7NILIqMv4AEhdz+BCY3OYssAlGBoAJREpZwAlHEBO8lr8nSvNeVbu1dyX3JlzTrXfa+u/e9d7c5joWFhYVO1Fa8PwH2gK6m+BRwAvSlAdsrgr8E1jUuMH73GTAEzrkBWymTewZlihhLmgDXIAFuHgGVQOUF7OSYM1p6PgTuA1vAZlUEvAnPdapcMY0VICECekQ0XRfYrqoHsAGNgXfAoMomRiFDEhOZkkL3S88hMaB2LwXp0bj+ps2edpToZpjfoIDQtBeU+xjoDzP2G/gCPKZ5f8WsCAFJoJgOCcFdWSTeL9YQMSvTA1h9BkI5jaiXhLpSCL/8mVZY0UpyJ9ZdOkniu1dmJ96BpzQu9w6s28gcOq9j6pwLdR8/36NK5CQKwJSMrb2MhhSglBpt4UjsrdsnNu0B3J0HCozbCc4TjyY2srEgos/4RQljCzNxl4ireQD8FOq+T+W0mTB2g7njhlR+Sy2jsXFvU658U8YTbeaGpdIu7mWkEAq5ZtIjIhFZdtfX7QHckSvB2B6zC3VdAkZk0kAQwaXTk/CzTXK3wjIExCs6ZJpTnE4uY1KV+KzFzA3KTiFPENHJkOPcsfpLhwe4btoSuvUqAR+6TOxlCE6ZfKUsJLgsqGW8OpqAGx2X+sLxrwUog+JUeQRMDBIwyXOcnlPtPnL0/UsT/8LnOxYWFhZG4leAAQAAQHEaYuzHbAAAAABJRU5ErkJggg==\"/>\n","         <h2>\n","          Feed\n","         </h2>\n","        </a>\n","       </div>\n","       <div class=\"clear\">\n","       </div>\n","      </div>\n","     </div>\n","     <div class=\"section\" id=\"sidebar-bottom\">\n","      <div class=\"widget HTML\" data-version=\"1\" id=\"HTML5\">\n","       <div class=\"widget-content\">\n","        <div class=\"share followgooglewrapper\">\n","         <button data-href=\"https://twitter.com/intent/follow?original_referer=http://ai.googleblog.com/&amp;screen_name=googleai\" id=\"twitter-share\" onclick=\"sharingPopup(this);\">\n","          <span class=\"twitter-follow\">\n","           Follow @googleai\n","          </span>\n","         </button>\n","         <script>\n","          function sharingPopup (button) {\n","      var url = button.getAttribute(\"data-href\");\n","\t    window.open(\n"," url,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes');\n","          }\n","         </script>\n","        </div>\n","       </div>\n","       <div class=\"clear\">\n","       </div>\n","      </div>\n","      <div class=\"widget HTML\" data-version=\"1\" id=\"HTML1\">\n","       <div class=\"widget-content\">\n","        Give us feedback in our\n","        <a href=\"http://support.google.com/bin/static.py?hl=en&amp;page=portal_groups.cs\">\n","         Product Forums\n","        </a>\n","        .\n","       </div>\n","       <div class=\"clear\">\n","       </div>\n","      </div>\n","     </div>\n","    </div>\n","   </div>\n","   <div style=\"clear:both;\">\n","   </div>\n","  </div>\n","  <!-- Footer -->\n","  <div class=\"google-footer-outer loading\">\n","   <div id=\"google-footer\">\n","    <a href=\"//www.google.com/\">\n","     <img class=\"google-logo-dark\" height=\"36\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALgAAABICAYAAABFoT/eAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAACLVJREFUeNrsXd+L20YQ3vOprdLqiMXFXE2qB7dcwEcTSB7ykIc+9A/PQx/yEMq1TWhNuYIpJriNr7XpmZ5IxFEvmW2EKs3Ornb1w50PxIFP0kiz387OzM6uhGAwGAxGP3Ho+f7x7ri1O7LdccPqZjSNA4dEHsLfaHcEFedJom93x9Xu2OyOFTcBo6sED3fHZHeMEELrkAHJF0B8Rr+gDFsZ5n0luLTQ95AXs4W06D/tjpR50xtM4CjD0y48YGB4rnyZxNOzyA7zBHr+nLnDaJLg0mo/ALekCasg3Z4XbM0ZdTEgnDPeHY8bIne+Qz2GvwyGNwsuyT218KWvIIBMcwGpLiipcolecjMxfBDchNyS1EvxLiOSIecp31q6IJ/C3yrIrMqMm4jhg+AxkdwbIO3aUO4KjqqMjCT3uaazMBhWBJfuxH3CtRfiXf66DhSRZWbmlMnNaILgZxrXJQO/eO3wORZwvwm4JUxuhheCjzVBYAbW1ces45YDSoZrFNOEE835M8FT6oyeEnws8Fz3QnBxFKPHBMem4GU+m6fPGb0leCTwWcM5B36MPgeZI01gudyDdw3hPeXfo8L/rmCUWnuMMdqUL2WqWeRbhf+twfVsO7YagZGNC79fw7OthEVtkiJ4jJzTd3KPwf3CRqhhiTu23AP5sl0/0xiwISQXpNwLIJK87mHF+U8ddzzdmgKlGzlPYjyxGJQouIhNT4k9AqWEFkqfguIvagTWbcq3KW1WE3xS3m8NtA9WS451xofwjKT5kkDoK/b6mDk5FfXr1lWDL4BofZEv2/SRsK/EHGlGdBdu8QNRb8HMCFwt7Yy3DDI/QP7fx5z3VLhdlJEIs4rKNuXXJXdxZPdB7kfCzWqwCO4V1LHgLjInX3tQ1KzCR52Cz+vDj1dydeRuS74rcvs2Pi6fT5H8OaaUQPQPYcWwRSGXyhhscn5dpAnEFMkuEZetbfkTAnlSuH4DxisE+aMGeJAQ3lFl7C4LJE6QWCaCd583ORQ1jYAwjFctal7nOs2ZZvicwvlZx+RHGrcoAwKUVX8uwcc/9TT65INeDOr5shL9LDRB6QTeIy3zwfdh3WOi6axLCEhSjXU7F3h6LqggUtvyJxpynwu8tDkD98fXApOxRj8zoZ9MnGveYVIVZKaGrkBXCY65BCYNN9NkjpKOyQ81Q79JgdxS+Jn3SDTEXRI7SWzaiSTB32oI3nU3BvMfM0urhOVYgwKhuiAfc4tM07wXwm1ZRoQYSl2NUwiu01fEAHVcpixd745FvVz4dzUUc0o8rwoLy8ZSwU6CyFx1RP5II9+1bFPEFs9HWbNLiimDXE+vCm7u1CS47cofzD3aEhVY57mxRo5zlqdt+RFC1JUH2S7bcVXg4liTMakaBZZVxiTICRoivcn1sEUBlk24JmaC6kxUbYmWoqvyfck2xZGGnDFYa9MMzkYQ1ijkCX6qidybrgePiQ0QIQqoi6qRLeqQfIoRsEHaQJLBdHOnLGetSdm/IPcymJuS1PAnbQPH0MOw/39C1vL11DiLOqIsbDI8QcHvGiLnySi2qUXBicaqUSxN5LEB0g7Jt3ENXJLPJ5S1tnaZBoWbpRqrmjRE7qHmpSmNHdQcYrEUadoh+TbBnc9ri7iycI1kzPeNcLDIvbiqXpez9Tmdq6zGREPuzECBoxrPMiI2WtvyNwhJba2wy3JZ6ky5dD1lSvmZS3e4SPA1wcf1VTFHKX+cGwZzdUYcqpvUtvwrD/InDttVlyZeAKlNN5MKbAiurHhKIPlUuJvlTCCiDjSKSCsUmCFWbGLZwCESfK07JB8LvMYWVtw0D00JEHV8Mq2HkqPbE0oHLvvK2g0o8ETg+4cfwTlZDT9JDoWygu4uQQE/ivIvtcnfPkaCqhiupz7jWOAzqL/vjtcdkv9G4MVMt+EaylfuImiPAXEUjRF3pjjaHiPPZ6If9TGGAO4ZY0am6jOCb+DQ+ZCqLkIpOIPrdNfIjnFPY6nyFut7TS/fanrziOBOKMupKw94WaLMtuVnSFt9CPrWWdJE6PeltCX432DEBoh+5Dv8RRhdis8YAv9uyq4/JAwtlEApgBe9Cw9xDD3tdk4Jn0MDfiHwPHcRPxBePCMER3GuIx7kGlv9fkZ4V9lolx2Uv4X7hEj7qJ3LDoAMGbTRMRibu4L2xQ8bgt8AyU+Q+x7nYrvDnH4iuO5LxKsYwPVbkPMvKF9Zky9wXzRfVWizi62r9X5VHf55h+WHhDjGBZ4WRhyTr6z5SlCoLMxLSpBZFsQ9F80uQFbF/6aFWi+Ev51vzzsuX+msyzuQXXjUz8zEBy+zpq9yweXAoxJW4JbYrDS6gYDqGHxPl+TKeiBfxj9/EBIElPYeOA4y8/qRQfknjvSzgRgtq0Pw/M1eQeMdOSb2Bnrhr6Led+1vcp2x7oTFHMnedFW+Ivlty062BUt74oHgSj+vHepnhunn0JJAMtBZgDI/qmGtMujRv8DDpo47zBJ8UtPOuAR/7rKn8t9AJ0tBdmBAmJ/Fu71yxp4I3qh+DhyRqbi5Y1ShVPlSb8X7bRNcfgZFl+WRGYo7uecrWq1r8X5bhmzP5OdlDwsGRm1suSxkg5rYm7ConyGQ3Zl+DgSD8V/kPwrWBMG9YcBtyShBnTLdTiHgttw7qAW7cqh/ZnmPKr/6ignOaKsdyxbsToT5UkPsW00bJjijDXficcX/JsLs6w2BwGtherdckH3w/kNXRPVI0OqJQoHX42/66IMfMj/2huRjxIidgKV/W0JS+bsstDoTeAHcrI8E5zTh/sDkqxL5rZup55/3USlswfcHf4IrQplVDgW9XFlOqnwr6pVPMMEZTuC60EttvdzbLbaZ4PsFVa3nohhO+vW+yn/ZB2fUhpysmQrzBcTSai9EszuZMcEZ1lCFVrp9zGXhm69iLyY4oxFIa178lPe12I/P2DAYDAaDwWAwGAwGg8FgMBgMBoPBYDD2Cf8IMADDRGoQTe+E9AAAAABJRU5ErkJggg==\" style=\"margin-top: -16px;\" width=\"92\"/>\n","    </a>\n","    <ul>\n","     <li>\n","      <a href=\"//www.google.com/\">\n","       Google\n","      </a>\n","     </li>\n","     <li>\n","      <a href=\"//www.google.com/policies/privacy/\">\n","       Privacy\n","      </a>\n","     </li>\n","     <li>\n","      <a href=\"//www.google.com/policies/terms/\">\n","       Terms\n","      </a>\n","     </li>\n","    </ul>\n","   </div>\n","  </div>\n","  <script type=\"text/javascript\">\n","   //<![CDATA[\n","      // Social sharing popups.\n","      var postEl = document.getElementsByClassName('social-wrapper');\n","      var postCount = postEl.length;\n","      for(i=0; i<postCount;i++){\n","        postEl[i].addEventListener(\"click\", function(event){\n","          var postUrl = this.getAttribute(\"data-href\");\n","          window.open(\n","            postUrl,'popUpWindow','height=500,width=500,left=10,top=10,resizable=yes,scrollbars=yes,toolbar=yes,menubar=no,location=no,directories=no,status=yes');\n","        });}\n","      //]]>\n","  </script>\n","  <script type=\"text/javascript\">\n","   //<![CDATA[\n","      var BreakpointHandler = function() {\n","        this.initted = false;\n","        this.isHomePage = false;\n","        this.isMobile = false;\n","      };\n","      BreakpointHandler.prototype.finalizeSummary = function(summaryHtml, lastNode) {\n","        // Use $.trim for IE8 compatibility\n","        summaryHtml = $.trim(summaryHtml).replace(/(<br>|\\s)+$/,'');\n","        if (lastNode.nodeType == 3) {\n","          var lastChar = summaryHtml.slice(-1);\n","          if (!lastChar.match(/[.”\"?]/)) {\n","            if (!lastChar.match(/[A-Za-z]/)) {\n","              summaryHtml = summaryHtml.slice(0, -1);\n","            }\n","            summaryHtml += ' ...';\n","          }\n","        } else if (lastNode.nodeType == 1 && (lastNode.nodeName == 'I' || lastNode.nodeName == 'A')) {\n","          summaryHtml += ' ...';\n","        }\n","        return summaryHtml;\n","      };\n","      BreakpointHandler.prototype.generateSummaryFromContent = function(content, numWords) {\n","        var seenWords = 0;\n","        var summaryHtml = '';\n","        for (var i=0; i < content.childNodes.length; i++) {\n","          var node = content.childNodes[i];\n","          var nodeText;\n","          if (node.nodeType == 1) {\n","            if (node.hasAttribute('data-about-pullquote')) {\n","              continue;\n","            }\n","            nodeText = node.textContent;\n","            if (nodeText === undefined) {\n","              // innerText for IE8\n","              nodeText = node.innerText;\n","            }\n","            if (node.nodeName == 'DIV' || node.nodeName == 'B') {\n","              // Don't end early if we haven't seen enough words.\n","              if (seenWords < 10) {\n","                continue;\n","              }\n","              if (i > 0) {\n","                summaryHtml = this.finalizeSummary(summaryHtml, content.childNodes[i-1]);\n","              }\n","              break;\n","            }\n","            summaryHtml += node.outerHTML;\n","          } else if (node.nodeType == 3) {\n","            nodeText = node.nodeValue;\n","            summaryHtml += nodeText + ' ';\n","          }\n","          var words = nodeText.match(/\\S+\\s*/g);\n","          if (!words) {\n","            continue;\n","          }\n","          var remain = numWords - seenWords;\n","          if (words.length >= remain) {\n","            summaryHtml = this.finalizeSummary(summaryHtml, node);\n","            break;\n","          }\n","          seenWords += words.length;\n","        }\n","        return summaryHtml;\n","      };\n","      BreakpointHandler.prototype.detect = function() {\n","        var match,\n","            pl     = /\\+/g,\n","            search = /([^&=]+)=?([^&]*)/g,\n","            decode = function (s) { return decodeURIComponent(s.replace(pl, \" \")); },\n","            query  = window.location.search.substring(1);\n","        var urlParams = {};\n","        while (match = search.exec(query))\n","          urlParams[decode(match[1])] = decode(match[2]);\n","        this.isListPage = $('html').hasClass('list-page');\n","        this.isMobile = urlParams['m'] === '1';\n","        this.isHomePage = window.location.pathname == '/';\n","      };\n","      BreakpointHandler.prototype.initContent = function() {\n","        var self = this;\n","        $('.post').each(function(index) {\n","          var body = $(this).children('.post-body')[0];\n","          var content = $(body).children('.post-content')[0];\n","          $(content).addClass('post-original');\n","          var data = $(content).children('script').html();\n","          data = self.rewriteForSSL(data);\n","\n","          if (document.body.className.indexOf('is-preview') !== -1) {\n","            // If exists, extract specified editor's preview.\n","            var match = data.match(/([\\s\\S]+?)<div data-is-preview.+?>([\\s\\S]+)<\\/div>/m);\n","            if (match) {\n","              data = match[1];\n","            }\n","          }\n","\n","          // Prevent big images from loading when they aren't needed.\n","          // This must be done as a pre-injection step, since image loading can't be\n","          // canceled once embedded into the DOM.\n","          if (self.isListPage && self.isMobile) {\n","            data = data.replace(/<(img|iframe) .+?>/g, '');\n","          }\n","          // Insert template to be rendered as nodes.\n","          content.innerHTML = data;\n","          if (self.isListPage) {\n","            var summary = document.createElement('div');\n","            $(summary).addClass('post-content');\n","            $(summary).addClass('post-summary');\n","            body.insertBefore(summary, content);\n","            if (match) {\n","              // Use provided summary.\n","              summary.innerHTML = match[2];\n","            } else {\n","              // Generate a summary.\n","              // Summary generation relies on DOM, so it must occur after content is\n","              // inserted into the page.\n","              summary.innerHTML = self.generateSummaryFromContent(content, 30);\n","            }\n","            // Add read more link to summary.\n","            var titleAnchor = $(this).find('.title a')[0];\n","            var link = titleAnchor.cloneNode(true);\n","            link.innerHTML = 'Read More';\n","            $(link).addClass('read-more');\n","            summary.appendChild(link);\n","          }\n","        });\n","        // Firefox does not allow for proper styling of BR.\n","        if (navigator.userAgent.indexOf('Firefox') > -1) {\n","          $('.post-content br').replaceWith('<span class=\"space\"></span>');\n","        }\n","        $('.loading').removeClass('loading');\n","      };\n","      BreakpointHandler.prototype.process = function() {\n","        if (!this.initted) {\n","          var makeInsecureImageRegex = function(hosts) {\n","            var whitelist = hosts.join('|').replace(/\\./g,'\\\\.');\n","            // Normal image tags, plus input images (yes, this is possible!)\n","            return new RegExp('(<(img|input)[^>]+?src=(\"|\\'))http:\\/\\/(' + whitelist +')', 'g');\n","          };\n","          this.sslImageRegex = makeInsecureImageRegex(BreakpointHandler.KNOWN_HTTPS_HOSTS);\n","          this.sslImageCurrentDomainRegex = makeInsecureImageRegex([window.location.hostname]);\n","          this.detect();\n","          this.initContent();\n","          this.initted = true;\n","        }\n","      };\n","      BreakpointHandler.KNOWN_HTTPS_HOSTS = [\n","        \"www.google.org\",\n","        \"www.google.com\",\n","        \"services.google.com\",\n","        \"blogger.com\",\n","        \"draft.blogger.com\",\n","        \"www.blogger.com\",\n","        \"photos1.blogger.com\",\n","        \"photos2.blogger.com\",\n","        \"photos3.blogger.com\",\n","        \"blogblog.com\",\n","        \"img1.blogblog.com\",\n","        \"img2.blogblog.com\",\n","        \"www.blogblog.com\",\n","        \"www1.blogblog.com\",\n","        \"www2.blogblog.com\",\n","        \"0.bp.blogspot.com\",\n","        \"1.bp.blogspot.com\",\n","        \"2.bp.blogspot.com\",\n","        \"3.bp.blogspot.com\",\n","        \"4.bp.blogspot.com\",\n","        \"lh3.googleusercontent.com\",\n","        \"lh4.googleusercontent.com\",\n","        \"lh5.googleusercontent.com\",\n","        \"lh6.googleusercontent.com\",\n","        \"themes.googleusercontent.com\",\n","      ];\n","        BreakpointHandler.prototype.rewriteForSSL = function(html) {\n","        // Handle HTTP -> HTTPS source replacement of images, movies, and other embedded content.\n","        return html.replace(this.sslImageRegex, '$1https://$4')\n","        .replace(this.sslImageCurrentDomainRegex, '$1//$4')\n","        .replace(/(<(embed|iframe)[^>]+?src=(\"|'))http:\\/\\/([^\"']*?(youtube|picasaweb\\.google)\\.com)/g, '$1https://$4')\n","        // Slideshow SWF takes a image host, so we need to rewrite that parameter.\n","        .replace(/(<embed[^>]+?feed=http(?=[^s]))/g, '$1s');\n","        };\n","        $(document).ready(function() {\n","        var handler = new BreakpointHandler();\n","        handler.process();\n","        // Top-level navigation.\n","        $(\".BlogArchive .tab\").click(function(ev) {\n","        ev.preventDefault();\n","        $(this).parent().toggleClass('active');\n","        $(this).siblings().slideToggle(300);\n","        });\n","        $(\".Label .tab\").click(function(ev) {\n","        ev.preventDefault();\n","        $(this).parent().toggleClass('active');\n","        $(this).siblings().slideToggle(300);\n","        });\n","        // Blog archive year expansion.\n","        $('.BlogArchive .intervalToggle').click(function(ev) {\n","        ev.preventDefault();\n","        if ($(this).parent().hasClass('collapsed')) {\n","        $(this).parent().removeClass('collapsed');\n","        $(this).parent().addClass('expanded');\n","        } else {\n","        $(this).parent().removeClass('expanded');\n","        $(this).parent().addClass('collapsed');\n","        }\n","        });\n","        // Reverse order of months.\n","        $('.BlogArchive .intervalToggle + div').each(function(_, items) {\n","        var year = $(this);\n","        year.children().each(function(_, month) {\n","        year.prepend(month);\n","        });\n","        });\n","        // Set anchors to open in new tab.\n","        $('.post-content img').parent().each(function(_, node) {\n","        if (node.nodeName == 'A') {\n","        $(this).attr('target', '_blank');\n","        }\n","        });\n","        // Process search requests.\n","        $('.searchBox input').on(\"keypress\", function(ev) {\n","        if (ev.which == 13) {\n","        window.location.href = 'https://www.google.com/search?q=site%3A' + window.location.hostname + '%20' + encodeURIComponent ($(this).val());\n","        }\n","        });\n","        });\n","        //]]>\n","  </script>\n","  <link href=\"https://cdnjs.cloudflare.com/ajax/libs/prettify/r298/prettify.min.css\" rel=\"stylesheet\" type=\"text/css\"/>\n","  <script language=\"javascript\" src=\"https://cdnjs.cloudflare.com/ajax/libs/prettify/r298/prettify.min.js\" type=\"text/javascript\">\n","  </script>\n","  <script language=\"javascript\" src=\"https://cdnjs.cloudflare.com/ajax/libs/prettify/r298/lang-css.min.js\" type=\"text/javascript\">\n","  </script>\n","  <script type=\"text/javascript\">\n","   document.addEventListener('DOMContentLoaded',function() {\n","            prettyPrint();\n","        });\n","  </script>\n","  <script src=\"https://www.blogger.com/static/v1/widgets/2465480620-widgets.js\" type=\"text/javascript\">\n","  </script>\n","  <script type=\"text/javascript\">\n","   window['__wavt'] = 'AOuZoY7RDHH_5f7SK6WHe78naGkyE2rgjA:1648764791361';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\\x3d8474926331452026626','//ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html','8474926331452026626');\n","_WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '8474926331452026626', 'title': 'Google AI Blog', 'url': 'http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html', 'canonicalUrl': 'http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html', 'homepageUrl': 'http://ai.googleblog.com/', 'searchUrl': 'http://ai.googleblog.com/search', 'canonicalHomepageUrl': 'http://ai.googleblog.com/', 'blogspotFaviconUrl': 'http://ai.googleblog.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': true, 'httpsEnabled': false, 'enabledCommentProfileImages': true, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': 'UA-961555-69', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\\x3clink rel\\x3d\\x22alternate\\x22 type\\x3d\\x22application/atom+xml\\x22 title\\x3d\\x22Google AI Blog - Atom\\x22 href\\x3d\\x22http://ai.googleblog.com/feeds/posts/default\\x22 /\\x3e\\n\\x3clink rel\\x3d\\x22alternate\\x22 type\\x3d\\x22application/rss+xml\\x22 title\\x3d\\x22Google AI Blog - RSS\\x22 href\\x3d\\x22http://ai.googleblog.com/feeds/posts/default?alt\\x3drss\\x22 /\\x3e\\n\\x3clink rel\\x3d\\x22service.post\\x22 type\\x3d\\x22application/atom+xml\\x22 title\\x3d\\x22Google AI Blog - Atom\\x22 href\\x3d\\x22https://www.blogger.com/feeds/8474926331452026626/posts/default\\x22 /\\x3e\\n\\n\\x3clink rel\\x3d\\x22alternate\\x22 type\\x3d\\x22application/atom+xml\\x22 title\\x3d\\x22Google AI Blog - Atom\\x22 href\\x3d\\x22http://ai.googleblog.com/feeds/1455778591516274568/comments/default\\x22 /\\x3e\\n', 'meTag': '', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'ieCssRetrofitLinks': '\\x3c!--[if IE]\\x3e\\x3cscript type\\x3d\\x22text/javascript\\x22 src\\x3d\\x22https://www.blogger.com/static/v1/jsbin/2287435483-ieretrofit.js\\x22\\x3e\\x3c/script\\x3e\\n\\x3c![endif]--\\x3e', 'boqCommentIframeForm': false, 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/d38e2600065d39bd', 'plusOneApiSrc': 'https://apis.google.com/js/plusone.js', 'disableGComments': true, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'Twitter', 'key': 'twitter', 'shareMessage': 'Share to Twitter', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\\x3cscript type\\x3d\\x22text/javascript\\x22\\x3ewindow.___gcfg \\x3d {\\x27lang\\x27: \\x27en\\x27};\\x3c/script\\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'item', 'postId': '1455778591516274568', 'postImageThumbnailUrl': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s72-c/Auto-generated%20summaries%20.gif', 'postImageUrl': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s16000/Auto-generated%20summaries%20.gif', 'pageName': 'Auto-generated Summaries in Google Docs', 'pageTitle': 'Google AI Blog: Auto-generated Summaries in Google Docs'}}, {'name': 'features', 'data': {'sharing_get_link_dialog': 'true', 'sharing_native': 'false'}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\\x3dtimeslide'}, 'isMobile': false, 'title': 'Auto-generated Summaries in Google Docs', 'description': 'Posted by Mohammad Saleh, Software Engineer, Google Research, Brain Team and Anjuli Kannan, Software Engineer, Google Docs    For many of us...', 'featuredImage': 'https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEh3EIjUhf4n0l_yYLvlHDEDzIiYo0v6LLQgiG4rD0YthuUBY2zeO8-NqPD44jJdkhCKPKcLvzmE5MnMwXVNodxTnocbGIinWPW7BiQf5kzPcTDgWiCKV8_YJX0H6vcXBfEMLr0D2UPl8rH4CRml_5b6guq8Mf8PE13zJRLTW7RnJ8_hEURzbbFbgMaqsQ/s16000/Auto-generated%20summaries%20.gif', 'url': 'http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 1455778591516274568}}]);\n","_WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull'));\n","_WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false}, 'displayModeFull'));\n","_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML8', 'sidebar-top', document.getElementById('HTML8'), {}, 'displayModeFull'));\n","_WidgetManager._RegisterWidget('_LabelView', new _WidgetInfo('Label1', 'sidebar', document.getElementById('Label1'), {}, 'displayModeFull'));\n","_WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive1', 'sidebar', document.getElementById('BlogArchive1'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\\x26hellip;'}, 'displayModeFull'));\n","_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML6', 'sidebar', document.getElementById('HTML6'), {}, 'displayModeFull'));\n","_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML5', 'sidebar-bottom', document.getElementById('HTML5'), {}, 'displayModeFull'));\n","_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML1', 'sidebar-bottom', document.getElementById('HTML1'), {}, 'displayModeFull'));\n","  </script>\n"," </body>\n","</html>\n"]}]},{"cell_type":"code","source":["##Extract required Titles and text\n","results=soup.find_all(['h1',h2','p','b'])\n","results"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ArgtIfeIFDcI","executionInfo":{"status":"ok","timestamp":1648799585066,"user_tz":-360,"elapsed":435,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"04b214a9-9c68-4315-820e-5e28b43a72bb"},"execution_count":59,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[<h2 class=\"title\" itemprop=\"name\">\n"," <a href=\"http://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html\" itemprop=\"url\" title=\"Auto-generated Summaries in Google Docs\">\n"," Auto-generated Summaries in Google Docs\n"," </a>\n"," </h2>, <p>\n"," For many of us, it can be challenging to keep up with the volume of documents that arrive in our inboxes every day: reports, reviews, briefs, policies and the list goes on. When a new document is received, readers often wish it included a brief summary of the main points in order to effectively prioritize it. However, composing a document summary can be cognitively challenging and time-consuming, especially when a document writer is starting from scratch.\n"," </p>, <p>\n"," To help with this, we recently <a href=\"https://cloud.google.com/blog/products/workspace/delivering-new-innovations-in-google-workspace-with-smart-canvas\">announced</a> that Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available. Today we describe how this was enabled using a machine learning (ML) model that comprehends document text and, when confident, generates a 1-2 sentence natural language description of the document content. However, the document writer maintains full control — accepting the suggestion as-is, making necessary edits to better capture the document summary or ignoring the suggestion altogether. Readers can also use this section, along with the outline, to understand and navigate the document at a high level. While all users can add summaries, auto-generated suggestions are currently only <a href=\"https://workspaceupdates.googleblog.com/2022/02/new-smart-canvas-features-in-google-docs.html\">available</a> to Google Workspace business customers. Building on <a href=\"https://cloud.google.com/blog/products/g-suite/let-grammar-suggestions-in-google-docs-help-you-write-even-better\">grammar suggestions</a>, <a href=\"https://workspaceupdates.googleblog.com/2020/02/smart-compose-ga.html\">Smart Compose,</a> and <a href=\"https://workspaceupdates.googleblog.com/2020/02/autocorrect-docs-ga.html\">autocorrect</a>, we see this as another valuable step toward improving written communication in the workplace.\n"," </p>, <p>\n"," <b>Model Details</b>\n"," <br/>\n"," Automatically generated summaries would not be possible without the tremendous advances in ML for <a href=\"https://en.wikipedia.org/wiki/Natural-language_understanding\">natural language understanding</a> (NLU) and <a href=\"https://en.wikipedia.org/wiki/Natural_language_generation\">natural language generation</a> (NLG) over the past five years, especially with the introduction of <a href=\"https://arxiv.org/abs/1706.03762\">Transformer</a> and <a href=\"https://ai.googleblog.com/2020/06/pegasus-state-of-art-model-for.html\">Pegasus</a>.\n"," </p>, <b>Model Details</b>, <p>\n"," <a href=\"https://ai.googleblog.com/2016/08/text-summarization-with-tensorflow.html\">Abstractive text summarization</a>, which combines the individually challenging tasks of long document language understanding and generation, has been a long-standing problem in NLU and NLG research. A popular method for combining NLU and NLG is training an ML model using <a href=\"https://arxiv.org/abs/1409.3215\">sequence-to-sequence</a> learning, where the inputs are the document words, and the outputs are the summary words. A neural network then learns to map input tokens to output tokens. <a href=\"https://arxiv.org/abs/1602.06023\">Early applications</a> of the sequence-to-sequence paradigm used <a href=\"https://en.wikipedia.org/wiki/Recurrent_neural_network\">recurrent neural networks</a> (RNNs) for both the encoder and decoder. \n"," </p>, <p>\n"," The introduction of Transformers provided a promising alternative to RNNs because Transformers use <a href=\"https://en.wikipedia.org/wiki/Attention_(machine_learning)\">self-attention</a> to provide better modeling of long input and output dependencies, which is critical in document summarization. Still, these models require large amounts of manually labeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization.\n"," </p>, <p>\n"," The combination of Transformers with self-supervised pre-training (e.g., <a href=\"https://arxiv.org/abs/1810.04805\">BERT</a>, <a href=\"https://openai.com/blog/better-language-models/\">GPT</a>, <a href=\"https://arxiv.org/abs/1910.10683\">T5</a>) led to a major breakthrough in many NLU tasks for which limited labeled data is available. In self-supervised pre-training, a model uses large amounts of unlabeled text to learn general language understanding and generation capabilities. Then, in a subsequent fine-tuning stage, the model learns to apply these abilities on a specific task, such as summarization or question answering.\n"," </p>, <p>\n"," The Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization. In Pegasus pre-training, also called <a href=\"https://arxiv.org/abs/1912.08777\">Gap Sentence Prediction</a> (GSP), full sentences from unlabeled news articles and web documents are masked from the input and the model is required to reconstruct them, conditioned on the remaining unmasked sentences. In particular, GSP attempts to mask sentences that are considered essential to the document through different heuristics. The intuition is to make the pre-training as close as possible to the summarization task. <a href=\"https://ai.googleblog.com/2020/06/pegasus-state-of-art-model-for.html\">Pegasus achieved state-of-the-art results</a> on a varied set of summarization datasets. However, a number of challenges remained to apply this research advancement into a product.\n"," </p>, <p>\n"," <b>Applying Recent Research Advances to Google Docs</b>\n"," <br/>\n"," </p>, <b>Applying Recent Research Advances to Google Docs</b>, <p>\n"," \n","     Self-supervised pre-training results in an ML model that has general language understanding and generation capabilities, but a subsequent fine-tuning stage is critical for the model to adapt to the application domain. We fine-tuned early versions of our model on a corpus of documents with manually-generated summaries that were consistent with typical use cases.\n"," </p>, <p>\n","     However, early versions of this corpus suffered from inconsistencies and high variation because they included many types of documents, as well as many ways to write a summary — e.g., academic abstracts are typically long and detailed, while executive summaries are brief and punchy. This led to a model that was easily confused because it had been trained on so many different types of documents and summaries that it struggled to learn the relationships between any of them.\n"," </p>, <p>\n","     Fortunately, one of the key findings in the Pegasus work was that an effective pre-training phase required less supervised data in the fine-tuning stage. Some summarization benchmarks required as few as 1,000 fine-tuning examples for Pegasus to match the performance of Transformer baselines that saw 10,000+ supervised examples — suggesting that one could focus on quality rather than quantity. \n"," </p>, <p>\n"," \n","     We carefully cleaned and filtered the fine-tuning data to contain training examples that were more consistent and represented a coherent definition of summaries. Despite the fact that we reduced the amount of training data, this led to a higher quality model. The key lesson, consistent with recent work in domains like <a href=\"https://ai.googleblog.com/2021/12/training-machine-learning-models-more.html\">dataset distillation</a>, was that it was better to have a smaller, high quality dataset, than a larger, high-variance dataset.</p>, <p>\n","     Once we trained the high quality model, we turned to the challenge of serving the model in production. While the Transformer version of the encoder-decoder architecture is the dominant approach to train models for sequence-to-sequence tasks like abstractive summarization, it can be inefficient and impractical to serve in real-world applications. The main inefficiency comes from the Transformer decoder where we generate the output summary token by token through <a href=\"https://en.wikipedia.org/wiki/Autoregressive_model\">autoregressive</a> decoding. The decoding process becomes noticeably slow when summaries get longer since the decoder attends to all previously generated tokens at each step. RNNs are a more efficient architecture for decoding since there is no self-attention with previous tokens as in a Transformer model. \n"," </p>, <p>\n","     We used <a href=\"https://arxiv.org/abs/1503.02531\">knowledge distillation</a>, which is the process of transferring knowledge from a large model to a smaller more efficient model, to distill the Pegasus model into a <a href=\"https://arxiv.org/abs/1804.09849\">hybrid architecture</a> of a Transformer encoder and an RNN decoder. To improve efficiency we also reduced the number of RNN decoder layers. The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model. To further improve the latency and user experience, we serve the summarization model using <a href=\"https://cloud.google.com/blog/products/ai-machine-learning/cloud-tpu-breaks-scalability-records-for-ai-inference\">TPUs</a>, which provide significant speed ups and allow more requests to be handled by a single machine.\n"," </p>, <p>\n"," <b>Ongoing Challenges and Next Steps</b>\n"," <br/>\n"," While we are excited by the progress so far, there are a few challenges we are continuing to tackle:\n"," </p>, <b>Ongoing Challenges and Next Steps</b>, <p>\n"," <b>Conclusion</b>\n"," <br/>\n"," Overall, we are thrilled that we can apply recent progress in NLU and NLG to continue assisting users with reading and writing. We hope the automatic suggestions now offered in Google Workspace make it easier for writers to annotate their documents with summaries, and help readers comprehend and navigate documents more easily.</p>, <b>Conclusion</b>, <p>\n"," <b>Acknowledgements</b>\n"," <br/>\n"," <i>The authors would like to thank the many people across Google that contributed to this work: AJ Motika, Matt Pearson-Beck, Mia Chen, Mahdis Mahdieh, Halit Erdogan, Benjamin Lee, Ali Abdelhadi, Michelle Danoff, Vishnu Sivaji, Sneha Keshav, Aliya Baptista, Karishma Damani, DJ Lick, Yao Zhao, Peter Liu, Aurko Roy, Yonghui Wu, Shubhi Sareen, Andrew Dai, Mekhola Mukherjee, Yinan Wang, Mike Colagrosso, and Behnoosh Hariri.\n"," </i>.\n"," </p>, <b>Acknowledgements</b>, <h2>\n"," Labels\n"," </h2>, <h2>\n"," Archive\n"," </h2>, <h2>Feed</h2>]"]},"metadata":{},"execution_count":59}]},{"cell_type":"code","source":["## Remove all the label from the text and joined back text\n","text=[result.text for result in results]\n","Article=' '.join(text)\n","Article"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":108},"id":"4Ps9ztznFDnl","executionInfo":{"status":"ok","timestamp":1648799596174,"user_tz":-360,"elapsed":421,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"b3ec13fc-78c8-4df5-a400-3d1b525d58f5"},"execution_count":60,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'\\n\\nAuto-generated Summaries in Google Docs\\n\\n \\nFor many of us, it can be challenging to keep up with the volume of documents that arrive in our inboxes every day: reports, reviews, briefs, policies and the list goes on. When a new document is received, readers often wish it included a brief summary of the main points in order to effectively prioritize it. However, composing a document summary can be cognitively challenging and time-consuming, especially when a document writer is starting from scratch.\\n \\nTo help with this, we recently announced that Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available. Today we describe how this was enabled using a machine learning (ML) model that comprehends document text and, when confident, generates a 1-2 sentence natural language description of the document content. However, the document writer maintains full control — accepting the suggestion as-is, making necessary edits to better capture the document summary or ignoring the suggestion altogether. Readers can also use this section, along with the outline, to understand and navigate the document at a high level. While all users can add summaries, auto-generated suggestions are currently only available to Google Workspace business customers. Building on grammar suggestions, Smart Compose, and autocorrect, we see this as another valuable step toward improving written communication in the workplace.\\n \\nModel Details\\n\\nAutomatically generated summaries would not be possible without the tremendous advances in ML for natural language understanding (NLU) and natural language generation (NLG) over the past five years, especially with the introduction of Transformer and Pegasus.\\n Model Details \\nAbstractive text summarization, which combines the individually challenging tasks of long document language understanding and generation, has been a long-standing problem in NLU and NLG research. A popular method for combining NLU and NLG is training an ML model using sequence-to-sequence learning, where the inputs are the document words, and the outputs are the summary words. A neural network then learns to map input tokens to output tokens. Early applications of the sequence-to-sequence paradigm used recurrent neural networks (RNNs) for both the encoder and decoder. \\n \\nThe introduction of Transformers provided a promising alternative to RNNs because Transformers use self-attention to provide better modeling of long input and output dependencies, which is critical in document summarization. Still, these models require large amounts of manually labeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization.\\n \\nThe combination of Transformers with self-supervised pre-training (e.g., BERT, GPT, T5) led to a major breakthrough in many NLU tasks for which limited labeled data is available. In self-supervised pre-training, a model uses large amounts of unlabeled text to learn general language understanding and generation capabilities. Then, in a subsequent fine-tuning stage, the model learns to apply these abilities on a specific task, such as summarization or question answering.\\n \\nThe Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization. In Pegasus pre-training, also called Gap Sentence Prediction (GSP), full sentences from unlabeled news articles and web documents are masked from the input and the model is required to reconstruct them, conditioned on the remaining unmasked sentences. In particular, GSP attempts to mask sentences that are considered essential to the document through different heuristics. The intuition is to make the pre-training as close as possible to the summarization task. Pegasus achieved state-of-the-art results on a varied set of summarization datasets. However, a number of challenges remained to apply this research advancement into a product.\\n \\nApplying Recent Research Advances to Google Docs\\n\\n Applying Recent Research Advances to Google Docs \\n\\n    Self-supervised pre-training results in an ML model that has general language understanding and generation capabilities, but a subsequent fine-tuning stage is critical for the model to adapt to the application domain. We fine-tuned early versions of our model on a corpus of documents with manually-generated summaries that were consistent with typical use cases.\\n \\n    However, early versions of this corpus suffered from inconsistencies and high variation because they included many types of documents, as well as many ways to write a summary — e.g., academic abstracts are typically long and detailed, while executive summaries are brief and punchy. This led to a model that was easily confused because it had been trained on so many different types of documents and summaries that it struggled to learn the relationships between any of them.\\n \\n    Fortunately, one of the key findings in the Pegasus work was that an effective pre-training phase required less supervised data in the fine-tuning stage. Some summarization benchmarks required as few as 1,000 fine-tuning examples for Pegasus to match the performance of Transformer baselines that saw 10,000+ supervised examples — suggesting that one could focus on quality rather than quantity. \\n \\n\\n    We carefully cleaned and filtered the fine-tuning data to contain training examples that were more consistent and represented a coherent definition of summaries. Despite the fact that we reduced the amount of training data, this led to a higher quality model. The key lesson, consistent with recent work in domains like dataset distillation, was that it was better to have a smaller, high quality dataset, than a larger, high-variance dataset. \\n    Once we trained the high quality model, we turned to the challenge of serving the model in production. While the Transformer version of the encoder-decoder architecture is the dominant approach to train models for sequence-to-sequence tasks like abstractive summarization, it can be inefficient and impractical to serve in real-world applications. The main inefficiency comes from the Transformer decoder where we generate the output summary token by token through autoregressive decoding. The decoding process becomes noticeably slow when summaries get longer since the decoder attends to all previously generated tokens at each step. RNNs are a more efficient architecture for decoding since there is no self-attention with previous tokens as in a Transformer model. \\n \\n    We used knowledge distillation, which is the process of transferring knowledge from a large model to a smaller more efficient model, to distill the Pegasus model into a hybrid architecture of a Transformer encoder and an RNN decoder. To improve efficiency we also reduced the number of RNN decoder layers. The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model. To further improve the latency and user experience, we serve the summarization model using TPUs, which provide significant speed ups and allow more requests to be handled by a single machine.\\n \\nOngoing Challenges and Next Steps\\n\\nWhile we are excited by the progress so far, there are a few challenges we are continuing to tackle:\\n Ongoing Challenges and Next Steps \\nConclusion\\n\\nOverall, we are thrilled that we can apply recent progress in NLU and NLG to continue assisting users with reading and writing. We hope the automatic suggestions now offered in Google Workspace make it easier for writers to annotate their documents with summaries, and help readers comprehend and navigate documents more easily. Conclusion \\nAcknowledgements\\n\\nThe authors would like to thank the many people across Google that contributed to this work: AJ Motika, Matt Pearson-Beck, Mia Chen, Mahdis Mahdieh, Halit Erdogan, Benjamin Lee, Ali Abdelhadi, Michelle Danoff, Vishnu Sivaji, Sneha Keshav, Aliya Baptista, Karishma Damani, DJ Lick, Yao Zhao, Peter Liu, Aurko Roy, Yonghui Wu, Shubhi Sareen, Andrew Dai, Mekhola Mukherjee, Yinan Wang, Mike Colagrosso, and Behnoosh Hariri.\\n.\\n Acknowledgements \\nLabels\\n \\nArchive\\n Feed'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":60}]},{"cell_type":"markdown","source":["Perforrm Abstractive Summarization using Pegasus-xsum"],"metadata":{"id":"2uGOjrkpJNU4"}},{"cell_type":"code","source":["##Import Dependencies\n","from transformers import PegasusForConditionalGeneration,PegasusTokenizer"],"metadata":{"id":"2l_CMY3PpAB2","executionInfo":{"status":"ok","timestamp":1648800432647,"user_tz":-360,"elapsed":383,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":68,"outputs":[]},{"cell_type":"code","source":["#Create tokenizer\n","tokenizer = PegasusTokenizer.from_pretrained('google/pegasus-xsum')\n"],"metadata":{"id":"4FRra3RYpCnF","executionInfo":{"status":"ok","timestamp":1648800434544,"user_tz":-360,"elapsed":416,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":69,"outputs":[]},{"cell_type":"code","source":["##Create Tokens - number representation of our text\n","tokens=tokenizer(Article,truncation=True,padding=\"longest\",return_tensors=\"pt\")"],"metadata":{"id":"qo22bV0kJuBK","executionInfo":{"status":"ok","timestamp":1648800459135,"user_tz":-360,"elapsed":56,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":71,"outputs":[]},{"cell_type":"code","source":["#Load model\n","model=PegasusForConditionalGeneration.from_pretrained(\"google/pegasus-xsum\")"],"metadata":{"id":"DmRe0i9PJbLk","executionInfo":{"status":"ok","timestamp":1648800459134,"user_tz":-360,"elapsed":19807,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":70,"outputs":[]},{"cell_type":"code","source":["tokens"],"metadata":{"id":"6aXIchstBhqA","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1648800459136,"user_tz":-360,"elapsed":53,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"3d54f26d-ef4b-4974-bc27-d645b3446991"},"execution_count":72,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'input_ids': tensor([[ 3339,   121, 22932, 95577,   115,  1058, 30556,   321,   223,   113,\n","           214,   108,   126,   137,   129,  2782,   112,   376,   164,   122,\n","           109,  2410,   113,  2010,   120,  3041,   115,   150,   115, 25347,\n","           290,   242,   151,  1574,   108,  1558,   108, 45939,   108,  2154,\n","           111,   109,   467,  1168,   124,   107,   434,   114,   177,  2199,\n","           117,   915,   108,  2557,   432,  1216,   126,   953,   114,  3598,\n","          5627,   113,   109,   674,   884,   115,   385,   112,  2448, 17195,\n","           126,   107,   611,   108, 14560,   114,  2199,  5627,   137,   129,\n","         80908,  2782,   111,   166,   121, 14499,   108,   704,   173,   114,\n","          2199,  2678,   117,  1215,   135,  5932,   107,   413,   225,   122,\n","           136,   108,   145,   938,  1487,   120,  1058, 30556,   239,  2093,\n","         11457,  3620,   112,  2637,  2199,  3655,   115,  1125,   601, 27533,\n","           108,   173,   157,   127,   293,   107,  1984,   145,  3626,   199,\n","           136,   140,  5443,   303,   114,  1157,   761,   143, 15929,   158,\n","           861,   120, 12888,   116,  2199,  1352,   111,   108,   173,  3375,\n","           108, 11457,   114,  9950,  5577,   710,  1261,  3180,   113,   109,\n","          2199,   601,   107,   611,   108,   109,  2199,  2678,  8273,   357,\n","           562,   492,  7013,   109,  7832,   130,   121,  1659,   108,   395,\n","           993, 24717,   112,   340,  3155,   109,  2199,  5627,   132, 16004,\n","           109,  7832,  9639,   107, 20378,   137,   163,   207,   136,  1201,\n","           108,   466,   122,   109,  7333,   108,   112,   630,   111,  5352,\n","           109,  2199,   134,   114,   281,   476,   107,  1041,   149,   780,\n","           137,   535, 27533,   108,  2253,   121, 22932,  3620,   127,   767,\n","           209,   293,   112,  1058, 56002,   260,   527,   107,  3671,   124,\n","         10947,  3620,   108,  4114,   110, 57519,   108,   111,  2253, 29188,\n","           108,   145,   236,   136,   130,   372,  2324,   863,  2153,  3024,\n","          1158,  1660,   115,   109,  4333,   107,  4870,  8247, 57336,  3943,\n","         27533,   192,   146,   129,   433,   347,   109,  6550,  8628,   115,\n","         22099,   118,   710,  1261,  1301,   143,  1400, 25703,   158,   111,\n","           710,  1261,  2233,   143, 21297,  1064,   158,   204,   109,   555,\n","           668,   231,   108,   704,   122,   109,  3709,   113, 51979,   111,\n","         51881,   107,  4870,  8247, 13609,  5551,  1352,  5906,  6520,  3884,\n","           108,   162,  5720,   109,  6367,  2782,  2722,   113,   300,  2199,\n","          1261,  1301,   111,  2233,   108,   148,   174,   114,   300,   121,\n","         11570,   575,   115,  1101, 25703,   111, 19778,  1064,   473,   107,\n","           202,   785,  1356,   118,  6839,  1101, 25703,   111, 19778,  1064,\n","           117,   569,   142, 22099,   861,   303,  5936,   121,   497,   121,\n","         69987,   761,   108,   241,   109, 12212,   127,   109,  2199,   989,\n","           108,   111,   109, 15487,   127,   109,  5627,   989,   107,   202,\n","         14849,   952,   237, 23153,   112,  1834,  3196, 14426,   112,  2940,\n","         14426,   107,  6236,  1160,   113,   109,  5936,   121,   497,   121,\n","         69987, 17142,   263, 27441, 14849,  3296,   143,  1156, 21323,   116,\n","           158,   118,   302,   109, 40753,   111, 44129,   107,   139,  3709,\n","           113, 38979,   735,   114,  7367,  2049,   112,   840, 21323,   116,\n","           262, 38979,   207,   813,   121, 65167,   112,   319,   340,  8177,\n","           113,   300,  3196,   111,  2940, 25233,   108,   162,   117,  1710,\n","           115,  2199,  5906,  6520,  3884,   107,  4587,   108,   219,  1581,\n","          1085,   423,  3912,   113,  6672, 11559,   335,   112,  1976, 16067,\n","           108,   167,   109, 15519,   113, 38979,  1600,   140,   146,   505,\n","           112,  2838,  2205,   109,   449,   121,  1313,   121,   544,   121,\n","          3904,   115,  2199,  5906,  6520,  3884,   107,   139,  1852,   113,\n","         38979,     1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","         1, 1, 1, 1, 1, 1, 1, 1]])}"]},"metadata":{},"execution_count":72}]},{"cell_type":"code","source":["#Summarize\n","summary= model.generate(**tokens)"],"metadata":{"id":"lnysFU8ZJxih","executionInfo":{"status":"ok","timestamp":1648800481374,"user_tz":-360,"elapsed":22250,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":73,"outputs":[]},{"cell_type":"code","source":["{**tokens}"],"metadata":{"id":"7iZSC6wsBh5u","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1648800481376,"user_tz":-360,"elapsed":73,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"b30ec37a-2adb-4131-a761-68c3239e029e"},"execution_count":74,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n","          1, 1, 1, 1, 1, 1, 1, 1]]),\n"," 'input_ids': tensor([[ 3339,   121, 22932, 95577,   115,  1058, 30556,   321,   223,   113,\n","            214,   108,   126,   137,   129,  2782,   112,   376,   164,   122,\n","            109,  2410,   113,  2010,   120,  3041,   115,   150,   115, 25347,\n","            290,   242,   151,  1574,   108,  1558,   108, 45939,   108,  2154,\n","            111,   109,   467,  1168,   124,   107,   434,   114,   177,  2199,\n","            117,   915,   108,  2557,   432,  1216,   126,   953,   114,  3598,\n","           5627,   113,   109,   674,   884,   115,   385,   112,  2448, 17195,\n","            126,   107,   611,   108, 14560,   114,  2199,  5627,   137,   129,\n","          80908,  2782,   111,   166,   121, 14499,   108,   704,   173,   114,\n","           2199,  2678,   117,  1215,   135,  5932,   107,   413,   225,   122,\n","            136,   108,   145,   938,  1487,   120,  1058, 30556,   239,  2093,\n","          11457,  3620,   112,  2637,  2199,  3655,   115,  1125,   601, 27533,\n","            108,   173,   157,   127,   293,   107,  1984,   145,  3626,   199,\n","            136,   140,  5443,   303,   114,  1157,   761,   143, 15929,   158,\n","            861,   120, 12888,   116,  2199,  1352,   111,   108,   173,  3375,\n","            108, 11457,   114,  9950,  5577,   710,  1261,  3180,   113,   109,\n","           2199,   601,   107,   611,   108,   109,  2199,  2678,  8273,   357,\n","            562,   492,  7013,   109,  7832,   130,   121,  1659,   108,   395,\n","            993, 24717,   112,   340,  3155,   109,  2199,  5627,   132, 16004,\n","            109,  7832,  9639,   107, 20378,   137,   163,   207,   136,  1201,\n","            108,   466,   122,   109,  7333,   108,   112,   630,   111,  5352,\n","            109,  2199,   134,   114,   281,   476,   107,  1041,   149,   780,\n","            137,   535, 27533,   108,  2253,   121, 22932,  3620,   127,   767,\n","            209,   293,   112,  1058, 56002,   260,   527,   107,  3671,   124,\n","          10947,  3620,   108,  4114,   110, 57519,   108,   111,  2253, 29188,\n","            108,   145,   236,   136,   130,   372,  2324,   863,  2153,  3024,\n","           1158,  1660,   115,   109,  4333,   107,  4870,  8247, 57336,  3943,\n","          27533,   192,   146,   129,   433,   347,   109,  6550,  8628,   115,\n","          22099,   118,   710,  1261,  1301,   143,  1400, 25703,   158,   111,\n","            710,  1261,  2233,   143, 21297,  1064,   158,   204,   109,   555,\n","            668,   231,   108,   704,   122,   109,  3709,   113, 51979,   111,\n","          51881,   107,  4870,  8247, 13609,  5551,  1352,  5906,  6520,  3884,\n","            108,   162,  5720,   109,  6367,  2782,  2722,   113,   300,  2199,\n","           1261,  1301,   111,  2233,   108,   148,   174,   114,   300,   121,\n","          11570,   575,   115,  1101, 25703,   111, 19778,  1064,   473,   107,\n","            202,   785,  1356,   118,  6839,  1101, 25703,   111, 19778,  1064,\n","            117,   569,   142, 22099,   861,   303,  5936,   121,   497,   121,\n","          69987,   761,   108,   241,   109, 12212,   127,   109,  2199,   989,\n","            108,   111,   109, 15487,   127,   109,  5627,   989,   107,   202,\n","          14849,   952,   237, 23153,   112,  1834,  3196, 14426,   112,  2940,\n","          14426,   107,  6236,  1160,   113,   109,  5936,   121,   497,   121,\n","          69987, 17142,   263, 27441, 14849,  3296,   143,  1156, 21323,   116,\n","            158,   118,   302,   109, 40753,   111, 44129,   107,   139,  3709,\n","            113, 38979,   735,   114,  7367,  2049,   112,   840, 21323,   116,\n","            262, 38979,   207,   813,   121, 65167,   112,   319,   340,  8177,\n","            113,   300,  3196,   111,  2940, 25233,   108,   162,   117,  1710,\n","            115,  2199,  5906,  6520,  3884,   107,  4587,   108,   219,  1581,\n","           1085,   423,  3912,   113,  6672, 11559,   335,   112,  1976, 16067,\n","            108,   167,   109, 15519,   113, 38979,  1600,   140,   146,   505,\n","            112,  2838,  2205,   109,   449,   121,  1313,   121,   544,   121,\n","           3904,   115,  2199,  5906,  6520,  3884,   107,   139,  1852,   113,\n","          38979,     1]])}"]},"metadata":{},"execution_count":74}]},{"cell_type":"code","source":["#Decode summary\n","tokenizer.decode(summary[0])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":36},"id":"_VU1YvqSJ2VK","executionInfo":{"status":"ok","timestamp":1648800481385,"user_tz":-360,"elapsed":70,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"3630ceac-2cf4-4c4a-945b-2ca4706f8669"},"execution_count":75,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'In our latest blog post, we describe how Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available.'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":75}]},{"cell_type":"markdown","source":["**Perforrm Abstractive Summarization using Pegasus-learge model**"],"metadata":{"id":"eMfQqLJg0RZx"}},{"cell_type":"code","source":["#Create Tokenizar using Pegasus-large model\n","tokenizer = PegasusTokenizer.from_pretrained('google/pegasus-large')"],"metadata":{"id":"RT4BRN3QJZSI","executionInfo":{"status":"ok","timestamp":1648800506083,"user_tz":-360,"elapsed":1116,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":76,"outputs":[]},{"cell_type":"code","source":["##Create Tokens - number representation of our text\n","tokens=tokenizer(Article,truncation=True,padding=\"longest\",return_tensors=\"pt\")"],"metadata":{"id":"03h8EshK0850","executionInfo":{"status":"ok","timestamp":1648800522768,"user_tz":-360,"elapsed":42,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":78,"outputs":[]},{"cell_type":"code","source":["#Load model\n","model=PegasusForConditionalGeneration.from_pretrained(\"google/pegasus-large\")"],"metadata":{"id":"rf9Gfv37o6yn","executionInfo":{"status":"ok","timestamp":1648800522767,"user_tz":-360,"elapsed":16688,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":77,"outputs":[]},{"cell_type":"code","source":["tokens"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xqy106QY1a9V","executionInfo":{"status":"ok","timestamp":1648800522770,"user_tz":-360,"elapsed":39,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"804a82d1-9f98-41e4-e58f-cc815ae95b63"},"execution_count":79,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'input_ids': tensor([[ 3339,   121, 22932,  ...,   109,   617,     1]]), 'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1]])}"]},"metadata":{},"execution_count":79}]},{"cell_type":"code","source":["#Summarize\n","summary= model.generate(**tokens)"],"metadata":{"id":"2O8p1JzL1_XO","executionInfo":{"status":"ok","timestamp":1648800679307,"user_tz":-360,"elapsed":156570,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":80,"outputs":[]},{"cell_type":"code","source":["{**tokens}"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hl0t4blD1_5r","executionInfo":{"status":"ok","timestamp":1648800679335,"user_tz":-360,"elapsed":42,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"266b02d8-c393-4286-f0c4-673e56b7ee7c"},"execution_count":81,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1]]),\n"," 'input_ids': tensor([[ 3339,   121, 22932,  ...,   109,   617,     1]])}"]},"metadata":{},"execution_count":81}]},{"cell_type":"code","source":["##Output/summary in tokens\n","summary[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"boDPqc-f1_8q","executionInfo":{"status":"ok","timestamp":1648800679340,"user_tz":-360,"elapsed":43,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"0423d595-0003-40f5-aafe-5a5f685e640a"},"execution_count":82,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([    0,  4870,  8247, 57336,  3943, 27533,   192,   146,   129,   433,\n","          347,   109,  6550,  8628,   115, 22099,   118,   710,  1261,  1301,\n","          143,  1400, 25703,   158,   111,   710,  1261,  2233,   143, 21297,\n","         1064,   158,   204,   109,   555,   668,   231,   108,   704,   122,\n","          109,  3709,   113, 51979,   111, 51881,   107,   222,   813,   121,\n","        83465,   761,   108,   114,   861, 23153,   112,  1138,   423,  3912,\n","          113,  1596, 53541,   335,   112,  1976, 16067,   108,   167,   109,\n","        15519,   113, 38979,  1600,   140,   146,   505,   112,  2838,  2205,\n","          109,   449,   121,  1313,   121,   544,   121,  3904,   115,  2199,\n","         5906,  6520,  3884,   107,   222,   813,   121, 83465,   761,   108,\n","          114,   861, 23153,   112,  1138,   423,  3912,   113,  1596, 53541,\n","          335,   112,  1976, 16067,   108,   167,   109, 15519,   113, 38979,\n","         1600,   140,   146,   505,   112,  2838,  2205,   109,   449,   121,\n","         1313,   121,   544,   121,  3904,   115,  2199,  5906,  6520,  3884,\n","          107,   222,   813,   121, 83465,   761,   108,   114,   861, 23153,\n","          112,  1138,   423,  3912,   113,  1596, 53541,   335,   112,  1976,\n","        16067,   108,   167,   109, 15519,   113, 38979,  1600,   140,   146,\n","          505,   112,  2838,  2205,   109,   449,   121,  1313,   121,   544,\n","          121,  3904,   115,  2199,  5906,  6520,  3884,   107,   222,   813,\n","          121, 83465,   761,   108,   114,   861, 23153,   112,  1138,   423,\n","         3912,   113,  1596, 53541,   335,   112,  1976, 16067,   108,   167,\n","          109, 15519,   113, 38979,  1600,   140,   146,   505,   112,  2838,\n","         2205,   109,   449,   121,  1313,   121,   544,   121,  3904,   115,\n","         2199,  5906,  6520,  3884,   107,     1])"]},"metadata":{},"execution_count":82}]},{"cell_type":"code","source":["#Decode summary\n","tokenizer.decode(summary[0])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":127},"id":"SLQINFkC2AA6","executionInfo":{"status":"ok","timestamp":1648800679342,"user_tz":-360,"elapsed":42,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"b28fe629-a772-4d96-ebfa-b677db11f6af"},"execution_count":83,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'Model Details Automatically generated summaries would not be possible without the tremendous advances in ML for natural language understanding (NLU) and natural language generation (NLG) over the past five years, especially with the introduction of Transformer and Pegasus. In self-supervised learning, a model learns to apply large amounts of unlabeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization. In self-supervised learning, a model learns to apply large amounts of unlabeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization. In self-supervised learning, a model learns to apply large amounts of unlabeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization. In self-supervised learning, a model learns to apply large amounts of unlabeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization.'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":83}]},{"cell_type":"code","source":[""],"metadata":{"id":"UkHPcUg93F19"},"execution_count":null,"outputs":[]}]}