Extractive Summarization for  long text using Base Transformer Pipeline.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Extractive Summarization for  long text using Base Transformer Pipeline.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyOzM0JeGhIbyHdMuJ5Yy/n3"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"80913417d4ae4471bf27eadea8f46ff2":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_f38bea0ee79849d28e122801150d12e9","IPY_MODEL_3a05d5c9996e48579884330b16d0add9","IPY_MODEL_83628ec2c00f45f0bb4fa88f4c8ca56e"],"layout":"IPY_MODEL_13c5e3ee209b4855ab6ece2e037f088d"}},"f38bea0ee79849d28e122801150d12e9":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_75c6766ab0804bc8a86f59d80da85e5b","placeholder":"​","style":"IPY_MODEL_2adf8cad6ada441b94a5168dc33d0631","value":"Downloading: 100%"}},"3a05d5c9996e48579884330b16d0add9":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_981765e64b834b63b8bc83b55fbfa03e","max":1802,"min":0,"orientation":"horizontal","style":"IPY_MODEL_266c8869c4a5416bbc0b57be3e353d81","value":1802}},"83628ec2c00f45f0bb4fa88f4c8ca56e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2010b60400174f5fb1c8465a66a0af7e","placeholder":"​","style":"IPY_MODEL_a9713d018681437cb73285a8d255407d","value":" 1.76k/1.76k [00:00&lt;00:00, 31.0kB/s]"}},"13c5e3ee209b4855ab6ece2e037f088d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"75c6766ab0804bc8a86f59d80da85e5b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"2adf8cad6ada441b94a5168dc33d0631":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"981765e64b834b63b8bc83b55fbfa03e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"266c8869c4a5416bbc0b57be3e353d81":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2010b60400174f5fb1c8465a66a0af7e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a9713d018681437cb73285a8d255407d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4d677a2db44546f5b42d52a8b7aaacae":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_bf4807fca29a4087b3cb1186258256da","IPY_MODEL_0593332902e149909474a973ea21b3de","IPY_MODEL_a5976923547849c28d0a0b80163b4d43"],"layout":"IPY_MODEL_27674fe706364c53988e8a4ca85e3966"}},"bf4807fca29a4087b3cb1186258256da":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_95243ab9debb4ff285222079639a1470","placeholder":"​","style":"IPY_MODEL_3743ba4454aa4599ba21cb9f42d5e256","value":"Downloading: 100%"}},"0593332902e149909474a973ea21b3de":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_2437fc0451dd4e2d9950e645255e4018","max":1222317369,"min":0,"orientation":"horizontal","style":"IPY_MODEL_ae12493d022c4bc6aad974901cc397c5","value":1222317369}},"a5976923547849c28d0a0b80163b4d43":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2fd3b02389e64df4bed518325b2ef131","placeholder":"​","style":"IPY_MODEL_b393017802c84bc083e14f285f4506b2","value":" 1.14G/1.14G [00:25&lt;00:00, 45.5MB/s]"}},"27674fe706364c53988e8a4ca85e3966":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"95243ab9debb4ff285222079639a1470":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3743ba4454aa4599ba21cb9f42d5e256":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2437fc0451dd4e2d9950e645255e4018":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ae12493d022c4bc6aad974901cc397c5":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2fd3b02389e64df4bed518325b2ef131":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b393017802c84bc083e14f285f4506b2":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5fae42ee765b48d48513690d8c867c12":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_433a78190c8445f18fa866c4d2199f0c","IPY_MODEL_8a15e421ba5f474fa2c6bf170c342abb","IPY_MODEL_bde21cee363d481689f4cafc78f39e3b"],"layout":"IPY_MODEL_32b333a675bc4757b236a21b0015dfdc"}},"433a78190c8445f18fa866c4d2199f0c":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_43f55c960ff7490c9e41ab3563fabb17","placeholder":"​","style":"IPY_MODEL_c4d1c21ab14843ecb6cf61297900b98b","value":"Downloading: 100%"}},"8a15e421ba5f474fa2c6bf170c342abb":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_65bce5dcfda746a2be2d9bb9a73bd863","max":26,"min":0,"orientation":"horizontal","style":"IPY_MODEL_fe43a9f386b9406398e17abdbb8cf0d4","value":26}},"bde21cee363d481689f4cafc78f39e3b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_15f124460a39440bafc4393f10d84457","placeholder":"​","style":"IPY_MODEL_9708778813fe4366a1fbbe818313d6a6","value":" 26.0/26.0 [00:00&lt;00:00, 488B/s]"}},"32b333a675bc4757b236a21b0015dfdc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"43f55c960ff7490c9e41ab3563fabb17":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c4d1c21ab14843ecb6cf61297900b98b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"65bce5dcfda746a2be2d9bb9a73bd863":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fe43a9f386b9406398e17abdbb8cf0d4":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"15f124460a39440bafc4393f10d84457":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9708778813fe4366a1fbbe818313d6a6":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ac1d8f1aff91478d9c6c8ebefbff5c43":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3e705b7a8056474e8990bad7e8b584ed","IPY_MODEL_14c9670c0dab4b949c8f14fdf4b27ff5","IPY_MODEL_6f80a6d8932b4822908f8c41dc51a80f"],"layout":"IPY_MODEL_0c19129c1c7d4940ac2f2bdfc32b5d95"}},"3e705b7a8056474e8990bad7e8b584ed":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_210dfed8792b40339a0353fec72611d4","placeholder":"​","style":"IPY_MODEL_3d3808a6750f4813bf678e2c0a122114","value":"Downloading: 100%"}},"14c9670c0dab4b949c8f14fdf4b27ff5":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_64fedfd8031744aa973ed7974df66727","max":898822,"min":0,"orientation":"horizontal","style":"IPY_MODEL_0cf2e6f1ba7947eda9bbd80c1ba4eb85","value":898822}},"6f80a6d8932b4822908f8c41dc51a80f":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_dfaa2302c6764c89adb729d472d5d7af","placeholder":"​","style":"IPY_MODEL_4d66d20e3b5b4abea79ac3e7dff89bab","value":" 878k/878k [00:00&lt;00:00, 1.76MB/s]"}},"0c19129c1c7d4940ac2f2bdfc32b5d95":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"210dfed8792b40339a0353fec72611d4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3d3808a6750f4813bf678e2c0a122114":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"64fedfd8031744aa973ed7974df66727":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0cf2e6f1ba7947eda9bbd80c1ba4eb85":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"dfaa2302c6764c89adb729d472d5d7af":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4d66d20e3b5b4abea79ac3e7dff89bab":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"16adffb42f3e4adbaf04a67a62690067":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_6e93b7e90ddd426db2f532bc472a28e4","IPY_MODEL_03cfa25b37cd43c78ee84dcad2a1cd76","IPY_MODEL_4a75660144724405bcac959a4e01b024"],"layout":"IPY_MODEL_b20effe13fa54227af47809d1624f5e5"}},"6e93b7e90ddd426db2f532bc472a28e4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c43c23ea432f4c8887e42d94f71d8415","placeholder":"​","style":"IPY_MODEL_af0c67121c504fd097c7568d91e38192","value":"Downloading: 100%"}},"03cfa25b37cd43c78ee84dcad2a1cd76":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_ca52bd17e0404f1ba307067c671ea3bb","max":456318,"min":0,"orientation":"horizontal","style":"IPY_MODEL_b358de2bc9ff4d3ba09079793699b6f8","value":456318}},"4a75660144724405bcac959a4e01b024":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b8d1c3fe9d7a4989901c46c82fa38a68","placeholder":"​","style":"IPY_MODEL_4189c1302c9745d8a657ce844b2cb435","value":" 446k/446k [00:00&lt;00:00, 3.90MB/s]"}},"b20effe13fa54227af47809d1624f5e5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c43c23ea432f4c8887e42d94f71d8415":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"af0c67121c504fd097c7568d91e38192":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ca52bd17e0404f1ba307067c671ea3bb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b358de2bc9ff4d3ba09079793699b6f8":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"b8d1c3fe9d7a4989901c46c82fa38a68":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4189c1302c9745d8a657ce844b2cb435":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"code","execution_count":2,"metadata":{"id":"K6mdpavuJD7r","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1648830047163,"user_tz":-360,"elapsed":11880,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"3e9fd737-96a3-4726-937c-3c4064b51846"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting transformers\n","  Downloading transformers-4.17.0-py3-none-any.whl (3.8 MB)\n","\u001b[K     |████████████████████████████████| 3.8 MB 4.5 MB/s \n","\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n","Collecting pyyaml>=5.1\n","  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n","\u001b[K     |████████████████████████████████| 596 kB 38.7 MB/s \n","\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Collecting tokenizers!=0.11.3,>=0.11.1\n","  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n","\u001b[K     |████████████████████████████████| 6.5 MB 51.2 MB/s \n","\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n","Collecting huggingface-hub<1.0,>=0.1.0\n","  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n","\u001b[K     |████████████████████████████████| 67 kB 4.7 MB/s \n","\u001b[?25hCollecting sacremoses\n","  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n","\u001b[K     |████████████████████████████████| 895 kB 46.1 MB/s \n","\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n","Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n","Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n","  Attempting uninstall: pyyaml\n","    Found existing installation: PyYAML 3.13\n","    Uninstalling PyYAML-3.13:\n","      Successfully uninstalled PyYAML-3.13\n","Successfully installed huggingface-hub-0.4.0 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.17.0\n"]}],"source":["#Install packages\n","!pip install transformers\n","#!pip install beautifulsoup4 pandas selenium\n","#!pip install lxml\n","#!pip install html5lib\n","#!pip install requests"]},{"cell_type":"code","source":["## Import Libraries\n","from transformers import pipeline\n","from bs4 import BeautifulSoup\n","import requests\n"],"metadata":{"id":"-aMkx3wPsaF_","executionInfo":{"status":"ok","timestamp":1648830058269,"user_tz":-360,"elapsed":11115,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":3,"outputs":[]},{"cell_type":"code","source":["#Source URL\n","URL=\"https://ai.googleblog.com/2022/03/auto-generated-summaries-in-google-docs.html\""],"metadata":{"id":"9sbWWzEPytOD","executionInfo":{"status":"ok","timestamp":1648830058271,"user_tz":-360,"elapsed":55,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["## Request for the website\n","source=requests.get(URL).text\n"],"metadata":{"id":"HJXQrp3byugE","executionInfo":{"status":"ok","timestamp":1648830058272,"user_tz":-360,"elapsed":53,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["##BeautifulSoup -For parsing webpage\n","soup=BeautifulSoup(source,'html')\n","#print(soup.prettify())"],"metadata":{"id":"TYx_V9QPVtQ4","executionInfo":{"status":"ok","timestamp":1648830058878,"user_tz":-360,"elapsed":655,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["#Extract titles and text \n","results=soup.find_all(['h2','b','p'])\n","#results"],"metadata":{"id":"68gb7sHf22ma","executionInfo":{"status":"ok","timestamp":1648830058880,"user_tz":-360,"elapsed":14,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["# Remove label\n","text=[result.text for result in results]\n","Article=' '.join(text)\n","Article"],"metadata":{"id":"0fbQn-Hwu_bH","executionInfo":{"status":"ok","timestamp":1648830079985,"user_tz":-360,"elapsed":442,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"colab":{"base_uri":"https://localhost:8080/","height":765},"outputId":"cf646008-e437-432d-adb0-d2fb086e63d0"},"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'\\n\\nAuto-generated Summaries in Google Docs\\n\\n \\nFor many of us, it can be challenging to keep up with the volume of documents that arrive in our inboxes every day: reports, reviews, briefs, policies and the list goes on. When a new document is received, readers often wish it included a brief summary of the main points in order to effectively prioritize it. However, composing a document summary can be cognitively challenging and time-consuming, especially when a document writer is starting from scratch.\\n \\nTo help with this, we recently announced that Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available. Today we describe how this was enabled using a machine learning (ML) model that comprehends document text and, when confident, generates a 1-2 sentence natural language description of the document content. However, the document writer maintains full control — accepting the suggestion as-is, making necessary edits to better capture the document summary or ignoring the suggestion altogether. Readers can also use this section, along with the outline, to understand and navigate the document at a high level. While all users can add summaries, auto-generated suggestions are currently only available to Google Workspace business customers. Building on grammar suggestions, Smart Compose, and autocorrect, we see this as another valuable step toward improving written communication in the workplace.\\n \\nModel Details\\n\\nAutomatically generated summaries would not be possible without the tremendous advances in ML for natural language understanding (NLU) and natural language generation (NLG) over the past five years, especially with the introduction of Transformer and Pegasus.\\n Model Details \\nAbstractive text summarization, which combines the individually challenging tasks of long document language understanding and generation, has been a long-standing problem in NLU and NLG research. A popular method for combining NLU and NLG is training an ML model using sequence-to-sequence learning, where the inputs are the document words, and the outputs are the summary words. A neural network then learns to map input tokens to output tokens. Early applications of the sequence-to-sequence paradigm used recurrent neural networks (RNNs) for both the encoder and decoder. \\n \\nThe introduction of Transformers provided a promising alternative to RNNs because Transformers use self-attention to provide better modeling of long input and output dependencies, which is critical in document summarization. Still, these models require large amounts of manually labeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization.\\n \\nThe combination of Transformers with self-supervised pre-training (e.g., BERT, GPT, T5) led to a major breakthrough in many NLU tasks for which limited labeled data is available. In self-supervised pre-training, a model uses large amounts of unlabeled text to learn general language understanding and generation capabilities. Then, in a subsequent fine-tuning stage, the model learns to apply these abilities on a specific task, such as summarization or question answering.\\n \\nThe Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization. In Pegasus pre-training, also called Gap Sentence Prediction (GSP), full sentences from unlabeled news articles and web documents are masked from the input and the model is required to reconstruct them, conditioned on the remaining unmasked sentences. In particular, GSP attempts to mask sentences that are considered essential to the document through different heuristics. The intuition is to make the pre-training as close as possible to the summarization task. Pegasus achieved state-of-the-art results on a varied set of summarization datasets. However, a number of challenges remained to apply this research advancement into a product.\\n \\nApplying Recent Research Advances to Google Docs\\n\\n Applying Recent Research Advances to Google Docs \\n\\n    Self-supervised pre-training results in an ML model that has general language understanding and generation capabilities, but a subsequent fine-tuning stage is critical for the model to adapt to the application domain. We fine-tuned early versions of our model on a corpus of documents with manually-generated summaries that were consistent with typical use cases.\\n \\n    However, early versions of this corpus suffered from inconsistencies and high variation because they included many types of documents, as well as many ways to write a summary — e.g., academic abstracts are typically long and detailed, while executive summaries are brief and punchy. This led to a model that was easily confused because it had been trained on so many different types of documents and summaries that it struggled to learn the relationships between any of them.\\n \\n    Fortunately, one of the key findings in the Pegasus work was that an effective pre-training phase required less supervised data in the fine-tuning stage. Some summarization benchmarks required as few as 1,000 fine-tuning examples for Pegasus to match the performance of Transformer baselines that saw 10,000+ supervised examples — suggesting that one could focus on quality rather than quantity. \\n \\n\\n    We carefully cleaned and filtered the fine-tuning data to contain training examples that were more consistent and represented a coherent definition of summaries. Despite the fact that we reduced the amount of training data, this led to a higher quality model. The key lesson, consistent with recent work in domains like dataset distillation, was that it was better to have a smaller, high quality dataset, than a larger, high-variance dataset. \\n    Once we trained the high quality model, we turned to the challenge of serving the model in production. While the Transformer version of the encoder-decoder architecture is the dominant approach to train models for sequence-to-sequence tasks like abstractive summarization, it can be inefficient and impractical to serve in real-world applications. The main inefficiency comes from the Transformer decoder where we generate the output summary token by token through autoregressive decoding. The decoding process becomes noticeably slow when summaries get longer since the decoder attends to all previously generated tokens at each step. RNNs are a more efficient architecture for decoding since there is no self-attention with previous tokens as in a Transformer model. \\n \\n    We used knowledge distillation, which is the process of transferring knowledge from a large model to a smaller more efficient model, to distill the Pegasus model into a hybrid architecture of a Transformer encoder and an RNN decoder. To improve efficiency we also reduced the number of RNN decoder layers. The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model. To further improve the latency and user experience, we serve the summarization model using TPUs, which provide significant speed ups and allow more requests to be handled by a single machine.\\n \\nOngoing Challenges and Next Steps\\n\\nWhile we are excited by the progress so far, there are a few challenges we are continuing to tackle:\\n Ongoing Challenges and Next Steps \\nConclusion\\n\\nOverall, we are thrilled that we can apply recent progress in NLU and NLG to continue assisting users with reading and writing. We hope the automatic suggestions now offered in Google Workspace make it easier for writers to annotate their documents with summaries, and help readers comprehend and navigate documents more easily. Conclusion \\nAcknowledgements\\n\\nThe authors would like to thank the many people across Google that contributed to this work: AJ Motika, Matt Pearson-Beck, Mia Chen, Mahdis Mahdieh, Halit Erdogan, Benjamin Lee, Ali Abdelhadi, Michelle Danoff, Vishnu Sivaji, Sneha Keshav, Aliya Baptista, Karishma Damani, DJ Lick, Yao Zhao, Peter Liu, Aurko Roy, Yonghui Wu, Shubhi Sareen, Andrew Dai, Mekhola Mukherjee, Yinan Wang, Mike Colagrosso, and Behnoosh Hariri.\\n.\\n Acknowledgements \\nLabels\\n \\nArchive\\n Feed'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":9}]},{"cell_type":"markdown","source":["**Chunk Article**"],"metadata":{"id":"d9SYMiVxNSjW"}},{"cell_type":"code","source":["## Modify punctuations\n","Article =Article .replace('.','.<eos>')\n","Article =Article .replace('!','!<eos>')\n","Article =Article .replace('?','?<eos>')\n","sentences=Article.split('<eos>')\n"],"metadata":{"id":"w1xVnvjEWtRn","executionInfo":{"status":"ok","timestamp":1648830253089,"user_tz":-360,"elapsed":404,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","source":["sentences"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PSqz8hC37NAW","executionInfo":{"status":"ok","timestamp":1648826997145,"user_tz":-360,"elapsed":94,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"a25aaa89-7ab3-43d9-e1e9-4c2bc4cc9ec1"},"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['\\n            Blog\\n           \\n\\nAuto-generated Summaries in Google Docs\\n\\n \\nFor many of us, it can be challenging to keep up with the volume of documents that arrive in our inboxes every day: reports, reviews, briefs, policies and the list goes on.',\n"," ' When a new document is received, readers often wish it included a brief summary of the main points in order to effectively prioritize it.',\n"," ' However, composing a document summary can be cognitively challenging and time-consuming, especially when a document writer is starting from scratch.',\n"," '\\n \\nTo help with this, we recently announced that Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available.',\n"," ' Today we describe how this was enabled using a machine learning (ML) model that comprehends document text and, when confident, generates a 1-2 sentence natural language description of the document content.',\n"," ' However, the document writer maintains full control — accepting the suggestion as-is, making necessary edits to better capture the document summary or ignoring the suggestion altogether.',\n"," ' Readers can also use this section, along with the outline, to understand and navigate the document at a high level.',\n"," ' While all users can add summaries, auto-generated suggestions are currently only available to Google Workspace business customers.',\n"," ' Building on grammar suggestions, Smart Compose, and autocorrect, we see this as another valuable step toward improving written communication in the workplace.',\n"," '\\n \\nModel Details\\n\\nAutomatically generated summaries would not be possible without the tremendous advances in ML for natural language understanding (NLU) and natural language generation (NLG) over the past five years, especially with the introduction of Transformer and Pegasus.',\n"," '\\n Model Details \\nAbstractive text summarization, which combines the individually challenging tasks of long document language understanding and generation, has been a long-standing problem in NLU and NLG research.',\n"," ' A popular method for combining NLU and NLG is training an ML model using sequence-to-sequence learning, where the inputs are the document words, and the outputs are the summary words.',\n"," ' A neural network then learns to map input tokens to output tokens.',\n"," ' Early applications of the sequence-to-sequence paradigm used recurrent neural networks (RNNs) for both the encoder and decoder.',\n"," ' \\n \\nThe introduction of Transformers provided a promising alternative to RNNs because Transformers use self-attention to provide better modeling of long input and output dependencies, which is critical in document summarization.',\n"," ' Still, these models require large amounts of manually labeled data to train sufficiently, so the advent of Transformers alone was not enough to significantly advance the state-of-the-art in document summarization.',\n"," '\\n \\nThe combination of Transformers with self-supervised pre-training (e.',\n"," 'g.',\n"," ', BERT, GPT, T5) led to a major breakthrough in many NLU tasks for which limited labeled data is available.',\n"," ' In self-supervised pre-training, a model uses large amounts of unlabeled text to learn general language understanding and generation capabilities.',\n"," ' Then, in a subsequent fine-tuning stage, the model learns to apply these abilities on a specific task, such as summarization or question answering.',\n"," '\\n \\nThe Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization.',\n"," ' In Pegasus pre-training, also called Gap Sentence Prediction (GSP), full sentences from unlabeled news articles and web documents are masked from the input and the model is required to reconstruct them, conditioned on the remaining unmasked sentences.',\n"," ' In particular, GSP attempts to mask sentences that are considered essential to the document through different heuristics.',\n"," ' The intuition is to make the pre-training as close as possible to the summarization task.',\n"," ' Pegasus achieved state-of-the-art results on a varied set of summarization datasets.',\n"," ' However, a number of challenges remained to apply this research advancement into a product.',\n"," '\\n \\nApplying Recent Research Advances to Google Docs\\n\\n Applying Recent Research Advances to Google Docs \\n\\n    Self-supervised pre-training results in an ML model that has general language understanding and generation capabilities, but a subsequent fine-tuning stage is critical for the model to adapt to the application domain.',\n"," ' We fine-tuned early versions of our model on a corpus of documents with manually-generated summaries that were consistent with typical use cases.',\n"," '\\n \\n    However, early versions of this corpus suffered from inconsistencies and high variation because they included many types of documents, as well as many ways to write a summary — e.',\n"," 'g.',\n"," ', academic abstracts are typically long and detailed, while executive summaries are brief and punchy.',\n"," ' This led to a model that was easily confused because it had been trained on so many different types of documents and summaries that it struggled to learn the relationships between any of them.',\n"," '\\n \\n    Fortunately, one of the key findings in the Pegasus work was that an effective pre-training phase required less supervised data in the fine-tuning stage.',\n"," ' Some summarization benchmarks required as few as 1,000 fine-tuning examples for Pegasus to match the performance of Transformer baselines that saw 10,000+ supervised examples — suggesting that one could focus on quality rather than quantity.',\n"," ' \\n \\n\\n    We carefully cleaned and filtered the fine-tuning data to contain training examples that were more consistent and represented a coherent definition of summaries.',\n"," ' Despite the fact that we reduced the amount of training data, this led to a higher quality model.',\n"," ' The key lesson, consistent with recent work in domains like dataset distillation, was that it was better to have a smaller, high quality dataset, than a larger, high-variance dataset.',\n"," ' \\n    Once we trained the high quality model, we turned to the challenge of serving the model in production.',\n"," ' While the Transformer version of the encoder-decoder architecture is the dominant approach to train models for sequence-to-sequence tasks like abstractive summarization, it can be inefficient and impractical to serve in real-world applications.',\n"," ' The main inefficiency comes from the Transformer decoder where we generate the output summary token by token through autoregressive decoding.',\n"," ' The decoding process becomes noticeably slow when summaries get longer since the decoder attends to all previously generated tokens at each step.',\n"," ' RNNs are a more efficient architecture for decoding since there is no self-attention with previous tokens as in a Transformer model.',\n"," ' \\n \\n    We used knowledge distillation, which is the process of transferring knowledge from a large model to a smaller more efficient model, to distill the Pegasus model into a hybrid architecture of a Transformer encoder and an RNN decoder.',\n"," ' To improve efficiency we also reduced the number of RNN decoder layers.',\n"," ' The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model.',\n"," ' To further improve the latency and user experience, we serve the summarization model using TPUs, which provide significant speed ups and allow more requests to be handled by a single machine.',\n"," '\\n \\nOngoing Challenges and Next Steps\\n\\nWhile we are excited by the progress so far, there are a few challenges we are continuing to tackle:\\n Ongoing Challenges and Next Steps \\nConclusion\\n\\nOverall, we are thrilled that we can apply recent progress in NLU and NLG to continue assisting users with reading and writing.',\n"," ' We hope the automatic suggestions now offered in Google Workspace make it easier for writers to annotate their documents with summaries, and help readers comprehend and navigate documents more easily.',\n"," ' Conclusion \\nAcknowledgements\\n\\nThe authors would like to thank the many people across Google that contributed to this work: AJ Motika, Matt Pearson-Beck, Mia Chen, Mahdis Mahdieh, Halit Erdogan, Benjamin Lee, Ali Abdelhadi, Michelle Danoff, Vishnu Sivaji, Sneha Keshav, Aliya Baptista, Karishma Damani, DJ Lick, Yao Zhao, Peter Liu, Aurko Roy, Yonghui Wu, Shubhi Sareen, Andrew Dai, Mekhola Mukherjee, Yinan Wang, Mike Colagrosso, and Behnoosh Hariri.',\n"," '\\n.',\n"," '\\n Acknowledgements \\nLabels\\n \\nArchive\\n Feed']"]},"metadata":{},"execution_count":9}]},{"cell_type":"code","source":["#Define Chunk size\n","max_chunk=500\n","current_chunk=0\n","chunks=[]\n","\n","for sentence in sentences:\n","  if len(chunks)==current_chunk +1:\n","    if len(chunks[current_chunk])+len(sentence.split(' '))<=max_chunk:\n","      chunks[current_chunk].extend(sentence.split(' '))\n","    else:\n","        current_chunk += 1\n","        chunks.append(sentence.split(' '))\n","  else:\n","          print(current_chunk)\n","          chunks.append(sentence.split(' '))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Bi-AEW7I8W7w","executionInfo":{"status":"ok","timestamp":1648830256220,"user_tz":-360,"elapsed":412,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"56eea27f-1ded-418e-d945-79a2aede2b00"},"execution_count":12,"outputs":[{"output_type":"stream","name":"stdout","text":["0\n"]}]},{"cell_type":"code","source":["#Join the chunked word \n","for chunk_id in range(len(chunks)):\n","  chunks[chunk_id]=' '.join(chunks[chunk_id])"],"metadata":{"id":"Q2jIajLP_zWn","executionInfo":{"status":"ok","timestamp":1648830259381,"user_tz":-360,"elapsed":678,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":13,"outputs":[]},{"cell_type":"code","source":["chunks[1]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":108},"id":"sJGTIe9TAJOz","executionInfo":{"status":"ok","timestamp":1648830260045,"user_tz":-360,"elapsed":13,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"826f06d5-5550-4363-aeef-6157409d5592"},"execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'\\n \\nThe Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization.  In Pegasus pre-training, also called Gap Sentence Prediction (GSP), full sentences from unlabeled news articles and web documents are masked from the input and the model is required to reconstruct them, conditioned on the remaining unmasked sentences.  In particular, GSP attempts to mask sentences that are considered essential to the document through different heuristics.  The intuition is to make the pre-training as close as possible to the summarization task.  Pegasus achieved state-of-the-art results on a varied set of summarization datasets.  However, a number of challenges remained to apply this research advancement into a product. \\n \\nApplying Recent Research Advances to Google Docs\\n\\n Applying Recent Research Advances to Google Docs \\n\\n    Self-supervised pre-training results in an ML model that has general language understanding and generation capabilities, but a subsequent fine-tuning stage is critical for the model to adapt to the application domain.  We fine-tuned early versions of our model on a corpus of documents with manually-generated summaries that were consistent with typical use cases. \\n \\n    However, early versions of this corpus suffered from inconsistencies and high variation because they included many types of documents, as well as many ways to write a summary — e. g. , academic abstracts are typically long and detailed, while executive summaries are brief and punchy.  This led to a model that was easily confused because it had been trained on so many different types of documents and summaries that it struggled to learn the relationships between any of them. \\n \\n    Fortunately, one of the key findings in the Pegasus work was that an effective pre-training phase required less supervised data in the fine-tuning stage.  Some summarization benchmarks required as few as 1,000 fine-tuning examples for Pegasus to match the performance of Transformer baselines that saw 10,000+ supervised examples — suggesting that one could focus on quality rather than quantity.  \\n \\n\\n    We carefully cleaned and filtered the fine-tuning data to contain training examples that were more consistent and represented a coherent definition of summaries.  Despite the fact that we reduced the amount of training data, this led to a higher quality model.  The key lesson, consistent with recent work in domains like dataset distillation, was that it was better to have a smaller, high quality dataset, than a larger, high-variance dataset.  \\n    Once we trained the high quality model, we turned to the challenge of serving the model in production.  While the Transformer version of the encoder-decoder architecture is the dominant approach to train models for sequence-to-sequence tasks like abstractive summarization, it can be inefficient and impractical to serve in real-world applications.  The main inefficiency comes from the Transformer decoder where we generate the output summary token by token through autoregressive decoding.'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":14}]},{"cell_type":"code","source":["len(chunks[1].split(' '))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"J3VU4rkz-09b","executionInfo":{"status":"ok","timestamp":1648830263011,"user_tz":-360,"elapsed":617,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"3d4b91c6-68d9-4289-ae39-b2cedb91fccc"},"execution_count":15,"outputs":[{"output_type":"execute_result","data":{"text/plain":["497"]},"metadata":{},"execution_count":15}]},{"cell_type":"markdown","source":["**Summarization Task**"],"metadata":{"id":"5cu8J_7hczN_"}},{"cell_type":"code","source":["#Load summarizationn pipeline\n","summarizer=pipeline('summarization')\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":196,"referenced_widgets":["80913417d4ae4471bf27eadea8f46ff2","f38bea0ee79849d28e122801150d12e9","3a05d5c9996e48579884330b16d0add9","83628ec2c00f45f0bb4fa88f4c8ca56e","13c5e3ee209b4855ab6ece2e037f088d","75c6766ab0804bc8a86f59d80da85e5b","2adf8cad6ada441b94a5168dc33d0631","981765e64b834b63b8bc83b55fbfa03e","266c8869c4a5416bbc0b57be3e353d81","2010b60400174f5fb1c8465a66a0af7e","a9713d018681437cb73285a8d255407d","4d677a2db44546f5b42d52a8b7aaacae","bf4807fca29a4087b3cb1186258256da","0593332902e149909474a973ea21b3de","a5976923547849c28d0a0b80163b4d43","27674fe706364c53988e8a4ca85e3966","95243ab9debb4ff285222079639a1470","3743ba4454aa4599ba21cb9f42d5e256","2437fc0451dd4e2d9950e645255e4018","ae12493d022c4bc6aad974901cc397c5","2fd3b02389e64df4bed518325b2ef131","b393017802c84bc083e14f285f4506b2","5fae42ee765b48d48513690d8c867c12","433a78190c8445f18fa866c4d2199f0c","8a15e421ba5f474fa2c6bf170c342abb","bde21cee363d481689f4cafc78f39e3b","32b333a675bc4757b236a21b0015dfdc","43f55c960ff7490c9e41ab3563fabb17","c4d1c21ab14843ecb6cf61297900b98b","65bce5dcfda746a2be2d9bb9a73bd863","fe43a9f386b9406398e17abdbb8cf0d4","15f124460a39440bafc4393f10d84457","9708778813fe4366a1fbbe818313d6a6","ac1d8f1aff91478d9c6c8ebefbff5c43","3e705b7a8056474e8990bad7e8b584ed","14c9670c0dab4b949c8f14fdf4b27ff5","6f80a6d8932b4822908f8c41dc51a80f","0c19129c1c7d4940ac2f2bdfc32b5d95","210dfed8792b40339a0353fec72611d4","3d3808a6750f4813bf678e2c0a122114","64fedfd8031744aa973ed7974df66727","0cf2e6f1ba7947eda9bbd80c1ba4eb85","dfaa2302c6764c89adb729d472d5d7af","4d66d20e3b5b4abea79ac3e7dff89bab","16adffb42f3e4adbaf04a67a62690067","6e93b7e90ddd426db2f532bc472a28e4","03cfa25b37cd43c78ee84dcad2a1cd76","4a75660144724405bcac959a4e01b024","b20effe13fa54227af47809d1624f5e5","c43c23ea432f4c8887e42d94f71d8415","af0c67121c504fd097c7568d91e38192","ca52bd17e0404f1ba307067c671ea3bb","b358de2bc9ff4d3ba09079793699b6f8","b8d1c3fe9d7a4989901c46c82fa38a68","4189c1302c9745d8a657ce844b2cb435"]},"id":"9bHG4CpSssUM","executionInfo":{"status":"ok","timestamp":1648830296767,"user_tz":-360,"elapsed":32601,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"1c9ba862-0f39-41fd-92d5-516c3aa1a463"},"execution_count":16,"outputs":[{"output_type":"stream","name":"stderr","text":["No model was supplied, defaulted to sshleifer/distilbart-cnn-12-6 (https://huggingface.co/sshleifer/distilbart-cnn-12-6)\n"]},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/1.76k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"80913417d4ae4471bf27eadea8f46ff2"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/1.14G [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"4d677a2db44546f5b42d52a8b7aaacae"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/26.0 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"5fae42ee765b48d48513690d8c867c12"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/878k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ac1d8f1aff91478d9c6c8ebefbff5c43"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"16adffb42f3e4adbaf04a67a62690067"}},"metadata":{}}]},{"cell_type":"code","source":["##Summmarize chunks\n","res=summarizer(chunks,max_length=80,min_length=30,do_sample=False)\n","res"],"metadata":{"id":"mgXbF3YHAyFh","executionInfo":{"status":"ok","timestamp":1648830334857,"user_tz":-360,"elapsed":38094,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"7c1baf76-0340-401b-cc51-30e82c0308e2"},"execution_count":17,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[{'summary_text': ' Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available . Auto-generated suggestions are currently only available to Google Workspace business customers . Building on grammar suggestions, Smart Compose, and autocorrect, we see this as another valuable step toward improving written communication in the workplace .'},\n"," {'summary_text': ' The Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization . Pegasus achieved state-of-the-art results on a varied set of summarization datasets . However, a number of challenges remained to apply this research advancement into a product .'},\n"," {'summary_text': ' We used knowledge distillation to distill Pegasus model into a hybrid architecture of a Transformer encoder and an RNN decoder . The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model . The authors would like to thank the many people across Google that contributed to this work .'}]"]},"metadata":{},"execution_count":17}]},{"cell_type":"code","source":["# Output to Text file\n","Text = ' '.join([summ['summary_text'] for summ in res])\n","Text"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":108},"id":"XrdXrfEVZGbO","executionInfo":{"status":"ok","timestamp":1648830334861,"user_tz":-360,"elapsed":37,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}},"outputId":"27f1cc27-7d9e-49b1-c60c-cdafb3361259"},"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["' Google Docs now automatically generates suggestions to aid document writers in creating content summaries, when they are available . Auto-generated suggestions are currently only available to Google Workspace business customers . Building on grammar suggestions, Smart Compose, and autocorrect, we see this as another valuable step toward improving written communication in the workplace .  The Pegasus work took this idea one step further, by introducing a pre-training objective customized to abstractive summarization . Pegasus achieved state-of-the-art results on a varied set of summarization datasets . However, a number of challenges remained to apply this research advancement into a product .  We used knowledge distillation to distill Pegasus model into a hybrid architecture of a Transformer encoder and an RNN decoder . The resulting model had significant improvements in latency and memory footprint while the quality was still on par with the original model . The authors would like to thank the many people across Google that contributed to this work .'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":18}]},{"cell_type":"code","source":["#Output File save\n","#with open('blogsummary.txt','w') as f:\n","       #f.write(Text)"],"metadata":{"id":"UR_BQ-2wBwiS","executionInfo":{"status":"ok","timestamp":1648827074750,"user_tz":-360,"elapsed":51,"user":{"displayName":"Jannatul Ferdous","userId":"17288488832087059331"}}},"execution_count":17,"outputs":[]}]}