how_to_contrained_beam_search.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyMbrLFngjzCGu+oIpGNSnSk",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "220de1c6744b4d36b14c2c657ebf83a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3a51b83b53d7404a8d8cf6ffb049619d",
              "IPY_MODEL_7371dc7e9ba845f4bf30be1be1ae0c25",
              "IPY_MODEL_7de9bd5bf6034d97802e0265f5842df7"
            ],
            "layout": "IPY_MODEL_8384b2d372b84a73a26b247bcb2f1f19"
          }
        },
        "3a51b83b53d7404a8d8cf6ffb049619d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27feb3edc8504160a261e1887add7d36",
            "placeholder": "​",
            "style": "IPY_MODEL_d7189f1ca26b4a0e99741aa68f69085f",
            "value": "Downloading: 100%"
          }
        },
        "7371dc7e9ba845f4bf30be1be1ae0c25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_263d599fa2ce467ab9057ed0d878c93f",
            "max": 2399,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fac1ee78b27c492d9a54c3add390c768",
            "value": 2399
          }
        },
        "7de9bd5bf6034d97802e0265f5842df7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50fe67a357e7489f8e5280fcc1172b51",
            "placeholder": "​",
            "style": "IPY_MODEL_7970bf0850bf44e886bc27ffec98a376",
            "value": " 2.40k/2.40k [00:00&lt;00:00, 96.2kB/s]"
          }
        },
        "8384b2d372b84a73a26b247bcb2f1f19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27feb3edc8504160a261e1887add7d36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7189f1ca26b4a0e99741aa68f69085f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "263d599fa2ce467ab9057ed0d878c93f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fac1ee78b27c492d9a54c3add390c768": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "50fe67a357e7489f8e5280fcc1172b51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7970bf0850bf44e886bc27ffec98a376": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20c47a8798584a3d9df18ccf8bfdd90b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_12f2cdf5469c414eb1d5f7bcdeb18d7d",
              "IPY_MODEL_9bf722cc02394fc5bd4b46d22941c355",
              "IPY_MODEL_eedc6ca86d9c44d289912a50251cd754"
            ],
            "layout": "IPY_MODEL_62ef118e9f08465bab3a651a525b951b"
          }
        },
        "12f2cdf5469c414eb1d5f7bcdeb18d7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d0c747c15da432f8d3c0f27c63a0643",
            "placeholder": "​",
            "style": "IPY_MODEL_6d2b3ed4f49a425ebbc008a0271f3382",
            "value": "Downloading: 100%"
          }
        },
        "9bf722cc02394fc5bd4b46d22941c355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9aa45d401a641ffbf74387811256a15",
            "max": 755569,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e163ce3a89b0433fb41f0632878cca37",
            "value": 755569
          }
        },
        "eedc6ca86d9c44d289912a50251cd754": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_828e4819c6c9405b9b1cbdcb58ec06c2",
            "placeholder": "​",
            "style": "IPY_MODEL_8692596f890044bcb41505a535dd9791",
            "value": " 756k/756k [00:00&lt;00:00, 15.6MB/s]"
          }
        },
        "62ef118e9f08465bab3a651a525b951b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d0c747c15da432f8d3c0f27c63a0643": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d2b3ed4f49a425ebbc008a0271f3382": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e9aa45d401a641ffbf74387811256a15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e163ce3a89b0433fb41f0632878cca37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "828e4819c6c9405b9b1cbdcb58ec06c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8692596f890044bcb41505a535dd9791": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "543b7c9b4c744496b6c4f95c229edb56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_30b35f0935a04040b8508f6fe38299e8",
              "IPY_MODEL_cabc4f6bbfcf4f23b5929cb58173b256",
              "IPY_MODEL_b9b5140e9c2246159059397e2ef842bd"
            ],
            "layout": "IPY_MODEL_3dab3c4d56e442d0857faad15db7f23a"
          }
        },
        "30b35f0935a04040b8508f6fe38299e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0cdb4dec2a644915bc71282bd3fda235",
            "placeholder": "​",
            "style": "IPY_MODEL_5f8f16b64e004dd38e46b1de72ea6a12",
            "value": "Downloading: 100%"
          }
        },
        "cabc4f6bbfcf4f23b5929cb58173b256": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb06ad14b84b40eab04001e972f1257e",
            "max": 2386119,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ac9b82682feb4214a99b53c638fe0dbf",
            "value": 2386119
          }
        },
        "b9b5140e9c2246159059397e2ef842bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_143ed3c3879f4a05ae64b0d5769a7e31",
            "placeholder": "​",
            "style": "IPY_MODEL_64940b1459fa4743934bedf722d1b385",
            "value": " 2.39M/2.39M [00:01&lt;00:00, 1.94MB/s]"
          }
        },
        "3dab3c4d56e442d0857faad15db7f23a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0cdb4dec2a644915bc71282bd3fda235": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f8f16b64e004dd38e46b1de72ea6a12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb06ad14b84b40eab04001e972f1257e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac9b82682feb4214a99b53c638fe0dbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "143ed3c3879f4a05ae64b0d5769a7e31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64940b1459fa4743934bedf722d1b385": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "faf5154ae3bd437e99db602b4c1e2aa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8d804d05052548b6bf416fb2c850b7ce",
              "IPY_MODEL_f5ba56de04d540288fde169bf87af348",
              "IPY_MODEL_ab9e186a4fdc41e696f7a53243c7a4be"
            ],
            "layout": "IPY_MODEL_d8f82b77494d47088b507f5d71995c27"
          }
        },
        "8d804d05052548b6bf416fb2c850b7ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7b2161e7ee9d433297c4f52d3d554e59",
            "placeholder": "​",
            "style": "IPY_MODEL_d1094a5d89b24905ba352a1c76462416",
            "value": "Downloading: 100%"
          }
        },
        "f5ba56de04d540288fde169bf87af348": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c044a7bbdf3e45109abcd3f80424a48d",
            "max": 2201,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_64c7fdae90f142a582f0e345d774265d",
            "value": 2201
          }
        },
        "ab9e186a4fdc41e696f7a53243c7a4be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_03a11af742c740cba529566af9e63040",
            "placeholder": "​",
            "style": "IPY_MODEL_d98a4003e7034d64bf5f8e5c9519a320",
            "value": " 2.20k/2.20k [00:00&lt;00:00, 62.7kB/s]"
          }
        },
        "d8f82b77494d47088b507f5d71995c27": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b2161e7ee9d433297c4f52d3d554e59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1094a5d89b24905ba352a1c76462416": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c044a7bbdf3e45109abcd3f80424a48d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64c7fdae90f142a582f0e345d774265d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "03a11af742c740cba529566af9e63040": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d98a4003e7034d64bf5f8e5c9519a320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4982b7fb23b746a0b15c78a137b913a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3d7e06ca1d5b48e29357d50229dcff2f",
              "IPY_MODEL_3b2da30cdd374ca6af08e1b5db7375ae",
              "IPY_MODEL_fd96c89d964b40db81006fb9c8df752f"
            ],
            "layout": "IPY_MODEL_1d7adb96078f4e789bba1bd79073a6f2"
          }
        },
        "3d7e06ca1d5b48e29357d50229dcff2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea235b691f03473d8dd12774cb78e1a0",
            "placeholder": "​",
            "style": "IPY_MODEL_7d2ea97a4ea744b1b72039001cfcefa4",
            "value": "Downloading: 100%"
          }
        },
        "3b2da30cdd374ca6af08e1b5db7375ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b1c9194825b479a8652359929049f40",
            "max": 759,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f0f64d39ed2548ec91567cf68dec9d3b",
            "value": 759
          }
        },
        "fd96c89d964b40db81006fb9c8df752f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b632c0d3a33a434e902c4e7c03abebcf",
            "placeholder": "​",
            "style": "IPY_MODEL_fb75418ee86247ad96b1911c2e2a0ca8",
            "value": " 759/759 [00:00&lt;00:00, 30.4kB/s]"
          }
        },
        "1d7adb96078f4e789bba1bd79073a6f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea235b691f03473d8dd12774cb78e1a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7d2ea97a4ea744b1b72039001cfcefa4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1b1c9194825b479a8652359929049f40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f0f64d39ed2548ec91567cf68dec9d3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b632c0d3a33a434e902c4e7c03abebcf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb75418ee86247ad96b1911c2e2a0ca8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ba2a44f8ba447e883efc35211cb5e73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1047b75bd8c54fecb4a4d1e207ecfa6c",
              "IPY_MODEL_901ca0086a3f4311a32607319f690809",
              "IPY_MODEL_c461e0f3c5b04154a944fd6bb886228f"
            ],
            "layout": "IPY_MODEL_225934afe6874840b34410a1fdcf82ac"
          }
        },
        "1047b75bd8c54fecb4a4d1e207ecfa6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0476cc4edbcc49779170dc415ca2d753",
            "placeholder": "​",
            "style": "IPY_MODEL_f911e4b72b564b9c93a222ff65c9db18",
            "value": "Downloading: 100%"
          }
        },
        "901ca0086a3f4311a32607319f690809": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc0e2578fddb4fe2a80540180bc19084",
            "max": 891700799,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cb310dad9e7c46799cb1319c8d32be92",
            "value": 891700799
          }
        },
        "c461e0f3c5b04154a944fd6bb886228f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0dbc1d3920348f0a6910e3924d13c36",
            "placeholder": "​",
            "style": "IPY_MODEL_1e91f6f5c096465aa136ea4f5adcf034",
            "value": " 892M/892M [00:15&lt;00:00, 67.5MB/s]"
          }
        },
        "225934afe6874840b34410a1fdcf82ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0476cc4edbcc49779170dc415ca2d753": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f911e4b72b564b9c93a222ff65c9db18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dc0e2578fddb4fe2a80540180bc19084": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb310dad9e7c46799cb1319c8d32be92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f0dbc1d3920348f0a6910e3924d13c36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e91f6f5c096465aa136ea4f5adcf034": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6954c3bb83794fdf9239271952f1fb45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e68da42ca32040b8ab49598d6394f836",
              "IPY_MODEL_53933f2782054b228279893f0e314600",
              "IPY_MODEL_75d0d259e8f74e7887e8784b4f6dde10"
            ],
            "layout": "IPY_MODEL_708156f113f14296b2995d35c1ad7d9a"
          }
        },
        "e68da42ca32040b8ab49598d6394f836": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dceac6f5c6ab4c5c9dc6d81352681ea4",
            "placeholder": "​",
            "style": "IPY_MODEL_14015b2e76c44152a9865704a5d3019a",
            "value": "Downloading data files: 100%"
          }
        },
        "53933f2782054b228279893f0e314600": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9ce82f1d13c949b5b4c684db916a11e5",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_77409f6414d147b99b5dfaa852fccddf",
            "value": 1
          }
        },
        "75d0d259e8f74e7887e8784b4f6dde10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d62146bfdeb4e9fb740bce29702054d",
            "placeholder": "​",
            "style": "IPY_MODEL_9e3b494c03f546718f267514008ac5a4",
            "value": " 1/1 [00:02&lt;00:00,  2.86s/it]"
          }
        },
        "708156f113f14296b2995d35c1ad7d9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dceac6f5c6ab4c5c9dc6d81352681ea4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "14015b2e76c44152a9865704a5d3019a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ce82f1d13c949b5b4c684db916a11e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77409f6414d147b99b5dfaa852fccddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5d62146bfdeb4e9fb740bce29702054d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e3b494c03f546718f267514008ac5a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cf2b00c2df254338ae7dbb3d8d387712": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5145b94d29d54e3783fed64976c7c84c",
              "IPY_MODEL_d6167c546db34a659521aebcdf909af1",
              "IPY_MODEL_7dad2c741fb34c87b4a87b96267bf1e5"
            ],
            "layout": "IPY_MODEL_8a0a6861f869461194752fa966b2244d"
          }
        },
        "5145b94d29d54e3783fed64976c7c84c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2dd9793ab5a7406db31562d09cef0926",
            "placeholder": "​",
            "style": "IPY_MODEL_5912e357db28415fab9f3910f3983780",
            "value": "Downloading data: 100%"
          }
        },
        "d6167c546db34a659521aebcdf909af1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f897216e3a7460d81d07aedf66a8a74",
            "max": 39066393,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7e0d8fc79cf74f618cecd1d6b2363522",
            "value": 39066393
          }
        },
        "7dad2c741fb34c87b4a87b96267bf1e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82adb244041a4e75b2f6ce263a9a181e",
            "placeholder": "​",
            "style": "IPY_MODEL_41f72ce1b96f4fdfb495098882d6cfbf",
            "value": " 39.1M/39.1M [00:00&lt;00:00, 64.0MB/s]"
          }
        },
        "8a0a6861f869461194752fa966b2244d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2dd9793ab5a7406db31562d09cef0926": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5912e357db28415fab9f3910f3983780": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f897216e3a7460d81d07aedf66a8a74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e0d8fc79cf74f618cecd1d6b2363522": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "82adb244041a4e75b2f6ce263a9a181e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "41f72ce1b96f4fdfb495098882d6cfbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "105e8dc7c93b42739115481fd3bc0fe8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bcdfa7ca8bd84eebb9b956071b59d7ac",
              "IPY_MODEL_cd59a9e4395f4cbca0d0007ebe516f1b",
              "IPY_MODEL_4c1596cd65854539867eb4beef890baa"
            ],
            "layout": "IPY_MODEL_c79507654dfd4cf5b25f34fdd6cad49c"
          }
        },
        "bcdfa7ca8bd84eebb9b956071b59d7ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f69610bfcabd415b899de8832326550f",
            "placeholder": "​",
            "style": "IPY_MODEL_dbcd2f911bed4edc9bce1a38fa8a718f",
            "value": "Extracting data files: 100%"
          }
        },
        "cd59a9e4395f4cbca0d0007ebe516f1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2bbe61e47fea4137a3195a9772240646",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_715b7cf259534bc6974c5eb4b62c1a70",
            "value": 1
          }
        },
        "4c1596cd65854539867eb4beef890baa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ba8dcbffc3547d8821ce8f826c7939d",
            "placeholder": "​",
            "style": "IPY_MODEL_3fb5ee73e5014ee2bbe4805c2d5e2341",
            "value": " 1/1 [00:00&lt;00:00, 32.84it/s]"
          }
        },
        "c79507654dfd4cf5b25f34fdd6cad49c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f69610bfcabd415b899de8832326550f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbcd2f911bed4edc9bce1a38fa8a718f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2bbe61e47fea4137a3195a9772240646": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "715b7cf259534bc6974c5eb4b62c1a70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0ba8dcbffc3547d8821ce8f826c7939d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fb5ee73e5014ee2bbe4805c2d5e2341": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bbad978a2c744ce3882698e3dfce8681": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_91b1fba7cf4e4c269a85748994967fb1",
              "IPY_MODEL_50608051d0d04ea588e208bcb3ca08c6",
              "IPY_MODEL_f961e96228414401870ceb7c6b59a8d8"
            ],
            "layout": "IPY_MODEL_cd165997e017492b8946c443499b4e0e"
          }
        },
        "91b1fba7cf4e4c269a85748994967fb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a81904f1a1f4cc3a37975ae02329c05",
            "placeholder": "​",
            "style": "IPY_MODEL_bb6191736d9b4943966fe2cb7e454960",
            "value": ""
          }
        },
        "50608051d0d04ea588e208bcb3ca08c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_45d93ec095fc408b8fe7cce0c331b712",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_336c10b314fc44efaa6a38dd16765f36",
            "value": 1
          }
        },
        "f961e96228414401870ceb7c6b59a8d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8eb488f765d84cf592c5b50ad9773a80",
            "placeholder": "​",
            "style": "IPY_MODEL_7c73813598084f6eb3826de6308f9d83",
            "value": " 1/? [00:01&lt;00:00,  1.59s/ tables]"
          }
        },
        "cd165997e017492b8946c443499b4e0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "0a81904f1a1f4cc3a37975ae02329c05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb6191736d9b4943966fe2cb7e454960": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "45d93ec095fc408b8fe7cce0c331b712": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "336c10b314fc44efaa6a38dd16765f36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8eb488f765d84cf592c5b50ad9773a80": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c73813598084f6eb3826de6308f9d83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2cdebc1c88d74b959d55074762251240": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9f61c121c85c48a6bc22c66efd1d5c78",
              "IPY_MODEL_87193f8d13cd4b12abbd257f4ffab476",
              "IPY_MODEL_a018c52c922b4c7a941ae676cc2891f6"
            ],
            "layout": "IPY_MODEL_060f5217c77142b29c2b128a05374d0b"
          }
        },
        "9f61c121c85c48a6bc22c66efd1d5c78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdd180fbd2d24fada66370ac2aab2f2c",
            "placeholder": "​",
            "style": "IPY_MODEL_3559bcc8b7344c3e898e8ce1528987a8",
            "value": "100%"
          }
        },
        "87193f8d13cd4b12abbd257f4ffab476": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fe16b55594d48f09a3408dd7e2dab90",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9e2474ea293c4851bc14e0739beb5f08",
            "value": 1
          }
        },
        "a018c52c922b4c7a941ae676cc2891f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af88ea7f07db4d42b3d12fff68ab6055",
            "placeholder": "​",
            "style": "IPY_MODEL_dac1af8fb4844cd797be7dc0950b67c2",
            "value": " 1/1 [00:00&lt;00:00, 32.62it/s]"
          }
        },
        "060f5217c77142b29c2b128a05374d0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdd180fbd2d24fada66370ac2aab2f2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3559bcc8b7344c3e898e8ce1528987a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4fe16b55594d48f09a3408dd7e2dab90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e2474ea293c4851bc14e0739beb5f08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "af88ea7f07db4d42b3d12fff68ab6055": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dac1af8fb4844cd797be7dc0950b67c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d45f30ec556346c5ac7913300ccd23b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9ff57c6e0f8241c4a1fbb373cd8f8873",
              "IPY_MODEL_f5f58b3269b5450fa63bcbba11222ff4",
              "IPY_MODEL_ba64dee9c91842a29408ad7dd94c38bc"
            ],
            "layout": "IPY_MODEL_1a05add1270e497db9b77adcfb474267"
          }
        },
        "9ff57c6e0f8241c4a1fbb373cd8f8873": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_717370d4804f4f798e3f2fc2a3be73f6",
            "placeholder": "​",
            "style": "IPY_MODEL_90732145035c4baaab99a31e47a9623f",
            "value": "100%"
          }
        },
        "f5f58b3269b5450fa63bcbba11222ff4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a0949215fc6b48ada31fe667ac544159",
            "max": 322,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_24bab9d57e85495883c16081db42cae5",
            "value": 322
          }
        },
        "ba64dee9c91842a29408ad7dd94c38bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f32815e5e970431f9adfac6198100fd4",
            "placeholder": "​",
            "style": "IPY_MODEL_02d89183bb2f48f79110f1579154ff37",
            "value": " 322/322 [00:02&lt;00:00, 158.93ba/s]"
          }
        },
        "1a05add1270e497db9b77adcfb474267": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "717370d4804f4f798e3f2fc2a3be73f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "90732145035c4baaab99a31e47a9623f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a0949215fc6b48ada31fe667ac544159": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "24bab9d57e85495883c16081db42cae5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f32815e5e970431f9adfac6198100fd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02d89183bb2f48f79110f1579154ff37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/VanessaSchenkel/how_to/blob/main/how_to_contrained_beam_search.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tprln3Yjsh2Y",
        "outputId": "43a33e9b-8fec-46bb-dd01-348e6acff0fd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.23.1-py3-none-any.whl (5.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.3 MB 21.5 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 79.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Collecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",
            "\u001b[K     |████████████████████████████████| 163 kB 58.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (5.0.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.1 transformers-4.23.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.7/dist-packages (3.4.1)\n",
            "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy) (4.1.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.8)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (8.1.2)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.8)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (4.64.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.10.1)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.9.2)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.4.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.3.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.21.6)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.7)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (21.3)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.10)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.4.4)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.6.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.23.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy) (57.4.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.11.3)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy) (3.8.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy) (5.2.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.9.24)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.7.8)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.0.2)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy) (2.0.1)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pt-core-news-lg==3.4.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/pt_core_news_lg-3.4.0/pt_core_news_lg-3.4.0-py3-none-any.whl (568.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 568.2 MB 14 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from pt-core-news-lg==3.4.0) (3.4.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (4.64.1)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (3.0.7)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (3.3.0)\n",
            "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (4.1.1)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (1.0.3)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (1.9.2)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (1.21.6)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (3.0.10)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (1.0.8)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (8.1.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2.11.3)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (0.10.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (57.4.0)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2.0.8)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (0.6.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (21.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2.4.4)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (0.4.2)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (3.8.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (5.2.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2022.9.24)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (0.0.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (0.7.8)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.5.0,>=3.4.0->pt-core-news-lg==3.4.0) (2.0.1)\n",
            "Installing collected packages: pt-core-news-lg\n",
            "Successfully installed pt-core-news-lg-3.4.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('pt_core_news_lg')\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install spacy\n",
        "!python -m spacy download pt_core_news_lg"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model"
      ],
      "metadata": {
        "id": "-zeROUjXyUa5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "\n",
        "nlp = spacy.load(\"pt_core_news_lg\")\n",
        "\n",
        "doc = nlp(\"O médico acabou seu trabalho.\")"
      ],
      "metadata": {
        "id": "zwPxEsKgtrrt"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
        "\n",
        "model_name = 'VanessaSchenkel/pt-unicamp-handcrafted'\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n",
        "\n",
        "sentence = \"The doctor finished her work.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "outputs = model.generate(input_ids, num_beams=5)\n",
        "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281,
          "referenced_widgets": [
            "220de1c6744b4d36b14c2c657ebf83a3",
            "3a51b83b53d7404a8d8cf6ffb049619d",
            "7371dc7e9ba845f4bf30be1be1ae0c25",
            "7de9bd5bf6034d97802e0265f5842df7",
            "8384b2d372b84a73a26b247bcb2f1f19",
            "27feb3edc8504160a261e1887add7d36",
            "d7189f1ca26b4a0e99741aa68f69085f",
            "263d599fa2ce467ab9057ed0d878c93f",
            "fac1ee78b27c492d9a54c3add390c768",
            "50fe67a357e7489f8e5280fcc1172b51",
            "7970bf0850bf44e886bc27ffec98a376",
            "20c47a8798584a3d9df18ccf8bfdd90b",
            "12f2cdf5469c414eb1d5f7bcdeb18d7d",
            "9bf722cc02394fc5bd4b46d22941c355",
            "eedc6ca86d9c44d289912a50251cd754",
            "62ef118e9f08465bab3a651a525b951b",
            "4d0c747c15da432f8d3c0f27c63a0643",
            "6d2b3ed4f49a425ebbc008a0271f3382",
            "e9aa45d401a641ffbf74387811256a15",
            "e163ce3a89b0433fb41f0632878cca37",
            "828e4819c6c9405b9b1cbdcb58ec06c2",
            "8692596f890044bcb41505a535dd9791",
            "543b7c9b4c744496b6c4f95c229edb56",
            "30b35f0935a04040b8508f6fe38299e8",
            "cabc4f6bbfcf4f23b5929cb58173b256",
            "b9b5140e9c2246159059397e2ef842bd",
            "3dab3c4d56e442d0857faad15db7f23a",
            "0cdb4dec2a644915bc71282bd3fda235",
            "5f8f16b64e004dd38e46b1de72ea6a12",
            "bb06ad14b84b40eab04001e972f1257e",
            "ac9b82682feb4214a99b53c638fe0dbf",
            "143ed3c3879f4a05ae64b0d5769a7e31",
            "64940b1459fa4743934bedf722d1b385",
            "faf5154ae3bd437e99db602b4c1e2aa0",
            "8d804d05052548b6bf416fb2c850b7ce",
            "f5ba56de04d540288fde169bf87af348",
            "ab9e186a4fdc41e696f7a53243c7a4be",
            "d8f82b77494d47088b507f5d71995c27",
            "7b2161e7ee9d433297c4f52d3d554e59",
            "d1094a5d89b24905ba352a1c76462416",
            "c044a7bbdf3e45109abcd3f80424a48d",
            "64c7fdae90f142a582f0e345d774265d",
            "03a11af742c740cba529566af9e63040",
            "d98a4003e7034d64bf5f8e5c9519a320",
            "4982b7fb23b746a0b15c78a137b913a2",
            "3d7e06ca1d5b48e29357d50229dcff2f",
            "3b2da30cdd374ca6af08e1b5db7375ae",
            "fd96c89d964b40db81006fb9c8df752f",
            "1d7adb96078f4e789bba1bd79073a6f2",
            "ea235b691f03473d8dd12774cb78e1a0",
            "7d2ea97a4ea744b1b72039001cfcefa4",
            "1b1c9194825b479a8652359929049f40",
            "f0f64d39ed2548ec91567cf68dec9d3b",
            "b632c0d3a33a434e902c4e7c03abebcf",
            "fb75418ee86247ad96b1911c2e2a0ca8",
            "2ba2a44f8ba447e883efc35211cb5e73",
            "1047b75bd8c54fecb4a4d1e207ecfa6c",
            "901ca0086a3f4311a32607319f690809",
            "c461e0f3c5b04154a944fd6bb886228f",
            "225934afe6874840b34410a1fdcf82ac",
            "0476cc4edbcc49779170dc415ca2d753",
            "f911e4b72b564b9c93a222ff65c9db18",
            "dc0e2578fddb4fe2a80540180bc19084",
            "cb310dad9e7c46799cb1319c8d32be92",
            "f0dbc1d3920348f0a6910e3924d13c36",
            "1e91f6f5c096465aa136ea4f5adcf034"
          ]
        },
        "id": "wi0UKHuft2-S",
        "outputId": "4540c0cc-5834-4bba-b2e3-5ec1d200421e"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.40k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "220de1c6744b4d36b14c2c657ebf83a3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/756k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "20c47a8798584a3d9df18ccf8bfdd90b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.39M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "543b7c9b4c744496b6c4f95c229edb56"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "faf5154ae3bd437e99db602b4c1e2aa0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/759 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4982b7fb23b746a0b15c78a137b913a2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/892M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2ba2a44f8ba447e883efc35211cb5e73"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:1301: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 20 (`self.config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  UserWarning,\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['A médica terminou seu trabalho.']"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translation_google = nlp(\"O médico terminou seu trabalho.\")\n",
        "translation_model = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n",
        "\n",
        "print(translation_google)\n",
        "print(translation_model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IAR8ZOfquQEB",
        "outputId": "7a1b624e-09f8-45be-bdef-6cd06c7c8e60"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "O médico terminou seu trabalho.\n",
            "['A médica terminou seu trabalho.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import PhrasalConstraint\n",
        "\n",
        "sentence = \"The doctor finished her work.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "constraints = [\n",
        "    PhrasalConstraint(\n",
        "        tokenizer(\"acabou seu trabalho\", add_special_tokens=False).input_ids\n",
        "    )\n",
        "]\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    constraints=constraints,\n",
        "    num_beams=10,\n",
        "    num_return_sequences=3,\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NYL2mSUnu3Au",
        "outputId": "87ddbf96-f6ab-43e4-c736-d22b105e8f15"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "0: A médica acabou seu trabalho.\n",
            "1: O médico acabou seu trabalho.\n",
            "2: O doutor acabou seu trabalho.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import DisjunctiveConstraint\n",
        "\n",
        "sentence = \"The doctor finished her work.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "flexible_phrases = tokenizer([\"médica\", \"médico\"], add_special_tokens=False).input_ids\n",
        "\n",
        "constraints = [DisjunctiveConstraint(flexible_phrases)]\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    constraints=constraints,\n",
        "    num_beams=10,\n",
        "    num_return_sequences=3,\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JSuC2a5U1X8i",
        "outputId": "00603cbf-30e9-4292-c538-53fe2d2ea6c5"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "0: A médica terminou seu trabalho.\n",
            "1: A médica terminou o seu trabalho.\n",
            "2: O médico terminou seu trabalho.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(translation_google)\n",
        "print(translation_model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sQXw3y_0Z3bS",
        "outputId": "2e7c4558-f11d-4385-9902-432377327358"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "O médico terminou seu trabalho.\n",
            "['A médica terminou seu trabalho.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translation_model = nlp(translation_model[0])\n",
        "translation_model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2npNhLLXiBE6",
        "outputId": "a5f425a7-7335-434d-f735-eb2cace98a26"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "A médica terminou seu trabalho."
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in translation_model:\n",
        "  print(token.text)\n",
        "  print(\"ancestors -> \", [ancestor for ancestor in token.ancestors])\n",
        "  print(\"children ->\", [child for child in token.children])\n",
        "  print(\"dep_ ->\",token.dep_)\n",
        "  print(\"head ->\", token.head)\n",
        "  print(\"is_ancestor ->\", token.is_ancestor(token))\n",
        "  print(\"lemma_ ->\", token.lemma_)\n",
        "  print(\"morph ->\", token.morph)\n",
        "  print(\"pos_ ->\", token.pos_)\n",
        "  print(\"norm_ ->\", token.norm_)\n",
        "  print(\"pos_ ->\", token.pos_)\n",
        "  print(\"tag_ ->\", token.tag_)\n",
        "  print(\"suffix_ ->\", token.suffix_ )\n",
        "  print(\"------\")\n",
        " "
      ],
      "metadata": {
        "id": "lWclUI8YiZjW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8dd7575c-b56e-4138-d7e2-e4bfd758eb41"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A\n",
            "ancestors ->  [médica, terminou]\n",
            "children -> []\n",
            "dep_ -> det\n",
            "head -> médica\n",
            "is_ancestor -> False\n",
            "lemma_ -> o\n",
            "morph -> Definite=Def|Gender=Fem|Number=Sing|PronType=Art\n",
            "pos_ -> DET\n",
            "norm_ -> a\n",
            "pos_ -> DET\n",
            "tag_ -> DET\n",
            "suffix_ -> A\n",
            "------\n",
            "médica\n",
            "ancestors ->  [terminou]\n",
            "children -> [A]\n",
            "dep_ -> nsubj\n",
            "head -> terminou\n",
            "is_ancestor -> False\n",
            "lemma_ -> médica\n",
            "morph -> Gender=Fem|Number=Sing\n",
            "pos_ -> NOUN\n",
            "norm_ -> médica\n",
            "pos_ -> NOUN\n",
            "tag_ -> NOUN\n",
            "suffix_ -> ica\n",
            "------\n",
            "terminou\n",
            "ancestors ->  []\n",
            "children -> [médica, trabalho, .]\n",
            "dep_ -> ROOT\n",
            "head -> terminou\n",
            "is_ancestor -> False\n",
            "lemma_ -> terminar\n",
            "morph -> Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\n",
            "pos_ -> VERB\n",
            "norm_ -> terminou\n",
            "pos_ -> VERB\n",
            "tag_ -> VERB\n",
            "suffix_ -> nou\n",
            "------\n",
            "seu\n",
            "ancestors ->  [trabalho, terminou]\n",
            "children -> []\n",
            "dep_ -> det\n",
            "head -> trabalho\n",
            "is_ancestor -> False\n",
            "lemma_ -> seu\n",
            "morph -> Gender=Masc|Number=Sing|PronType=Prs\n",
            "pos_ -> DET\n",
            "norm_ -> seu\n",
            "pos_ -> DET\n",
            "tag_ -> DET\n",
            "suffix_ -> seu\n",
            "------\n",
            "trabalho\n",
            "ancestors ->  [terminou]\n",
            "children -> [seu]\n",
            "dep_ -> obj\n",
            "head -> terminou\n",
            "is_ancestor -> False\n",
            "lemma_ -> trabalho\n",
            "morph -> Gender=Masc|Number=Sing\n",
            "pos_ -> NOUN\n",
            "norm_ -> trabalho\n",
            "pos_ -> NOUN\n",
            "tag_ -> NOUN\n",
            "suffix_ -> lho\n",
            "------\n",
            ".\n",
            "ancestors ->  [terminou]\n",
            "children -> []\n",
            "dep_ -> punct\n",
            "head -> terminou\n",
            "is_ancestor -> False\n",
            "lemma_ -> .\n",
            "morph -> \n",
            "pos_ -> PUNCT\n",
            "norm_ -> .\n",
            "pos_ -> PUNCT\n",
            "tag_ -> PUNCT\n",
            "suffix_ -> .\n",
            "------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "teste = nlp(\"A médica terminou o trabalho, porém o enfermeiro ainda não. Outra frase aqui.\")\n",
        "\n",
        "for n in teste.noun_chunks:\n",
        "  print(n)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8UrWi37vwZAr",
        "outputId": "63ad1887-55b8-4ae8-c84b-0e8eac4641ff"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A médica\n",
            "o trabalho\n",
            ", porém o enfermeiro\n",
            "Outra frase\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for ex in translation_google.noun_chunks:\n",
        "  print(ex)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AxPp9oTWA-NK",
        "outputId": "f1ad6b61-13cc-4213-f3aa-0f584290a310"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "O médico\n",
            "seu trabalho\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "teste = nlp(\"A médica era linda, o enfermeiro nem tanto\")\n",
        "for ex in teste.noun_chunks:\n",
        "  print(ex)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YHntYj1oBdLF",
        "outputId": "5baf76c4-dfb0-4969-c9e0-fdb37d7a3f5f"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A médica\n",
            "o enfermeiro\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in teste:\n",
        "  print(\"TOKEN:  \", token.text)\n",
        "  print(\"ancestors -> \", [ancestor for ancestor in token.ancestors])\n",
        "  print(\"children ->\", [child for child in token.children])\n",
        "  print(\"dep_ ->\",token.dep_)\n",
        "  print(\"head ->\", token.head)\n",
        "  print(\"is_ancestor ->\", token.is_ancestor(token))\n",
        "  print(\"lemma_ ->\", token.lemma_)\n",
        "  print(\"morph ->\", token.morph)\n",
        "  print(\"pos_ ->\", token.pos_)\n",
        "  print(\"norm_ ->\", token.norm_)\n",
        "  print(\"tag_ ->\", token.tag_)\n",
        "  print(\"suffix_ ->\", token.suffix_ )\n",
        "  print(\"--------------------\")"
      ],
      "metadata": {
        "id": "C_ET2A5CHbLr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0bd1267e-6e29-4d55-e236-5ccf2626feea"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TOKEN:   A\n",
            "ancestors ->  [médica, linda]\n",
            "children -> []\n",
            "dep_ -> det\n",
            "head -> médica\n",
            "is_ancestor -> False\n",
            "lemma_ -> o\n",
            "morph -> Definite=Def|Gender=Fem|Number=Sing|PronType=Art\n",
            "pos_ -> DET\n",
            "norm_ -> a\n",
            "tag_ -> DET\n",
            "suffix_ -> A\n",
            "--------------------\n",
            "TOKEN:   médica\n",
            "ancestors ->  [linda]\n",
            "children -> [A]\n",
            "dep_ -> nsubj\n",
            "head -> linda\n",
            "is_ancestor -> False\n",
            "lemma_ -> médica\n",
            "morph -> Gender=Fem|Number=Sing\n",
            "pos_ -> NOUN\n",
            "norm_ -> médica\n",
            "tag_ -> NOUN\n",
            "suffix_ -> ica\n",
            "--------------------\n",
            "TOKEN:   era\n",
            "ancestors ->  [linda]\n",
            "children -> []\n",
            "dep_ -> cop\n",
            "head -> linda\n",
            "is_ancestor -> False\n",
            "lemma_ -> ser\n",
            "morph -> Mood=Ind|Number=Sing|Person=3|Tense=Imp|VerbForm=Fin\n",
            "pos_ -> AUX\n",
            "norm_ -> era\n",
            "tag_ -> AUX\n",
            "suffix_ -> era\n",
            "--------------------\n",
            "TOKEN:   linda\n",
            "ancestors ->  []\n",
            "children -> [médica, era, ,, enfermeiro, tanto]\n",
            "dep_ -> ROOT\n",
            "head -> linda\n",
            "is_ancestor -> False\n",
            "lemma_ -> lindo\n",
            "morph -> Gender=Fem|Number=Sing\n",
            "pos_ -> ADJ\n",
            "norm_ -> linda\n",
            "tag_ -> ADJ\n",
            "suffix_ -> nda\n",
            "--------------------\n",
            "TOKEN:   ,\n",
            "ancestors ->  [linda]\n",
            "children -> []\n",
            "dep_ -> punct\n",
            "head -> linda\n",
            "is_ancestor -> False\n",
            "lemma_ -> ,\n",
            "morph -> \n",
            "pos_ -> PUNCT\n",
            "norm_ -> ,\n",
            "tag_ -> PUNCT\n",
            "suffix_ -> ,\n",
            "--------------------\n",
            "TOKEN:   o\n",
            "ancestors ->  [enfermeiro, linda]\n",
            "children -> []\n",
            "dep_ -> det\n",
            "head -> enfermeiro\n",
            "is_ancestor -> False\n",
            "lemma_ -> o\n",
            "morph -> Definite=Def|Gender=Masc|Number=Sing|PronType=Art\n",
            "pos_ -> DET\n",
            "norm_ -> o\n",
            "tag_ -> DET\n",
            "suffix_ -> o\n",
            "--------------------\n",
            "TOKEN:   enfermeiro\n",
            "ancestors ->  [linda]\n",
            "children -> [o]\n",
            "dep_ -> nsubj\n",
            "head -> linda\n",
            "is_ancestor -> False\n",
            "lemma_ -> enfermeiro\n",
            "morph -> Gender=Masc|Number=Sing\n",
            "pos_ -> NOUN\n",
            "norm_ -> enfermeiro\n",
            "tag_ -> NOUN\n",
            "suffix_ -> iro\n",
            "--------------------\n",
            "TOKEN:   nem\n",
            "ancestors ->  [tanto, linda]\n",
            "children -> []\n",
            "dep_ -> case\n",
            "head -> tanto\n",
            "is_ancestor -> False\n",
            "lemma_ -> nem\n",
            "morph -> \n",
            "pos_ -> ADP\n",
            "norm_ -> nem\n",
            "tag_ -> ADP\n",
            "suffix_ -> nem\n",
            "--------------------\n",
            "TOKEN:   tanto\n",
            "ancestors ->  [linda]\n",
            "children -> [nem]\n",
            "dep_ -> conj\n",
            "head -> linda\n",
            "is_ancestor -> False\n",
            "lemma_ -> tanto\n",
            "morph -> \n",
            "pos_ -> ADV\n",
            "norm_ -> tanto\n",
            "tag_ -> ADV\n",
            "suffix_ -> nto\n",
            "--------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "teste = nlp(\"A médica comeu a comida dela, o enfermeiro comeu o sanduíche dele\")\n",
        "\n",
        "table = {}\n",
        "text_list = []\n",
        "anc = []\n",
        "child = []\n",
        "dep = []\n",
        "head = []\n",
        "lemma = []\n",
        "morph = []\n",
        "pos = []\n",
        "tag = []\n",
        "norm = []\n",
        "suffix = []\n",
        "pref = []\n",
        "\n",
        "for token in teste:\n",
        "  text_list.append(token.text)\n",
        "  anc.append([ancestor for ancestor in token.ancestors])\n",
        "  child.append([child for child in token.children])\n",
        "  dep.append(token.dep_)\n",
        "  head.append(token.head)\n",
        "  lemma.append(token.lemma_)\n",
        "  morph.append(token.morph)\n",
        "  pos.append(token.pos_)\n",
        "  norm.append(token.norm_)\n",
        "  pref.append(token.prefix_)\n",
        "  suffix.append(token.suffix_)\n",
        "\n",
        "table['text'] = text_list\n",
        "table['anc'] = anc\n",
        "table['child'] = child\n",
        "table['dep'] = dep\n",
        "table['head'] = head\n",
        "table['lemma'] = lemma\n",
        "table['morph'] = morph\n",
        "table['pos'] = pos\n",
        "table['norm'] = norm\n",
        "table['pref'] = pref\n",
        "table['suffix'] = suffix"
      ],
      "metadata": {
        "id": "2XW81OeCC5BZ"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.DataFrame(table)\n",
        "\n",
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 457
        },
        "id": "ess1PvBYJK_W",
        "outputId": "6fa305a8-cf2a-4870-c9e2-1ce0f541ad3c"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "          text                         anc                       child    dep  \\\n",
              "0            A             [médica, comeu]                          []    det   \n",
              "1       médica                     [comeu]                         [A]  nsubj   \n",
              "2        comeu                          []  [médica, comida, ,, comeu]   ROOT   \n",
              "3            a             [comida, comeu]                          []    det   \n",
              "4       comida                     [comeu]                   [a, dela]    obj   \n",
              "5         dela             [comida, comeu]                          []   nmod   \n",
              "6            ,                     [comeu]                          []  punct   \n",
              "7            o  [enfermeiro, comeu, comeu]                          []    det   \n",
              "8   enfermeiro              [comeu, comeu]                         [o]  nsubj   \n",
              "9        comeu                     [comeu]     [enfermeiro, sanduíche]   conj   \n",
              "10           o   [sanduíche, comeu, comeu]                          []    det   \n",
              "11   sanduíche              [comeu, comeu]                   [o, dele]    obj   \n",
              "12        dele   [sanduíche, comeu, comeu]                          []   nmod   \n",
              "\n",
              "          head       lemma                                              morph  \\\n",
              "0       médica           o  (Definite=Def, Gender=Fem, Number=Sing, PronTy...   \n",
              "1        comeu      médica                          (Gender=Fem, Number=Sing)   \n",
              "2        comeu       comeu  (Mood=Sub, Number=Sing, Person=3, Tense=Past, ...   \n",
              "3       comida           o  (Definite=Def, Gender=Fem, Number=Sing, PronTy...   \n",
              "4        comeu      comida                          (Gender=Fem, Number=Sing)   \n",
              "5       comida      de ele  (Gender=Fem, Number=Sing, Person=3, PronType=Prs)   \n",
              "6        comeu           ,                                                 ()   \n",
              "7   enfermeiro           o  (Definite=Def, Gender=Masc, Number=Sing, PronT...   \n",
              "8        comeu  enfermeiro                         (Gender=Masc, Number=Sing)   \n",
              "9        comeu       comeu  (Mood=Sub, Number=Sing, Person=3, Tense=Past, ...   \n",
              "10   sanduíche           o  (Definite=Def, Gender=Masc, Number=Sing, PronT...   \n",
              "11       comeu   sanduíche                         (Gender=Masc, Number=Sing)   \n",
              "12   sanduíche      de ele  (Gender=Masc, Number=Sing, Person=3, PronType=...   \n",
              "\n",
              "      pos        norm pref suffix  \n",
              "0     DET           a    A      A  \n",
              "1    NOUN      médica    m    ica  \n",
              "2    VERB       comeu    c    meu  \n",
              "3     DET           a    a      a  \n",
              "4    NOUN      comida    c    ida  \n",
              "5    PRON        dela    d    ela  \n",
              "6   PUNCT           ,    ,      ,  \n",
              "7     DET           o    o      o  \n",
              "8    NOUN  enfermeiro    e    iro  \n",
              "9    VERB       comeu    c    meu  \n",
              "10    DET           o    o      o  \n",
              "11   NOUN   sanduíche    s    che  \n",
              "12   PRON        dele    d    ele  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-db32e629-8384-4959-ae02-fbcbc7663189\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>anc</th>\n",
              "      <th>child</th>\n",
              "      <th>dep</th>\n",
              "      <th>head</th>\n",
              "      <th>lemma</th>\n",
              "      <th>morph</th>\n",
              "      <th>pos</th>\n",
              "      <th>norm</th>\n",
              "      <th>pref</th>\n",
              "      <th>suffix</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A</td>\n",
              "      <td>[médica, comeu]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>médica</td>\n",
              "      <td>o</td>\n",
              "      <td>(Definite=Def, Gender=Fem, Number=Sing, PronTy...</td>\n",
              "      <td>DET</td>\n",
              "      <td>a</td>\n",
              "      <td>A</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>médica</td>\n",
              "      <td>[comeu]</td>\n",
              "      <td>[A]</td>\n",
              "      <td>nsubj</td>\n",
              "      <td>comeu</td>\n",
              "      <td>médica</td>\n",
              "      <td>(Gender=Fem, Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>médica</td>\n",
              "      <td>m</td>\n",
              "      <td>ica</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>comeu</td>\n",
              "      <td>[]</td>\n",
              "      <td>[médica, comida, ,, comeu]</td>\n",
              "      <td>ROOT</td>\n",
              "      <td>comeu</td>\n",
              "      <td>comeu</td>\n",
              "      <td>(Mood=Sub, Number=Sing, Person=3, Tense=Past, ...</td>\n",
              "      <td>VERB</td>\n",
              "      <td>comeu</td>\n",
              "      <td>c</td>\n",
              "      <td>meu</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>a</td>\n",
              "      <td>[comida, comeu]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>comida</td>\n",
              "      <td>o</td>\n",
              "      <td>(Definite=Def, Gender=Fem, Number=Sing, PronTy...</td>\n",
              "      <td>DET</td>\n",
              "      <td>a</td>\n",
              "      <td>a</td>\n",
              "      <td>a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>comida</td>\n",
              "      <td>[comeu]</td>\n",
              "      <td>[a, dela]</td>\n",
              "      <td>obj</td>\n",
              "      <td>comeu</td>\n",
              "      <td>comida</td>\n",
              "      <td>(Gender=Fem, Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>comida</td>\n",
              "      <td>c</td>\n",
              "      <td>ida</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>dela</td>\n",
              "      <td>[comida, comeu]</td>\n",
              "      <td>[]</td>\n",
              "      <td>nmod</td>\n",
              "      <td>comida</td>\n",
              "      <td>de ele</td>\n",
              "      <td>(Gender=Fem, Number=Sing, Person=3, PronType=Prs)</td>\n",
              "      <td>PRON</td>\n",
              "      <td>dela</td>\n",
              "      <td>d</td>\n",
              "      <td>ela</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>,</td>\n",
              "      <td>[comeu]</td>\n",
              "      <td>[]</td>\n",
              "      <td>punct</td>\n",
              "      <td>comeu</td>\n",
              "      <td>,</td>\n",
              "      <td>()</td>\n",
              "      <td>PUNCT</td>\n",
              "      <td>,</td>\n",
              "      <td>,</td>\n",
              "      <td>,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>o</td>\n",
              "      <td>[enfermeiro, comeu, comeu]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>enfermeiro</td>\n",
              "      <td>o</td>\n",
              "      <td>(Definite=Def, Gender=Masc, Number=Sing, PronT...</td>\n",
              "      <td>DET</td>\n",
              "      <td>o</td>\n",
              "      <td>o</td>\n",
              "      <td>o</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>enfermeiro</td>\n",
              "      <td>[comeu, comeu]</td>\n",
              "      <td>[o]</td>\n",
              "      <td>nsubj</td>\n",
              "      <td>comeu</td>\n",
              "      <td>enfermeiro</td>\n",
              "      <td>(Gender=Masc, Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>enfermeiro</td>\n",
              "      <td>e</td>\n",
              "      <td>iro</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>comeu</td>\n",
              "      <td>[comeu]</td>\n",
              "      <td>[enfermeiro, sanduíche]</td>\n",
              "      <td>conj</td>\n",
              "      <td>comeu</td>\n",
              "      <td>comeu</td>\n",
              "      <td>(Mood=Sub, Number=Sing, Person=3, Tense=Past, ...</td>\n",
              "      <td>VERB</td>\n",
              "      <td>comeu</td>\n",
              "      <td>c</td>\n",
              "      <td>meu</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>o</td>\n",
              "      <td>[sanduíche, comeu, comeu]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>sanduíche</td>\n",
              "      <td>o</td>\n",
              "      <td>(Definite=Def, Gender=Masc, Number=Sing, PronT...</td>\n",
              "      <td>DET</td>\n",
              "      <td>o</td>\n",
              "      <td>o</td>\n",
              "      <td>o</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>sanduíche</td>\n",
              "      <td>[comeu, comeu]</td>\n",
              "      <td>[o, dele]</td>\n",
              "      <td>obj</td>\n",
              "      <td>comeu</td>\n",
              "      <td>sanduíche</td>\n",
              "      <td>(Gender=Masc, Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>sanduíche</td>\n",
              "      <td>s</td>\n",
              "      <td>che</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>dele</td>\n",
              "      <td>[sanduíche, comeu, comeu]</td>\n",
              "      <td>[]</td>\n",
              "      <td>nmod</td>\n",
              "      <td>sanduíche</td>\n",
              "      <td>de ele</td>\n",
              "      <td>(Gender=Masc, Number=Sing, Person=3, PronType=...</td>\n",
              "      <td>PRON</td>\n",
              "      <td>dele</td>\n",
              "      <td>d</td>\n",
              "      <td>ele</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-db32e629-8384-4959-ae02-fbcbc7663189')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-db32e629-8384-4959-ae02-fbcbc7663189 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-db32e629-8384-4959-ae02-fbcbc7663189');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for n in teste.noun_chunks:\n",
        "  print(n)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-E9oxaGy2O9m",
        "outputId": "aae74b4b-076c-4bc9-f456-47578199100b"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A médica\n",
            "a comida\n",
            "dela\n",
            "o enfermeiro\n",
            "o sanduíche\n",
            "dele\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import PhrasalConstraint\n",
        "\n",
        "sentence = \"The doctor finished her work, the nurse has his job.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "constraints = [\n",
        "    PhrasalConstraint(\n",
        "        tokenizer(\"acabou seu trabalho\", add_special_tokens=False).input_ids\n",
        "    )\n",
        "]\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    constraints=constraints,\n",
        "    num_beams=10,\n",
        "    num_return_sequences=3,\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_vTU8wMpDqXI",
        "outputId": "4e3976f6-50a9-4c9d-9894-b5d095898641"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:1301: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 20 (`self.config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  UserWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "0: O médico acabou seu trabalho, a enfermeira tem o seu trabalho.\n",
            "1: O médico acabou seu trabalho, o enfermeiro tem o seu trabalho.\n",
            "2: A médica acabou seu trabalho, a enfermeira tem o seu trabalho.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import DisjunctiveConstraint\n",
        "\n",
        "sentence = \"The doctor finished her work, the nurse has his job.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "flexible_phrases = tokenizer([\"médica\", \"médico\", \"enfermeira\", \"enfermeiro\"], add_special_tokens=False).input_ids\n",
        "\n",
        "constraints = [DisjunctiveConstraint(flexible_phrases)]\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    constraints=constraints,\n",
        "    num_beams=10,\n",
        "    num_return_sequences=6,\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PIyRaiifUKFy",
        "outputId": "5d725118-cb56-4477-a797-d6a2e082800a"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "0: A médica terminou seu trabalho, a enfermeira tem o seu trabalho.\n",
            "1: O médico terminou seu trabalho, a enfermeira tem o seu trabalho.\n",
            "2: A médica terminou seu trabalho, a enfermeira tem o seu emprego.\n",
            "3: O médico terminou seu trabalho, o enfermeiro tem o seu trabalho.\n",
            "4: O médico terminou seu trabalho, a enfermeira tem o seu emprego.\n",
            "5: O médico terminou seu trabalho, o enfermeiro tem o seu emprego.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### roBERTa"
      ],
      "metadata": {
        "id": "cnL2ZMes4Ohj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torch torchvision\n",
        "!pip install fairseq\n",
        "!pip install sacremoses\n",
        "!python -m spacy download en_core_web_lg"
      ],
      "metadata": {
        "id": "HFnxJYJB4SSW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "18938b1d-d40a-44fd-fd90-b7854ce68f0e"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.12.1+cu113)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.13.1+cu113)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (4.1.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.21.6)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchvision) (2.23.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2022.9.24)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2.10)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting fairseq\n",
            "  Downloading fairseq-0.12.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (11.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 11.0 MB 28.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq) (2022.6.2)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq) (0.29.32)\n",
            "Collecting sacrebleu>=1.4.12\n",
            "  Downloading sacrebleu-2.2.1-py3-none-any.whl (116 kB)\n",
            "\u001b[K     |████████████████████████████████| 116 kB 79.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairseq) (1.21.6)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq) (4.64.1)\n",
            "Collecting omegaconf<2.1\n",
            "  Downloading omegaconf-2.0.6-py3-none-any.whl (36 kB)\n",
            "Collecting bitarray\n",
            "  Downloading bitarray-2.6.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (235 kB)\n",
            "\u001b[K     |████████████████████████████████| 235 kB 77.3 MB/s \n",
            "\u001b[?25hCollecting hydra-core<1.1,>=1.0.7\n",
            "  Downloading hydra_core-1.0.7-py3-none-any.whl (123 kB)\n",
            "\u001b[K     |████████████████████████████████| 123 kB 85.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq) (1.15.1)\n",
            "Requirement already satisfied: torchaudio>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from fairseq) (0.12.1+cu113)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq) (1.12.1+cu113)\n",
            "Collecting antlr4-python3-runtime==4.8\n",
            "  Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)\n",
            "\u001b[K     |████████████████████████████████| 112 kB 72.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1,>=1.0.7->fairseq) (5.9.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq) (4.1.1)\n",
            "Requirement already satisfied: PyYAML>=5.1.* in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq) (6.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq) (4.9.1)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.5-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq) (0.8.10)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.5.1-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq) (2.21)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources->hydra-core<1.1,>=1.0.7->fairseq) (3.8.1)\n",
            "Building wheels for collected packages: antlr4-python3-runtime\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=141230 sha256=7db5d9ff89ebab2a9b0e9af39b5ca2ed1ff27a609e2fb32412c22834f526b0c0\n",
            "  Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c\n",
            "Successfully built antlr4-python3-runtime\n",
            "Installing collected packages: portalocker, omegaconf, colorama, antlr4-python3-runtime, sacrebleu, hydra-core, bitarray, fairseq\n",
            "Successfully installed antlr4-python3-runtime-4.8 bitarray-2.6.0 colorama-0.4.5 fairseq-0.12.2 hydra-core-1.0.7 omegaconf-2.0.6 portalocker-2.5.1 sacrebleu-2.2.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.53.tar.gz (880 kB)\n",
            "\u001b[K     |████████████████████████████████| 880 kB 35.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2022.6.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.2.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sacremoses) (4.64.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895260 sha256=386ce912ccf970a779fa9da14e0cff62e8ece7fb51193ee4b4e484215cfbe8fe\n",
            "  Stored in directory: /root/.cache/pip/wheels/87/39/dd/a83eeef36d0bf98e7a4d1933a4ad2d660295a40613079bafc9\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses\n",
            "Successfully installed sacremoses-0.0.53\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting en-core-web-lg==3.4.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.4.0/en_core_web_lg-3.4.0-py3-none-any.whl (587.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 587.7 MB 9.5 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-lg==3.4.0) (3.4.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (3.0.10)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (4.64.1)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (1.9.2)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (8.1.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2.23.0)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (0.6.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (21.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2.4.4)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2.0.6)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2.11.3)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (1.0.8)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (57.4.0)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (0.10.1)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (3.3.0)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (1.0.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (3.0.7)\n",
            "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (4.1.1)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (1.21.6)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (0.4.2)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2.0.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (3.8.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (5.2.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2022.9.24)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (1.24.3)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (0.0.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (0.7.8)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.0) (2.0.1)\n",
            "Installing collected packages: en-core-web-lg\n",
            "Successfully installed en-core-web-lg-3.4.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_lg')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "roberta = torch.hub.load('pytorch/fairseq', 'roberta.large.wsc', user_dir='examples/roberta/wsc')"
      ],
      "metadata": {
        "id": "y8OJYVMk4eH6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0b2f9760-a9eb-43c6-dfc9-61be9515f12b"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading: \"https://github.com/pytorch/fairseq/zipball/main\" to /root/.cache/torch/hub/main.zip\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "running build_ext\n",
            "cythoning fairseq/data/data_utils_fast.pyx to fairseq/data/data_utils_fast.cpp\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/cpp_extension.py:411: UserWarning: Attempted to use ninja as the BuildExtension backend but we could not find ninja.. Falling back to using the slow distutils backend.\n",
            "  warnings.warn(msg.format('we could not find ninja.'))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cythoning fairseq/data/token_block_utils_fast.pyx to fairseq/data/token_block_utils_fast.cpp\n",
            "building 'fairseq.libbleu' extension\n",
            "creating build\n",
            "creating build/temp.linux-x86_64-3.7\n",
            "creating build/temp.linux-x86_64-3.7/fairseq\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/clib\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/clib/libbleu\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.7m -c fairseq/clib/libbleu/libbleu.cpp -o build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/libbleu.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=libbleu -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/python3.7m -c fairseq/clib/libbleu/module.cpp -o build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/module.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=libbleu -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "creating build/lib.linux-x86_64-3.7\n",
            "creating build/lib.linux-x86_64-3.7/fairseq\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 -Wl,-Bsymbolic-functions -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/libbleu.o build/temp.linux-x86_64-3.7/fairseq/clib/libbleu/module.o -o build/lib.linux-x86_64-3.7/fairseq/libbleu.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'fairseq.data.data_utils_fast' extension\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/data\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c fairseq/data/data_utils_fast.cpp -o build/temp.linux-x86_64-3.7/fairseq/data/data_utils_fast.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=data_utils_fast -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "creating build/lib.linux-x86_64-3.7/fairseq/data\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 -Wl,-Bsymbolic-functions -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/data/data_utils_fast.o -o build/lib.linux-x86_64-3.7/fairseq/data/data_utils_fast.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'fairseq.data.token_block_utils_fast' extension\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c fairseq/data/token_block_utils_fast.cpp -o build/temp.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.o -std=c++11 -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=token_block_utils_fast -D_GLIBCXX_USE_CXX11_ABI=0\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 -Wl,-Bsymbolic-functions -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.o -o build/lib.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'fairseq.libbase' extension\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/clib/libbase\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/torch/include -I/usr/local/lib/python3.7/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.7/dist-packages/torch/include/TH -I/usr/local/lib/python3.7/dist-packages/torch/include/THC -I/usr/include/python3.7m -c fairseq/clib/libbase/balanced_assignment.cpp -o build/temp.linux-x86_64-3.7/fairseq/clib/libbase/balanced_assignment.o -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=libbase -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 -Wl,-Bsymbolic-functions -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/clib/libbase/balanced_assignment.o -L/usr/local/lib/python3.7/dist-packages/torch/lib -lc10 -ltorch -ltorch_cpu -ltorch_python -o build/lib.linux-x86_64-3.7/fairseq/libbase.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'fairseq.libnat' extension\n",
            "creating build/temp.linux-x86_64-3.7/fairseq/clib/libnat\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/torch/include -I/usr/local/lib/python3.7/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.7/dist-packages/torch/include/TH -I/usr/local/lib/python3.7/dist-packages/torch/include/THC -I/usr/include/python3.7m -c fairseq/clib/libnat/edit_dist.cpp -o build/temp.linux-x86_64-3.7/fairseq/clib/libnat/edit_dist.o -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=libnat -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 -Wl,-Bsymbolic-functions -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/fairseq/clib/libnat/edit_dist.o -L/usr/local/lib/python3.7/dist-packages/torch/lib -lc10 -ltorch -ltorch_cpu -ltorch_python -o build/lib.linux-x86_64-3.7/fairseq/libnat.cpython-37m-x86_64-linux-gnu.so\n",
            "building 'alignment_train_cpu_binding' extension\n",
            "creating build/temp.linux-x86_64-3.7/examples\n",
            "creating build/temp.linux-x86_64-3.7/examples/operators\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/torch/include -I/usr/local/lib/python3.7/dist-packages/torch/include/torch/csrc/api/include -I/usr/local/lib/python3.7/dist-packages/torch/include/TH -I/usr/local/lib/python3.7/dist-packages/torch/include/THC -I/usr/include/python3.7m -c examples/operators/alignment_train_cpu.cpp -o build/temp.linux-x86_64-3.7/examples/operators/alignment_train_cpu.o -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE=\"_gcc\" -DPYBIND11_STDLIB=\"_libstdcpp\" -DPYBIND11_BUILD_ABI=\"_cxxabi1011\" -DTORCH_EXTENSION_NAME=alignment_train_cpu_binding -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 -Wl,-Bsymbolic-functions -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.7/examples/operators/alignment_train_cpu.o -L/usr/local/lib/python3.7/dist-packages/torch/lib -lc10 -ltorch -ltorch_cpu -ltorch_python -o build/lib.linux-x86_64-3.7/alignment_train_cpu_binding.cpython-37m-x86_64-linux-gnu.so\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/libbleu.cpython-37m-x86_64-linux-gnu.so -> fairseq\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/data/data_utils_fast.cpython-37m-x86_64-linux-gnu.so -> fairseq/data\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/data/token_block_utils_fast.cpython-37m-x86_64-linux-gnu.so -> fairseq/data\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/libbase.cpython-37m-x86_64-linux-gnu.so -> fairseq\n",
            "copying build/lib.linux-x86_64-3.7/fairseq/libnat.cpython-37m-x86_64-linux-gnu.so -> fairseq\n",
            "copying build/lib.linux-x86_64-3.7/alignment_train_cpu_binding.cpython-37m-x86_64-linux-gnu.so -> \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 655510705/655510705 [04:06<00:00, 2660603.65B/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "| dictionary: 50265 types\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "1042301B [00:01, 759586.91B/s]\n",
            "456318B [00:00, 551526.32B/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "source_sentence = \"The doctor finished her work.\"\n",
        "trans_google = \"O médico terminou seu trabalho.\"\n",
        "trans_model = \"A médica acabou seu trabalho.\""
      ],
      "metadata": {
        "id": "p0-j1lBE5jgk"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "roberta.disambiguate_pronoun('A médica terminou o trabalho [dela].')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "b1Prw_8R52UF",
        "outputId": "303daa6e-0706-467f-b168-a09b811a22bc"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'médica'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "roberta.disambiguate_pronoun('The doctor finished [her] work.')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "OFKOg-UBrwNA",
        "outputId": "8b03a9c7-cf92-4423-b4cf-2e1ebacbdd9f"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The doctor'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokens = roberta.encode(\"Hello world\")\n",
        "t = roberta.extract_features(tokens)\n"
      ],
      "metadata": {
        "id": "oy1WrILWwOxi"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tem indicativo de gênero na frase? "
      ],
      "metadata": {
        "id": "sMu6qoAy6t33"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "\n",
        "nlp_en = spacy.load(\"en_core_web_lg\")\n",
        "\n",
        "source_sentence = nlp_en(source_sentence)\n",
        "\n",
        "for token in source_sentence:\n",
        "  print(token.pos_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wAQIzVZ47vwc",
        "outputId": "21f10899-27cf-4eb0-c817-7a98d6e9572d"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DET\n",
            "NOUN\n",
            "VERB\n",
            "PRON\n",
            "NOUN\n",
            "PUNCT\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "has_pronoun = False\n",
        "\n",
        "for token in source_sentence:\n",
        "  if token.pos_ is 'PRON':\n",
        "    has_pronoun = True\n",
        "\n",
        "print(has_pronoun)    "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jRxMlaPYBa6u",
        "outputId": "7a2ba3a5-1838-4039-caf3-4ec67bc54b9c"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "True\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "get_pronoun = ''\n",
        "\n",
        "for token in source_sentence: \n",
        "  if token.pos_ is 'PRON':\n",
        "    get_pronoun = token\n",
        "\n",
        "print(get_pronoun)    "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dLxkC-w7CjUE",
        "outputId": "bda50bd5-9526-4d55-f566-e8ef55bf1f81"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "her\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pronoun_text = \"[\" + get_pronoun.text + \"]\"\n",
        "new_source_sentence = source_sentence.text.replace(get_pronoun.text, pronoun_text)\n",
        "\n",
        "new_source_sentence"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "zVnJvNc6GPtk",
        "outputId": "1ad6f777-79fe-4e76-d792-db31976e5be0"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The doctor finished [her] work.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "subj = roberta.disambiguate_pronoun(new_source_sentence)\n",
        "\n",
        "subj"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "yNLJOgW0Ek-4",
        "outputId": "83f3d39f-97f2-4242-d9d4-1e7d0169c96d"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The doctor'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "get_pronoun.morph\n",
        "\n",
        "\n",
        "is_fem = False\n",
        "is_masc = False \n",
        "is_neutral = False\n",
        "\n",
        "gender = get_pronoun.morph.get(\"Gender\")\n",
        "\n",
        "if gender == ['Fem']:\n",
        "  is_fem = True\n",
        "elif gender == ['Masc']:\n",
        "  is_masc = True\n",
        "else:\n",
        "  is_neutral = True\n",
        "\n",
        "print(is_fem)    \n",
        "print(is_masc)    \n",
        "print(is_neutral)    "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dNz5JmJ7FtVG",
        "outputId": "287ae3d3-57a4-497d-891a-6d1c96b61532"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "True\n",
            "False\n",
            "False\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "subj_model = roberta.disambiguate_pronoun('A médica terminou o trabalho [dela].')"
      ],
      "metadata": {
        "id": "JRZH1qNyLTtd"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "subj_model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "lx5sGab3LoaX",
        "outputId": "a2e09bfc-431a-43aa-feee-8a4677371845"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'médica'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "subj"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "MLnHKqRqLZ0N",
        "outputId": "41b3a95a-66e6-400e-9e6b-e2fb4b93f465"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'The doctor'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_model = nlp(subj_model)\n",
        "test_model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hOtqZLaTL1HX",
        "outputId": "5c440f54-368a-4c28-fb47-62a66bdecd8d"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "médica"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_source = nlp_en(subj)\n",
        "test_source"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4IRPpcVhMNDP",
        "outputId": "a47184bf-1ef5-4ba9-cd3a-f3eeebc7a052"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "The doctor"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in test_model: \n",
        "  print(i.morph)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JfHczOUIMQB-",
        "outputId": "360840dc-2229-4d47-d941-8305531d12b7"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Gender=Fem|Number=Sing\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in test_source: \n",
        "  print(i.morph)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TYpQD3TBMXU2",
        "outputId": "f01acdc7-babe-40b2-fec3-6f1db7afc0a9"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Definite=Def|PronType=Art\n",
            "Number=Sing\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "is_fem_trans = False\n",
        "is_masc_trans = False \n",
        "is_neutral_trans = False\n",
        "\n",
        "\n",
        "for g in test_model:\n",
        "  gender = g.morph.get(\"Gender\")\n",
        "  if gender == ['Fem']:\n",
        "    is_fem_trans = True\n",
        "  elif gender == ['Masc']:\n",
        "    is_masc_trans = True\n",
        "  else:\n",
        "    is_neutral_trans = True\n",
        "\n",
        "print(is_fem_trans)    \n",
        "print(is_masc_trans)    \n",
        "print(is_neutral_trans)   "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DsnsVxPkMpH6",
        "outputId": "590abb6b-70e2-48d3-b335-655121f4dac5"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "True\n",
            "False\n",
            "False\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "are_both_fem = is_fem and is_fem_trans\n",
        "are_both_masc = is_masc and is_masc_trans\n",
        "are_both_neutral = is_neutral and is_neutral_trans\n",
        "\n",
        "print(are_both_fem)\n",
        "print(are_both_masc)\n",
        "print(are_both_neutral)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8QSd02n0Neje",
        "outputId": "433e3b08-f0a7-440d-b9d8-21e0a83f03eb"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "True\n",
            "False\n",
            "False\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets"
      ],
      "metadata": {
        "id": "NC28HHvMOy6M",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "33a15015-bb2f-4715-aaa1-02346867dc1e"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.5.2-py3-none-any.whl (432 kB)\n",
            "\u001b[K     |████████████████████████████████| 432 kB 37.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.10.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.8.2)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 70.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (5.0.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.13-py37-none-any.whl (115 kB)\n",
            "\u001b[K     |████████████████████████████████| 115 kB 81.8 MB/s \n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.1.1)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.8.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 95.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.8.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: urllib3, xxhash, responses, multiprocess, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.5.2 multiprocess-0.70.13 responses-0.18.0 urllib3-1.25.11 xxhash-3.0.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "urllib3"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "remote_dataset = load_dataset(\"VanessaSchenkel/pt_gender\", field=\"data\")\n",
        "remote_dataset\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 321,
          "referenced_widgets": [
            "6954c3bb83794fdf9239271952f1fb45",
            "e68da42ca32040b8ab49598d6394f836",
            "53933f2782054b228279893f0e314600",
            "75d0d259e8f74e7887e8784b4f6dde10",
            "708156f113f14296b2995d35c1ad7d9a",
            "dceac6f5c6ab4c5c9dc6d81352681ea4",
            "14015b2e76c44152a9865704a5d3019a",
            "9ce82f1d13c949b5b4c684db916a11e5",
            "77409f6414d147b99b5dfaa852fccddf",
            "5d62146bfdeb4e9fb740bce29702054d",
            "9e3b494c03f546718f267514008ac5a4",
            "cf2b00c2df254338ae7dbb3d8d387712",
            "5145b94d29d54e3783fed64976c7c84c",
            "d6167c546db34a659521aebcdf909af1",
            "7dad2c741fb34c87b4a87b96267bf1e5",
            "8a0a6861f869461194752fa966b2244d",
            "2dd9793ab5a7406db31562d09cef0926",
            "5912e357db28415fab9f3910f3983780",
            "7f897216e3a7460d81d07aedf66a8a74",
            "7e0d8fc79cf74f618cecd1d6b2363522",
            "82adb244041a4e75b2f6ce263a9a181e",
            "41f72ce1b96f4fdfb495098882d6cfbf",
            "105e8dc7c93b42739115481fd3bc0fe8",
            "bcdfa7ca8bd84eebb9b956071b59d7ac",
            "cd59a9e4395f4cbca0d0007ebe516f1b",
            "4c1596cd65854539867eb4beef890baa",
            "c79507654dfd4cf5b25f34fdd6cad49c",
            "f69610bfcabd415b899de8832326550f",
            "dbcd2f911bed4edc9bce1a38fa8a718f",
            "2bbe61e47fea4137a3195a9772240646",
            "715b7cf259534bc6974c5eb4b62c1a70",
            "0ba8dcbffc3547d8821ce8f826c7939d",
            "3fb5ee73e5014ee2bbe4805c2d5e2341",
            "bbad978a2c744ce3882698e3dfce8681",
            "91b1fba7cf4e4c269a85748994967fb1",
            "50608051d0d04ea588e208bcb3ca08c6",
            "f961e96228414401870ceb7c6b59a8d8",
            "cd165997e017492b8946c443499b4e0e",
            "0a81904f1a1f4cc3a37975ae02329c05",
            "bb6191736d9b4943966fe2cb7e454960",
            "45d93ec095fc408b8fe7cce0c331b712",
            "336c10b314fc44efaa6a38dd16765f36",
            "8eb488f765d84cf592c5b50ad9773a80",
            "7c73813598084f6eb3826de6308f9d83",
            "2cdebc1c88d74b959d55074762251240",
            "9f61c121c85c48a6bc22c66efd1d5c78",
            "87193f8d13cd4b12abbd257f4ffab476",
            "a018c52c922b4c7a941ae676cc2891f6",
            "060f5217c77142b29c2b128a05374d0b",
            "cdd180fbd2d24fada66370ac2aab2f2c",
            "3559bcc8b7344c3e898e8ce1528987a8",
            "4fe16b55594d48f09a3408dd7e2dab90",
            "9e2474ea293c4851bc14e0739beb5f08",
            "af88ea7f07db4d42b3d12fff68ab6055",
            "dac1af8fb4844cd797be7dc0950b67c2"
          ]
        },
        "id": "BJx2o1yrOkRE",
        "outputId": "16a130ff-24f7-4995-a34d-96ec84298e19"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Using custom data configuration VanessaSchenkel--pt_gender-95b0f9b96d3847c6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset json/VanessaSchenkel--pt_gender to /root/.cache/huggingface/datasets/VanessaSchenkel___json/VanessaSchenkel--pt_gender-95b0f9b96d3847c6/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6954c3bb83794fdf9239271952f1fb45"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/39.1M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cf2b00c2df254338ae7dbb3d8d387712"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "105e8dc7c93b42739115481fd3bc0fe8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "0 tables [00:00, ? tables/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bbad978a2c744ce3882698e3dfce8681"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/VanessaSchenkel___json/VanessaSchenkel--pt_gender-95b0f9b96d3847c6/0.0.0/e6070c77f18f01a5ad4551a8b7edfba20b8438b7cad4d94e6ad9378022ce4aab. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2cdebc1c88d74b959d55074762251240"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['root', 'words'],\n",
              "        num_rows: 321265\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t = remote_dataset.filter(lambda example: \"médica\" in example['words'])\n",
        "\n",
        "t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153,
          "referenced_widgets": [
            "d45f30ec556346c5ac7913300ccd23b5",
            "9ff57c6e0f8241c4a1fbb373cd8f8873",
            "f5f58b3269b5450fa63bcbba11222ff4",
            "ba64dee9c91842a29408ad7dd94c38bc",
            "1a05add1270e497db9b77adcfb474267",
            "717370d4804f4f798e3f2fc2a3be73f6",
            "90732145035c4baaab99a31e47a9623f",
            "a0949215fc6b48ada31fe667ac544159",
            "24bab9d57e85495883c16081db42cae5",
            "f32815e5e970431f9adfac6198100fd4",
            "02d89183bb2f48f79110f1579154ff37"
          ]
        },
        "id": "QGpdPUgdPJpX",
        "outputId": "62cf959d-663f-4f61-cd0d-96db693753b6"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/322 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d45f30ec556346c5ac7913300ccd23b5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['root', 'words'],\n",
              "        num_rows: 1\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t['train'][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EXIULFy2SFiX",
        "outputId": "d052c81a-a596-4607-b473-8934cc0a6b5e"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'root': 'médico', 'words': ['médico', 'médicos', 'médica', 'médicas']}"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words_with_gender = t['train'][0]['words']\n",
        "words_with_gender"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9h6qw2tqqPeW",
        "outputId": "86aab551-0849-47d5-e8cf-84b4c0bbe80b"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['médico', 'médicos', 'médica', 'médicas']"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"The doctor finished her work.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "flexible_phrases = tokenizer([\"médica\", \"médico\"], add_special_tokens=False).input_ids\n",
        "\n",
        "constraints = [DisjunctiveConstraint(flexible_phrases)]\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    constraints=constraints,\n",
        "    num_beams=10,\n",
        "    num_return_sequences=6,\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kxa4V7CqqZ4V",
        "outputId": "241f4d57-e2e6-49c2-dd32-ae1f886b2508"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:1301: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 20 (`self.config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  UserWarning,\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "0: A médica terminou seu trabalho.\n",
            "1: A médica terminou o seu trabalho.\n",
            "2: O médico terminou seu trabalho.\n",
            "3: A médica acabou seu trabalho.\n",
            "4: O médico terminou o seu trabalho.\n",
            "5: A médica acabou o seu trabalho.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**constraints** (List[Constraint], optional) — Custom constraints that can be added to the generation to ensure that the output will contain the use of certain tokens as defined by Constraint objects, in the most sensible way possible.\n",
        "\n",
        "**force_words_ids**(List[List[int]] or List[List[List[int]]], optional) — List of token ids that must be generated. If given a List[List[int]], this is treated as a simple list of words that must be included, the opposite to bad_words_ids. If given List[List[List[int]]], this triggers a disjunctive constraint, where one can allow different forms of each word."
      ],
      "metadata": {
        "id": "7rlGQd19w1-i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"The developer finished her work.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "force_word = \"terminou o trabalho\"\n",
        "force_flexible = [\"desenvolvedor\", \"desenvolvedora\"]\n",
        "\n",
        "force_words_ids = [\n",
        "    tokenizer([force_word], add_special_tokens=False).input_ids,\n",
        "    tokenizer(force_flexible, add_special_tokens=False).input_ids,\n",
        "]\n",
        "\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    force_words_ids=force_words_ids,\n",
        "    num_beams=20,\n",
        "    num_return_sequences=3,\n",
        "    top_k=50, \n",
        "    top_p=0.95,\n",
        "    temperature=0.7,\n",
        "    no_repeat_ngram_size=4\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z0FXpxiL5kHz",
        "outputId": "cd7b7d06-fb9d-40f9-e839-b5264884c97a"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Output:\n",
            "----------------------------------------------------------------------------------------------------\n",
            "0: A desenvolvedora terminou o trabalho.\n",
            "1: O desenvolvedor terminou o trabalho.\n",
            "2: O desenvolvedora terminou o trabalho.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Source tem gênero e é a mesma da tradução? "
      ],
      "metadata": {
        "id": "2UXL22PWAbTI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "source_sentence = \"The doctor finished her work\"\n",
        "\n",
        "print(source_sentence)\n",
        "print(translation_google)\n",
        "print(translation_model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RFs6e7btAaRP",
        "outputId": "2dcb4eae-9103-4383-9817-591c601bb42a"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The doctor finished her work\n",
            "O médico terminou seu trabalho.\n",
            "A médica terminou seu trabalho.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_pronoun(sentence):\n",
        "  pronoun_list = []\n",
        "  for token in sentence:\n",
        "    if token.pos_ is 'PRON':\n",
        "      pronoun_list.append(token)\n",
        "\n",
        "  return pronoun_list\n"
      ],
      "metadata": {
        "id": "YX0VVJBNDRKy"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "source_sentence = nlp_en(\"The doctor finished her work.\")\n",
        "\n",
        "pronouns_list = get_pronoun(source_sentence)\n",
        "\n",
        "print(pronouns_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ubjHa3gLEfms",
        "outputId": "6ce06ba6-5850-4a94-88d0-712bb77cbfa8"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[her]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# def get_disambiguate_pronoun(sentence, pronouns_list):\n",
        "#   noun_chunks_list = []\n",
        "  \n",
        "#   for pronoun in pronouns_list:\n",
        "#     pronoun_text_formatted = \"[\" + pronoun.text + \"]\"\n",
        "#     new_source_sentence = sentence.text.replace(pronoun.text, pronoun_text_formatted)\n",
        "#     chunk = roberta.disambiguate_pronoun(new_source_sentence)\n",
        "#     noun_chunks_list.append(chunk)\n",
        "    \n",
        "#   return noun_chunks_list\n",
        "\n",
        "\n",
        "def get_disambiguate_pronoun(sentence, pronoun):\n",
        "  pronoun_text_formatted = \"[\" + pronoun.text + \"]\"\n",
        "  new_source_sentence = sentence.text.replace(pronoun.text, pronoun_text_formatted)\n",
        "  return roberta.disambiguate_pronoun(new_source_sentence)"
      ],
      "metadata": {
        "id": "mqdSKY06J4V9"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_sentence_gender(sentence):\n",
        "  gender_list = []\n",
        "  for token in sentence:\n",
        "    gender = token.morph.get(\"Gender\")\n",
        "    if len(gender) > 0:\n",
        "      gender_list.append(gender.pop())\n",
        "    \n",
        "  return gender_list  \n"
      ],
      "metadata": {
        "id": "vP-8WWrsKXa4"
      },
      "execution_count": 52,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "get_sentence_gender(source_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QwrDT9HuNYYx",
        "outputId": "9be32a0f-6902-4e5c-c3d6-5885b969eb96"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Fem']"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def is_source_and_translated_same_gender(source, translated):\n",
        "  source_gender = get_sentence_gender(source)\n",
        "  translated_gender = get_sentence_gender(translated)\n",
        "  print(source_gender, translated_gender)\n",
        "  return source_gender == translated_gender"
      ],
      "metadata": {
        "id": "g-UQPt8TOqrO"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "translated = nlp(\"A médica terminou o seu trabalho\")\n",
        "\n",
        "is_source_and_translated_same_gender(source_sentence, translated)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V-jTsg56PEZO",
        "outputId": "f6f5db52-31a8-41cd-ac8a-d86e62949607"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Fem'] ['Fem', 'Fem', 'Masc', 'Masc', 'Masc']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns_list_pt = get_pronoun(translated)\n",
        "pronouns_list_pt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-GFBCYlLROty",
        "outputId": "4c572f37-73f0-463a-eaf0-ffa463c02594"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in translated.noun_chunks:\n",
        "  print(token)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vjGJw0alSpwD",
        "outputId": "0c00f2a7-88b4-404f-a4bb-b87de624d2e8"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A médica\n",
            "o seu trabalho\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in source_sentence.noun_chunks:\n",
        "  print(token)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZjklC40FgKzO",
        "outputId": "2d12e61d-721c-4fc2-f474-d7e216f5b52a"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The doctor\n",
            "her work\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in source_sentence:\n",
        "  print(token.dep_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "foW1Nx4ggs1i",
        "outputId": "c695d18f-b650-46de-ef69-6ffc624ed5a0"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "det\n",
            "nsubj\n",
            "ROOT\n",
            "poss\n",
            "dobj\n",
            "punct\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in translated:\n",
        "  print(token.dep_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CfKJygC1gzK4",
        "outputId": "5433554e-af77-4de4-d093-853bb4b884d2"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "det\n",
            "nsubj\n",
            "ROOT\n",
            "det\n",
            "det\n",
            "obj\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spacy.explain('nsubj')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "XR7lkH6VhQFd",
        "outputId": "0d4ef556-caeb-4576-aa24-d97a3fc6f5ee"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'nominal subject'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_nsub_sentence(sentence):\n",
        "  for token in sentence:\n",
        "    if token.dep_ == 'nsubj':\n",
        "      return token"
      ],
      "metadata": {
        "id": "TCHXc6LliNsB"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nsub_translated = get_nsub_sentence(translated)\n",
        "nsub_translated"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ocD-51c4iw3q",
        "outputId": "0f168397-1b49-4807-a1c5-350beeb47930"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "médica"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "is_source_and_translated_same_gender(source_sentence, nsub_translated)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 311
        },
        "id": "44He7w--i5KK",
        "outputId": "f91951ea-4824-4101-d58b-5b2fbe38c4d0"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-64-23273ef8483b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mis_source_and_translated_same_gender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_sentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsub_translated\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-54-f33f784ef9e8>\u001b[0m in \u001b[0;36mis_source_and_translated_same_gender\u001b[0;34m(source, translated)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mis_source_and_translated_same_gender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtranslated\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m   \u001b[0msource_gender\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_sentence_gender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m   \u001b[0mtranslated_gender\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_sentence_gender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtranslated\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_gender\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtranslated_gender\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0msource_gender\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mtranslated_gender\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-52-6b00c6eb56ab>\u001b[0m in \u001b[0;36mget_sentence_gender\u001b[0;34m(sentence)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_sentence_gender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m   \u001b[0mgender_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mgender\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmorph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Gender\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgender\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'spacy.tokens.token.Token' object is not iterable"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translation_google = nlp(\"O médico acabou seu trabalho\")"
      ],
      "metadata": {
        "id": "Ub6jagAwkCH6"
      },
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nsub_translated_google = get_nsub_sentence(translation_google)\n",
        "nsub_translated_google"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GubT-FFfkQHY",
        "outputId": "a992e08c-0d3a-41e4-c445-aed3a92db98c"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "médico"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "table = {}\n",
        "text_list = []\n",
        "anc = []\n",
        "child = []\n",
        "dep = []\n",
        "head = []\n",
        "lemma = []\n",
        "morph = []\n",
        "pos = []\n",
        "tag = []\n",
        "norm = []\n",
        "suffix = []\n",
        "pref = []\n",
        "\n",
        "for token in translation_google:\n",
        "  text_list.append(token.text)\n",
        "  anc.append([ancestor for ancestor in token.ancestors])\n",
        "  child.append([child for child in token.children])\n",
        "  dep.append(token.dep_)\n",
        "  head.append(token.head)\n",
        "  lemma.append(token.lemma_)\n",
        "  morph.append(token.morph)\n",
        "  pos.append(token.pos_)\n",
        "  norm.append(token.norm_)\n",
        "  pref.append(token.prefix_)\n",
        "  suffix.append(token.suffix_)\n",
        "\n",
        "table['text'] = text_list\n",
        "table['anc'] = anc\n",
        "table['child'] = child\n",
        "table['dep'] = dep\n",
        "table['head'] = head\n",
        "table['lemma'] = lemma\n",
        "table['morph'] = morph\n",
        "table['pos'] = pos\n",
        "table['norm'] = norm\n",
        "table['pref'] = pref\n",
        "table['suffix'] = suffix\n",
        "\n",
        "\n",
        "\n",
        "df = pd.DataFrame(table)\n",
        "\n",
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "7jKv2JsWlLfP",
        "outputId": "c8a83109-c3f5-4d23-9e3e-754a5ce3cc0b"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "       text                 anc               child    dep      head  \\\n",
              "0         O    [médico, acabou]                  []    det    médico   \n",
              "1    médico            [acabou]                 [O]  nsubj    acabou   \n",
              "2    acabou                  []  [médico, trabalho]   ROOT    acabou   \n",
              "3       seu  [trabalho, acabou]                  []    det  trabalho   \n",
              "4  trabalho            [acabou]               [seu]    obj    acabou   \n",
              "\n",
              "      lemma                                              morph   pos  \\\n",
              "0         o  (Definite=Def, Gender=Masc, Number=Sing, PronT...   DET   \n",
              "1    médico                         (Gender=Masc, Number=Sing)  NOUN   \n",
              "2    acabar  (Mood=Ind, Number=Sing, Person=3, Tense=Past, ...  VERB   \n",
              "3       seu           (Gender=Masc, Number=Sing, PronType=Prs)   DET   \n",
              "4  trabalho                         (Gender=Masc, Number=Sing)  NOUN   \n",
              "\n",
              "       norm pref suffix  \n",
              "0         o    O      O  \n",
              "1    médico    m    ico  \n",
              "2    acabou    a    bou  \n",
              "3       seu    s    seu  \n",
              "4  trabalho    t    lho  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ea2da805-d42c-4c77-8b0b-7107896acbc2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>anc</th>\n",
              "      <th>child</th>\n",
              "      <th>dep</th>\n",
              "      <th>head</th>\n",
              "      <th>lemma</th>\n",
              "      <th>morph</th>\n",
              "      <th>pos</th>\n",
              "      <th>norm</th>\n",
              "      <th>pref</th>\n",
              "      <th>suffix</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>O</td>\n",
              "      <td>[médico, acabou]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>médico</td>\n",
              "      <td>o</td>\n",
              "      <td>(Definite=Def, Gender=Masc, Number=Sing, PronT...</td>\n",
              "      <td>DET</td>\n",
              "      <td>o</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>médico</td>\n",
              "      <td>[acabou]</td>\n",
              "      <td>[O]</td>\n",
              "      <td>nsubj</td>\n",
              "      <td>acabou</td>\n",
              "      <td>médico</td>\n",
              "      <td>(Gender=Masc, Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>médico</td>\n",
              "      <td>m</td>\n",
              "      <td>ico</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>acabou</td>\n",
              "      <td>[]</td>\n",
              "      <td>[médico, trabalho]</td>\n",
              "      <td>ROOT</td>\n",
              "      <td>acabou</td>\n",
              "      <td>acabar</td>\n",
              "      <td>(Mood=Ind, Number=Sing, Person=3, Tense=Past, ...</td>\n",
              "      <td>VERB</td>\n",
              "      <td>acabou</td>\n",
              "      <td>a</td>\n",
              "      <td>bou</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>seu</td>\n",
              "      <td>[trabalho, acabou]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>trabalho</td>\n",
              "      <td>seu</td>\n",
              "      <td>(Gender=Masc, Number=Sing, PronType=Prs)</td>\n",
              "      <td>DET</td>\n",
              "      <td>seu</td>\n",
              "      <td>s</td>\n",
              "      <td>seu</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>trabalho</td>\n",
              "      <td>[acabou]</td>\n",
              "      <td>[seu]</td>\n",
              "      <td>obj</td>\n",
              "      <td>acabou</td>\n",
              "      <td>trabalho</td>\n",
              "      <td>(Gender=Masc, Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>trabalho</td>\n",
              "      <td>t</td>\n",
              "      <td>lho</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ea2da805-d42c-4c77-8b0b-7107896acbc2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ea2da805-d42c-4c77-8b0b-7107896acbc2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ea2da805-d42c-4c77-8b0b-7107896acbc2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_constrained_sentence(translation, nsub):\n",
        "  constrained_sentence = \"\"\n",
        "  children = [child for child in nsub[0].children]\n",
        "  for token in translation:\n",
        "    if token not in nsub and token not in children:\n",
        "      constrained_sentence += token.text + \" \"\n",
        "\n",
        "  return constrained_sentence"
      ],
      "metadata": {
        "id": "vSsH5WQ4lfE5"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "constrained_sentence = get_constrained_sentence(translation_google, nsub_translated_google)\n",
        "constrained_sentence"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "id": "Csw7aOmNmCTH",
        "outputId": "b3e89a24-8c37-4ce1-9ffa-b344d0132cb1"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-69-7c3a4e3da8bc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mconstrained_sentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_constrained_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtranslation_google\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsub_translated_google\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mconstrained_sentence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-68-e519577b3bf1>\u001b[0m in \u001b[0;36mget_constrained_sentence\u001b[0;34m(translation, nsub)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_constrained_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtranslation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsub\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m   \u001b[0mconstrained_sentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m   \u001b[0mchildren\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mchild\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mchild\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnsub\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtranslation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnsub\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mchildren\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'spacy.tokens.token.Token' object is not subscriptable"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"The doctor finished her work.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "force_word = constrained_sentence.strip()\n",
        "\n",
        "force_words_ids = [\n",
        "    tokenizer([force_word], add_special_tokens=False).input_ids,\n",
        "]\n",
        "\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    force_words_ids=force_words_ids,\n",
        "    num_beams=20,\n",
        "    num_return_sequences=3,\n",
        "    top_k=50, \n",
        "    top_p=0.95,\n",
        "    temperature=0.7,\n",
        "    no_repeat_ngram_size=4\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "id": "VwQFwPvppWTK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Source tem gênero e não é a mesma da tradução?"
      ],
      "metadata": {
        "id": "iDJipWJ1sEAW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "translated_masc = nlp(\"O médico acabou seu trabalho\")\n",
        "nsub_translated = get_nsub_sentence(translated_masc)\n",
        "nsub_translated"
      ],
      "metadata": {
        "id": "lU3VQIGzLT2J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "is_source_and_translated_same_gender(source_sentence, nsub_translated)"
      ],
      "metadata": {
        "id": "RhkjPwInLvRi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "constrained_sentence = get_constrained_sentence(translated_masc, nsub_translated)\n",
        "constrained_sentence"
      ],
      "metadata": {
        "id": "WFXo9uTgL29B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"The doctor finished her work.\"\n",
        "input_ids = tokenizer(sentence, return_tensors=\"pt\").input_ids\n",
        "\n",
        "force_word = constrained_sentence.strip()\n",
        "\n",
        "force_words_ids = [\n",
        "    tokenizer([force_word], add_special_tokens=False).input_ids,\n",
        "]\n",
        "\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids,\n",
        "    force_words_ids=force_words_ids,\n",
        "    num_beams=20,\n",
        "    num_return_sequences=3,\n",
        "    top_k=50, \n",
        "    top_p=0.95,\n",
        "    temperature=0.7,\n",
        "    no_repeat_ngram_size=4\n",
        ")\n",
        "\n",
        "print(\"Output:\\n\" + 100 * '-')\n",
        "for i, sample_output in enumerate(outputs):\n",
        "  print(\"{}: {}\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "id": "17Y4g_jjMLhU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Source tem mais de um gênero?"
      ],
      "metadata": {
        "id": "sQRUXM3jsRA0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "source_sentence = nlp_en(\"The doctor finished her work, the nurse was still on his break\")\n",
        "translation_google = nlp(\"O médico terminou seu trabalho, a enfermeira ainda estava de folga\")\n",
        "translation_model = nlp(\"O médico terminou seu trabalho, a enfermeira ainda era a sua pausa.\")\n",
        "\n",
        "print(source_sentence)\n",
        "print(translation_google)\n",
        "print(translation_model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "De9tSuKZMkl6",
        "outputId": "740aa8a9-eb15-4f7c-edc3-c9161204cf4d"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The doctor finished her work, the nurse was still on his break\n",
            "O médico terminou seu trabalho, a enfermeira ainda estava de folga\n",
            "O médico terminou seu trabalho, a enfermeira ainda era a sua pausa.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns_list = get_pronoun(source_sentence)\n",
        "\n",
        "print(pronouns_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jBmMbXJoN8fp",
        "outputId": "309eb2f0-2386-40a2-dbe8-77b05e53ae2e"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[her, his]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "get_sentence_gender(source_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X4aIqUFgOMhx",
        "outputId": "b0ffb810-c2a8-48d6-a015-d34c08abbd2f"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Fem', 'Masc']"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "table = {}\n",
        "text_list = []\n",
        "anc = []\n",
        "child = []\n",
        "dep = []\n",
        "head = []\n",
        "lemma = []\n",
        "morph = []\n",
        "pos = []\n",
        "tag = []\n",
        "norm = []\n",
        "suffix = []\n",
        "pref = []\n",
        "\n",
        "for token in source_sentence:\n",
        "  text_list.append(token.text)\n",
        "  anc.append([ancestor for ancestor in token.ancestors])\n",
        "  child.append([child for child in token.children])\n",
        "  dep.append(token.dep_)\n",
        "  head.append(token.head)\n",
        "  lemma.append(token.lemma_)\n",
        "  morph.append(token.morph)\n",
        "  pos.append(token.pos_)\n",
        "  norm.append(token.norm_)\n",
        "  pref.append(token.prefix_)\n",
        "  suffix.append(token.suffix_)\n",
        "\n",
        "table['text'] = text_list\n",
        "table['anc'] = anc\n",
        "table['child'] = child\n",
        "table['dep'] = dep\n",
        "table['head'] = head\n",
        "table['lemma'] = lemma\n",
        "table['morph'] = morph\n",
        "table['pos'] = pos\n",
        "table['norm'] = norm\n",
        "table['pref'] = pref\n",
        "table['suffix'] = suffix\n",
        "\n",
        "\n",
        "\n",
        "df = pd.DataFrame(table)\n",
        "\n",
        "display(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 457
        },
        "id": "XWZxofZxPEZ9",
        "outputId": "52cc3e9b-e7d0-42a3-83da-7599cb066539"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "        text                      anc                            child  \\\n",
              "0        The  [doctor, finished, was]                               []   \n",
              "1     doctor          [finished, was]                            [The]   \n",
              "2   finished                    [was]                   [doctor, work]   \n",
              "3        her    [work, finished, was]                               []   \n",
              "4       work          [finished, was]                            [her]   \n",
              "5          ,                    [was]                               []   \n",
              "6        the             [nurse, was]                               []   \n",
              "7      nurse                    [was]                            [the]   \n",
              "8        was                       []  [finished, ,, nurse, still, on]   \n",
              "9      still                    [was]                               []   \n",
              "10        on                    [was]                          [break]   \n",
              "11       his         [break, on, was]                               []   \n",
              "12     break                [on, was]                            [his]   \n",
              "\n",
              "       dep      head   lemma  \\\n",
              "0      det    doctor     the   \n",
              "1    nsubj  finished  doctor   \n",
              "2    ccomp       was  finish   \n",
              "3     poss      work     her   \n",
              "4     dobj  finished    work   \n",
              "5    punct       was       ,   \n",
              "6      det     nurse     the   \n",
              "7    nsubj       was   nurse   \n",
              "8     ROOT       was      be   \n",
              "9   advmod       was   still   \n",
              "10    prep       was      on   \n",
              "11    poss     break     his   \n",
              "12    pobj        on   break   \n",
              "\n",
              "                                                morph    pos      norm pref  \\\n",
              "0                        (Definite=Def, PronType=Art)    DET       the    T   \n",
              "1                                       (Number=Sing)   NOUN    doctor    d   \n",
              "2                          (Tense=Past, VerbForm=Fin)   VERB  finished    f   \n",
              "3   (Gender=Fem, Number=Sing, Person=3, Poss=Yes, ...   PRON       her    h   \n",
              "4                                       (Number=Sing)   NOUN      work    w   \n",
              "5                                    (PunctType=Comm)  PUNCT         ,    ,   \n",
              "6                        (Definite=Def, PronType=Art)    DET       the    t   \n",
              "7                                       (Number=Sing)   NOUN     nurse    n   \n",
              "8   (Mood=Ind, Number=Sing, Person=3, Tense=Past, ...    AUX       was    w   \n",
              "9                                                  ()    ADV     still    s   \n",
              "10                                                 ()    ADP        on    o   \n",
              "11  (Gender=Masc, Number=Sing, Person=3, Poss=Yes,...   PRON       his    h   \n",
              "12                                      (Number=Sing)   NOUN     break    b   \n",
              "\n",
              "   suffix  \n",
              "0     The  \n",
              "1     tor  \n",
              "2     hed  \n",
              "3     her  \n",
              "4     ork  \n",
              "5       ,  \n",
              "6     the  \n",
              "7     rse  \n",
              "8     was  \n",
              "9     ill  \n",
              "10     on  \n",
              "11    his  \n",
              "12    eak  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-58d032df-0636-43e7-93b1-2fb86b170321\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>anc</th>\n",
              "      <th>child</th>\n",
              "      <th>dep</th>\n",
              "      <th>head</th>\n",
              "      <th>lemma</th>\n",
              "      <th>morph</th>\n",
              "      <th>pos</th>\n",
              "      <th>norm</th>\n",
              "      <th>pref</th>\n",
              "      <th>suffix</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The</td>\n",
              "      <td>[doctor, finished, was]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>doctor</td>\n",
              "      <td>the</td>\n",
              "      <td>(Definite=Def, PronType=Art)</td>\n",
              "      <td>DET</td>\n",
              "      <td>the</td>\n",
              "      <td>T</td>\n",
              "      <td>The</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>doctor</td>\n",
              "      <td>[finished, was]</td>\n",
              "      <td>[The]</td>\n",
              "      <td>nsubj</td>\n",
              "      <td>finished</td>\n",
              "      <td>doctor</td>\n",
              "      <td>(Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>doctor</td>\n",
              "      <td>d</td>\n",
              "      <td>tor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>finished</td>\n",
              "      <td>[was]</td>\n",
              "      <td>[doctor, work]</td>\n",
              "      <td>ccomp</td>\n",
              "      <td>was</td>\n",
              "      <td>finish</td>\n",
              "      <td>(Tense=Past, VerbForm=Fin)</td>\n",
              "      <td>VERB</td>\n",
              "      <td>finished</td>\n",
              "      <td>f</td>\n",
              "      <td>hed</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>her</td>\n",
              "      <td>[work, finished, was]</td>\n",
              "      <td>[]</td>\n",
              "      <td>poss</td>\n",
              "      <td>work</td>\n",
              "      <td>her</td>\n",
              "      <td>(Gender=Fem, Number=Sing, Person=3, Poss=Yes, ...</td>\n",
              "      <td>PRON</td>\n",
              "      <td>her</td>\n",
              "      <td>h</td>\n",
              "      <td>her</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>work</td>\n",
              "      <td>[finished, was]</td>\n",
              "      <td>[her]</td>\n",
              "      <td>dobj</td>\n",
              "      <td>finished</td>\n",
              "      <td>work</td>\n",
              "      <td>(Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>work</td>\n",
              "      <td>w</td>\n",
              "      <td>ork</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>,</td>\n",
              "      <td>[was]</td>\n",
              "      <td>[]</td>\n",
              "      <td>punct</td>\n",
              "      <td>was</td>\n",
              "      <td>,</td>\n",
              "      <td>(PunctType=Comm)</td>\n",
              "      <td>PUNCT</td>\n",
              "      <td>,</td>\n",
              "      <td>,</td>\n",
              "      <td>,</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>the</td>\n",
              "      <td>[nurse, was]</td>\n",
              "      <td>[]</td>\n",
              "      <td>det</td>\n",
              "      <td>nurse</td>\n",
              "      <td>the</td>\n",
              "      <td>(Definite=Def, PronType=Art)</td>\n",
              "      <td>DET</td>\n",
              "      <td>the</td>\n",
              "      <td>t</td>\n",
              "      <td>the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>nurse</td>\n",
              "      <td>[was]</td>\n",
              "      <td>[the]</td>\n",
              "      <td>nsubj</td>\n",
              "      <td>was</td>\n",
              "      <td>nurse</td>\n",
              "      <td>(Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>nurse</td>\n",
              "      <td>n</td>\n",
              "      <td>rse</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>was</td>\n",
              "      <td>[]</td>\n",
              "      <td>[finished, ,, nurse, still, on]</td>\n",
              "      <td>ROOT</td>\n",
              "      <td>was</td>\n",
              "      <td>be</td>\n",
              "      <td>(Mood=Ind, Number=Sing, Person=3, Tense=Past, ...</td>\n",
              "      <td>AUX</td>\n",
              "      <td>was</td>\n",
              "      <td>w</td>\n",
              "      <td>was</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>still</td>\n",
              "      <td>[was]</td>\n",
              "      <td>[]</td>\n",
              "      <td>advmod</td>\n",
              "      <td>was</td>\n",
              "      <td>still</td>\n",
              "      <td>()</td>\n",
              "      <td>ADV</td>\n",
              "      <td>still</td>\n",
              "      <td>s</td>\n",
              "      <td>ill</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>on</td>\n",
              "      <td>[was]</td>\n",
              "      <td>[break]</td>\n",
              "      <td>prep</td>\n",
              "      <td>was</td>\n",
              "      <td>on</td>\n",
              "      <td>()</td>\n",
              "      <td>ADP</td>\n",
              "      <td>on</td>\n",
              "      <td>o</td>\n",
              "      <td>on</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>his</td>\n",
              "      <td>[break, on, was]</td>\n",
              "      <td>[]</td>\n",
              "      <td>poss</td>\n",
              "      <td>break</td>\n",
              "      <td>his</td>\n",
              "      <td>(Gender=Masc, Number=Sing, Person=3, Poss=Yes,...</td>\n",
              "      <td>PRON</td>\n",
              "      <td>his</td>\n",
              "      <td>h</td>\n",
              "      <td>his</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>break</td>\n",
              "      <td>[on, was]</td>\n",
              "      <td>[his]</td>\n",
              "      <td>pobj</td>\n",
              "      <td>on</td>\n",
              "      <td>break</td>\n",
              "      <td>(Number=Sing)</td>\n",
              "      <td>NOUN</td>\n",
              "      <td>break</td>\n",
              "      <td>b</td>\n",
              "      <td>eak</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-58d032df-0636-43e7-93b1-2fb86b170321')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-58d032df-0636-43e7-93b1-2fb86b170321 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-58d032df-0636-43e7-93b1-2fb86b170321');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for s in source_sentence[1].head.subtree:\n",
        "  print(s)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m0Q0gimhXKoj",
        "outputId": "a9ee7169-77f5-4928-febb-b8d6d58bec2b"
      },
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The\n",
            "doctor\n",
            "finished\n",
            "her\n",
            "work\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "teste = nlp_en(\", the nurse was still on his break\")\n",
        "\n",
        "for s in teste[2].head.subtree:\n",
        "  print(s)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DMKi1tIUgHv8",
        "outputId": "23303c24-26b3-49f1-fe4d-1b18563889c9"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ",\n",
            "the\n",
            "nurse\n",
            "was\n",
            "still\n",
            "on\n",
            "his\n",
            "break\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nsub = get_nsub_sentence(source_sentence)\n",
        "nsub"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aQdpWdYy0MAg",
        "outputId": "de2ccd93-b596-4645-ca28-2eac32bba266"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "doctor"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def split_sentences_by_nsubj(source_sentence, nsub):\n",
        "  splitted = []\n",
        "  sentence_complete = source_sentence\n",
        "  sentence_to_remove = \"\"\n",
        "  is_all_the_same = is_all_same_pronoun(source_sentence)\n",
        "\n",
        "  if is_all_the_same:\n",
        "    return [source_sentence.text_with_ws]\n",
        "\n",
        "  for sub in nsub:\n",
        "    sentence = get_new_sentence_without_subj(sentence_complete, sentence_to_remove)\n",
        "    for token in sentence:\n",
        "      if token.text == str(sub):\n",
        "        sentence_to_remove = get_subj_subtree(sentence, token.i)\n",
        "        splitted.append(sentence_to_remove)\n",
        "\n",
        "  return splitted"
      ],
      "metadata": {
        "id": "qZDUXRq9OpYg"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_new_sentence_without_subj(sentence_complete, sentence_to_remove):\n",
        "  if len(sentence_to_remove) > 0:\n",
        "    new_sentence = sentence_complete.text.split(sentence_to_remove)[-1]\n",
        "    print(\"new_sentence\", new_sentence)\n",
        "\n",
        "  else:\n",
        "    new_sentence = sentence_complete.text\n",
        "\n",
        "  print(\"sentence_to_remove\", sentence_to_remove)\n",
        "  return nlp_en(new_sentence)\n"
      ],
      "metadata": {
        "id": "rC_A-ZmqWOyI"
      },
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_subj_subtree(source_sentence, index):\n",
        "  sentence = \"\"\n",
        "  \n",
        "  for subtree in source_sentence[index].head.subtree:\n",
        "    sentence += subtree.text_with_ws\n",
        "\n",
        "  return sentence  "
      ],
      "metadata": {
        "id": "NnvJ2OiUSnCH"
      },
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentences_splitted = split_sentences_by_nsubj(source_sentence, nsub)\n",
        "sentences_splitted"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 322
        },
        "id": "H0IueQ8T1fl1",
        "outputId": "b4f2466b-17ad-4571-c5b0-709e9fc55790"
      },
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['her', 'his']\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-85-44cc21a9c8e5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msentences_splitted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msplit_sentences_by_nsubj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_sentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsub\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0msentences_splitted\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-77-05f3f0608883>\u001b[0m in \u001b[0;36msplit_sentences_by_nsubj\u001b[0;34m(source_sentence, nsub)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msource_sentence\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext_with_ws\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0msub\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnsub\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0msentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_new_sentence_without_subj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence_complete\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentence_to_remove\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'spacy.tokens.token.Token' object is not iterable"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns_list"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YFSe1fz2fvjv",
        "outputId": "e1ca415c-1b4a-4cb7-bc1a-d48d3a098cd0"
      },
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[her, his]"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for index, sentence in enumerate(sentences_splitted):\n",
        "  sentence = nlp_en(sentence)\n",
        "  pro = get_disambiguate_pronoun(sentence, pronouns_list[index])\n",
        "  print(pro)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "id": "eb_UndNHfFgE",
        "outputId": "857a4691-a63d-4d58-e0ae-10b89ad806b1"
      },
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-82-9e61466c80c3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentence\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences_splitted\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m   \u001b[0msentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp_en\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m   \u001b[0mpro\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_disambiguate_pronoun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpronouns_list\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m   \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpro\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'sentences_splitted' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "get_sentence_gender(source_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0Lh4eqRxJeEz",
        "outputId": "39d26797-0de6-44a1-dd5c-83a7ad5372d5"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Fem']"
            ]
          },
          "metadata": {},
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "def format_translation(translation):\n",
        "  regex = r\".,\"\n",
        "  subst = \",\"\n",
        "\n",
        "  result = re.sub(regex, subst, translation, 0, re.MULTILINE)\n",
        "\n",
        "  return result"
      ],
      "metadata": {
        "id": "_Ob1D04En7_b"
      },
      "execution_count": 109,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_contrained_translation(sentences_splitted):\n",
        "  translation = \"\"\n",
        "  for sentence in sentences_splitted:\n",
        "    traslation_google_splitted = get_google_translation(sentence)\n",
        "    translation_model_splitted = get_nbest_translation_model(sentence)\n",
        "\n",
        "    nsub_google = get_nsub_sentence(traslation_google_splitted)\n",
        "\n",
        "    constrained_sentence = get_constrained_sentence(traslation_google_splitted, nsub_google)\n",
        "\n",
        "    translation_contrained = get_contrained_translation(sentence, constrained_sentence)\n",
        "    translation += translation_contrained\n",
        "    \n",
        "  return translation\n",
        "    "
      ],
      "metadata": {
        "id": "8XGmAfBAJ-4-"
      },
      "execution_count": 108,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_google_translation(sentence):\n",
        "  if \"doctor\" in sentence:\n",
        "    return nlp(\"O médico terminou seu trabalho\")\n",
        "  elif \"nurse\" in sentence:\n",
        "    return nlp(\", a enfermeira ainda estava de folga\")"
      ],
      "metadata": {
        "id": "tOMm6yszWLuk"
      },
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_nbest_translation_model(source):\n",
        "  source += \".\"\n",
        "  input_ids = tokenizer(source, return_tensors=\"pt\").input_ids  \n",
        "  output = model.generate(input_ids, num_beams=10, num_return_sequences=1)\n",
        "  translation = tokenizer.decode(output[0], skip_special_tokens=True)\n",
        "\n",
        "  return translation"
      ],
      "metadata": {
        "id": "PBi51yZPKDC3"
      },
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_contrained_translation(source, constrained_sentence):\n",
        "\n",
        "  source = source.strip() + \".\"\n",
        "  constrained_sentence = constrained_sentence.strip().lstrip(\",\") + \".\"\n",
        "  \n",
        "  input_ids = tokenizer(source, return_tensors=\"pt\").input_ids\n",
        "\n",
        "  force_words_ids = [\n",
        "    tokenizer([constrained_sentence], add_special_tokens=False).input_ids,\n",
        "  ]\n",
        "\n",
        "  output = model.generate(\n",
        "    input_ids,\n",
        "    force_words_ids=force_words_ids,\n",
        "    num_beams=20,\n",
        "    num_return_sequences=3,\n",
        "    max_new_tokens=50\n",
        "  )\n",
        "\n",
        "  translation = tokenizer.decode(output[0], skip_special_tokens=True)\n",
        " \n",
        "  return translation"
      ],
      "metadata": {
        "id": "O-ZrWo_sXenc"
      },
      "execution_count": 107,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "translations_sep = generate_contrained_translation(sentences_splitted)\n",
        "translations_sep"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 366
        },
        "id": "YOzxaKgnOJIf",
        "outputId": "dce3e932-2476-4bce-fa06-1bb98d2c5964"
      },
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:1301: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 20 (`self.config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  UserWarning,\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-111-82d0346900ba>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtranslations_sep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_contrained_translation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentences_splitted\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtranslations_sep\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-108-5bc62cb0e880>\u001b[0m in \u001b[0;36mgenerate_contrained_translation\u001b[0;34m(sentences_splitted)\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mtranslation_model_splitted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_nbest_translation_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mnsub_google\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_nsub_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraslation_google_splitted\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mconstrained_sentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_constrained_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraslation_google_splitted\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnsub_google\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-62-520c23ededd0>\u001b[0m in \u001b[0;36mget_nsub_sentence\u001b[0;34m(sentence)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_nsub_sentence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msentence\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdep_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'nsubj'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mtoken\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: 'NoneType' object is not iterable"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "format_translation(translations_sep)"
      ],
      "metadata": {
        "id": "ifir1HYplWnn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Source tem mais de uma entidade e só um gênero?"
      ],
      "metadata": {
        "id": "AimSHVdhMv4f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "source_sentence = nlp_en(\"The doctor finished her work, the nurse was still working.\")\n",
        "translation_google = nlp(\"A médica terminou seu trabalho, a enfermeira ainda estava trabalhando.\")\n",
        "\n",
        "print(source_sentence)\n",
        "print(translation_google)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AaGdaTnkpi9r",
        "outputId": "4af4198b-78da-4823-b244-ce26df3d57b7"
      },
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The doctor finished her work, the nurse was still working.\n",
            "A médica terminou seu trabalho, a enfermeira ainda estava trabalhando.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns_list = get_pronoun(source_sentence)\n",
        "\n",
        "print(pronouns_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WnTsvN_rqEQV",
        "outputId": "8aa0ac98-88b8-479c-e259-135411a26fd5"
      },
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[her]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "get_sentence_gender(source_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PwUh4DTJqKu7",
        "outputId": "9b5ce815-7e2e-400b-819c-21066e92235d"
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Fem']"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nsubs_source = get_nsub_sentence(source_sentence)\n",
        "nsubs_source"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p3zY2HzEqfbM",
        "outputId": "1175b9fb-bd18-4cbb-8ffa-90d594b5f837"
      },
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "doctor"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def has_more_suject_than_pronoun(pronouns_list, nsubs_source):\n",
        "  return len(pronouns_list) == len(nsubs_source)"
      ],
      "metadata": {
        "id": "coZKEsfqq2Rn"
      },
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "has_more_suject_than_pronoun(pronouns_list, nsubs_source)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xZcEK0FDrOzi",
        "outputId": "b54a3ca4-b3de-4e47-9c7d-87154d8b77c0"
      },
      "execution_count": 91,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Source não tem gênero?"
      ],
      "metadata": {
        "id": "pfLQO5jxsIWZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "source_neutral = nlp_en(\"I am tired\")\n",
        "google_trans_neutral = nlp(\"estou cansado\")\n",
        "model_trans_neutral = nlp(\"Eu sou cansada\")"
      ],
      "metadata": {
        "id": "DyKyGJTCQOlL"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "get_sentence_gender(source_neutral)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AE0PpvMFRv29",
        "outputId": "7e513653-7d84-4d57-c4f3-97d43a4aab0c"
      },
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nsubs_source_neutral = get_nsub_sentence(source_neutral)\n",
        "nsubs_source_neutral"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vCwRUDZFR008",
        "outputId": "46ffc534-454f-4c6c-e996-b45ea043f29a"
      },
      "execution_count": 94,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "I"
            ]
          },
          "metadata": {},
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns_list = get_pronoun(source_neutral)\n",
        "\n",
        "print(pronouns_list)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gz32qf-CSWoW",
        "outputId": "3325d3f7-4d18-4751-8dc1-8ba4383c5e6c"
      },
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[I]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "has_more_suject_than_pronoun(pronouns_list, nsubs_source_neutral)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sAgZr_jLSfmj",
        "outputId": "d9637d17-aee3-4797-eb1d-9f627d36a65b"
      },
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences_splitted = split_sentences_by_nsubj(source_neutral, nsubs_source_neutral)\n",
        "sentences_splitted"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2js6wmkqTL0y",
        "outputId": "1dde58f7-1627-44b4-8732-2b068fa42a92"
      },
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I am tired']"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translations = generate_multiple_translations(sentences_splitted)"
      ],
      "metadata": {
        "id": "1sNEr4F9UIb9"
      },
      "execution_count": 112,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "src_t = nlp_en(\"I am tired and I like to read alot\")\n",
        "nsubs_t = get_nsub_sentence(src_t)\n",
        "sentences_splitted_t = split_sentences_by_nsubj(src_t, nsubs_t)\n",
        "sentences_splitted_t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qI8tAiZ9tG7k",
        "outputId": "57b6976c-07ef-4d4a-b8f0-37d529cccb40"
      },
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I', 'I']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I am tired and I like to read alot']"
            ]
          },
          "metadata": {},
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nsubs_t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8WRtblpvvqI0",
        "outputId": "558ec5f8-2dce-41d6-c37a-2f14b3be5bea"
      },
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "I"
            ]
          },
          "metadata": {},
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "is_all_same_pronoun(src_t)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y9Let8S0ymYT",
        "outputId": "1bb98905-2a6c-40c2-ab6c-4156bbf55b1e"
      },
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I', 'I']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def is_all_same_pronoun(sentence):\n",
        "  pronoun_list = []\n",
        "  for token in sentence:\n",
        "    if token.pos_ is 'PRON':\n",
        "      pronoun_list.append(token.text)\n",
        "\n",
        "  print(pronoun_list)  \n",
        "\n",
        "  return len(set(pronoun_list)) == 1"
      ],
      "metadata": {
        "id": "dmaT2gJ1v73V"
      },
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def split_sentences_by_nsubj_same_subj(source_sentence, nsub):\n",
        "  splitted = []\n",
        "  sentence_complete = source_sentence\n",
        "  sentence_to_remove = \"\"\n",
        "  is_all_same = is_all_same_pronoun(source_sentence)\n",
        "\n",
        "  if is_all_same:\n",
        "    pronoun = nsub[0] \n",
        "\n",
        "    for words in sentence_complete:\n",
        "      if words.pos_ == 'CCONJ':\n",
        "        split_one = sentence_complete[:words.i]\n",
        "        split_two = sentence_complete[words.i]\n",
        "        split_three = sentence_complete[words.i+1:]\n",
        "        splitted.append(split_one.text)\n",
        "        splitted.append(split_two.text)\n",
        "        splitted.append(split_three.text)\n",
        "\n",
        "    # for sentence in splitted:\n",
        "    #   sentences = [pronoun.text + x for x in sentence.split(pronoun.text) if x]\n",
        "    #   teste.append(sentences)\n",
        "          \n",
        "\n",
        "    return splitted"
      ],
      "metadata": {
        "id": "Pi5b9lRHCHW9"
      },
      "execution_count": 84,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns = get_pronoun(src_t)"
      ],
      "metadata": {
        "id": "NPZuvAMmCoYz"
      },
      "execution_count": 117,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "src_t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "axZvHh5gEwFB",
        "outputId": "1acdbfe9-b47f-4c53-d92b-c69a567bff19"
      },
      "execution_count": 118,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "I am tired and I like to read alot"
            ]
          },
          "metadata": {},
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hdYmyXmfIjSt",
        "outputId": "0f427371-c3a6-47c7-b991-30eef2e43a65"
      },
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[I, I]"
            ]
          },
          "metadata": {},
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences_splitted_teste = split_sentences_by_nsubj_same_subj(src_t, pronouns)\n",
        "sentences_splitted_teste"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l_msclASCVl6",
        "outputId": "d8d4e095-49a3-416d-b473-22916fae294c"
      },
      "execution_count": 120,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['I', 'I']\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I am tired', 'and', 'I like to read alot']"
            ]
          },
          "metadata": {},
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for token in src_t:\n",
        "  print(token, \"--->\", token.pos_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kV2YauOOOe9c",
        "outputId": "291da316-0f41-494a-e355-5efdb2a3c396"
      },
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I ---> PRON\n",
            "am ---> AUX\n",
            "tired ---> ADJ\n",
            "and ---> CCONJ\n",
            "I ---> PRON\n",
            "like ---> VERB\n",
            "to ---> PART\n",
            "read ---> VERB\n",
            "alot ---> NOUN\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences_splitted_teste"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yijSJ-14S5Q7",
        "outputId": "5f761c0a-e23d-4328-ab2c-658960bfadca"
      },
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I am tired', 'and', 'I like to read alot']"
            ]
          },
          "metadata": {},
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_constrained_sentence_multiple(translation, nsub):  \n",
        "  constrained_sentences = []\n",
        "\n",
        "  children = [child for child in nsub.children]\n",
        "  new_sentence = \"\"\n",
        "  \n",
        "  for token in translation:\n",
        "    if token.is_sent_start and token.text.lower() == \"eu\":\n",
        "      new_sentence += token.text_with_ws\n",
        "\n",
        "    if token != nsub and token not in children and token.tag_ != 'ADJ':\n",
        "      new_sentence += token.text_with_ws\n",
        "\n",
        "    if token.tag_ == 'ADJ' or token.is_sent_end:\n",
        "      constrained_sentences.append(new_sentence.strip())\n",
        "      new_sentence = \"\"\n",
        "\n",
        "  return constrained_sentences"
      ],
      "metadata": {
        "id": "FWYdYtzwUrbn"
      },
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_google_translation_multiple(sentence):\n",
        "  if \"tired\" in sentence:\n",
        "    return nlp(\"estou cansado\")\n",
        "  elif \"and\" in sentence:\n",
        "    return nlp(\"e\")  \n",
        "  elif \"read\" in sentence:\n",
        "    return nlp(\"gosto de ler\")  "
      ],
      "metadata": {
        "id": "s45pD5oXKxut"
      },
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_multiple_translations(sentences_splitted):\n",
        "  translations = []\n",
        "  \n",
        "  for sentence in sentences_splitted:\n",
        "    is_one = sentence.split()  \n",
        "    if len(is_one) == 1:\n",
        "      traslation_google_splitted = get_google_translation_multiple(sentence)\n",
        "      translations.append(traslation_google_splitted.text_with_ws)\n",
        "    \n",
        "    else:  \n",
        "      traslation_google_splitted = get_google_translation_multiple(sentence)\n",
        "      translation_model_splitted = get_nbest_translation_model(sentence)\n",
        "\n",
        "      #TODO criar regra para adicionar EU nas traduções do google quando for I AM \n",
        "      if sentence.startswith(\"I \"):\n",
        "        new_sentence = \"Eu \" + traslation_google_splitted.text_with_ws\n",
        "        traslation_google_splitted = nlp(new_sentence)\n",
        "\n",
        "      nsub_google = get_nsub_sentence(traslation_google_splitted)\n",
        "\n",
        "      constrained_sentence = get_constrained_sentence_multiple(traslation_google_splitted, nsub_google)\n",
        "\n",
        "      translation_contrained = get_contrained_translation_multiple(sentence, constrained_sentence)\n",
        "      translations.append(translation_contrained)\n",
        "   \n",
        "  return translations"
      ],
      "metadata": {
        "id": "SlZ2eNjjSrO6"
      },
      "execution_count": 99,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_contrained_translation_multiple(source, constrained_sentence):\n",
        "  source = source.strip() + \".\"\n",
        "  constrained_sentence = constrained_sentence\n",
        "  \n",
        "  input_ids = tokenizer(source, return_tensors=\"pt\").input_ids\n",
        "\n",
        "  force_words_ids = [\n",
        "    tokenizer(constrained_sentence, add_special_tokens=False).input_ids,\n",
        "  ]\n",
        "\n",
        "  output = model.generate(\n",
        "    input_ids,\n",
        "    force_words_ids=force_words_ids,\n",
        "    num_beams=50,\n",
        "    num_return_sequences=2,\n",
        "    no_repeat_ngram_size=5,\n",
        "  )\n",
        "\n",
        "  translations = []\n",
        "\n",
        "  for out in output:\n",
        "    translations.append(tokenizer.decode(out, skip_special_tokens=True))\n",
        " \n",
        "  return translations"
      ],
      "metadata": {
        "id": "kd5QG7hFTzQu"
      },
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "translations_t = generate_multiple_translations(sentences_splitted_teste)\n",
        "translations_t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sMIbn4aIt4uE",
        "outputId": "571a0dff-1e7d-4754-dea3-0d59683ac1cb"
      },
      "execution_count": 197,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/generation_utils.py:1301: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 20 (`self.config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
            "  UserWarning,\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Eu estou cansada.', 'Eu estou cansado.'],\n",
              " 'e',\n",
              " ['Eu gosto de ler muito.', 'Eu gosto de ler muita coisa.']]"
            ]
          },
          "metadata": {},
          "execution_count": 197
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "translations_t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a4ypzSNeuXJj",
        "outputId": "d005c72a-bcd7-4327-8ee4-716f19dd6ce0"
      },
      "execution_count": 200,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Eu estou cansada.', 'Eu estou cansado.'],\n",
              " 'e',\n",
              " ['Eu gosto de ler muito.', 'Eu gosto de ler muita coisa.']]"
            ]
          },
          "metadata": {},
          "execution_count": 200
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def format_lists(translations):\n",
        "  new_list = []\n",
        "\n",
        "  for index, sent in enumerate(translations):\n",
        "    if not isinstance(sent, list) and index > 0:\n",
        "      sent_before = translations[index-1]\n",
        "      list_before = []\n",
        "      list_after = []\n",
        "      for bf in sent_before:\n",
        "        if bf.endswith(\".\"):\n",
        "          new_sent_before = bf.rstrip(bf[-1])\n",
        "          list_before.append(new_sent_before)\n",
        "      \n",
        "      new_list.append(list_before)\n",
        "      new_list.append([translations[index]])\n",
        "\n",
        "      sent_after = translations[index+1]\n",
        "      for sa in sent_after:\n",
        "        new_sent_after = sa.lower()\n",
        "        list_after.append(new_sent_after)\n",
        "      \n",
        "      new_list.append(list_after)\n",
        "  \n",
        "  return new_list   "
      ],
      "metadata": {
        "id": "o5qpQAl6jVDB"
      },
      "execution_count": 220,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "format_multiple_sentence(translations_t)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cr1DUpahpIgT",
        "outputId": "999b2436-7c9d-4411-864f-684c3b484414"
      },
      "execution_count": 226,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Eu estou cansada e eu gosto de ler muito.',\n",
              " 'Eu estou cansada e eu gosto de ler muita coisa.',\n",
              " 'Eu estou cansado e eu gosto de ler muito.',\n",
              " 'Eu estou cansado e eu gosto de ler muita coisa.']"
            ]
          },
          "metadata": {},
          "execution_count": 226
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import itertools\n",
        "\n",
        "def format_multiple_sentence(translations):\n",
        "  sentences_formatted = []\n",
        "  formatted_lists = format_lists(translations)\n",
        "  all_combinations = list(itertools.product(*formatted_lists))\n",
        "\n",
        "  for sentence in all_combinations:\n",
        "    sent_joined = \" \".join(sentence)\n",
        "    sentences_formatted.append(sent_joined)\n",
        "\n",
        "  return sentences_formatted\n"
      ],
      "metadata": {
        "id": "dgC3LQf7oiYG"
      },
      "execution_count": 225,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Adicionando a tradução neutra"
      ],
      "metadata": {
        "id": "xh9-lyEUsWHf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "src_neutro = nlp('Eu estou cansada e eu gosto de ler muito.')\n",
        "src_neutro"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H3yXdIBCxtKu",
        "outputId": "6d975e57-af9e-4a20-c772-d6758a9b5f5a"
      },
      "execution_count": 313,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Eu estou cansada e eu gosto de ler muito."
            ]
          },
          "metadata": {},
          "execution_count": 313
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "make_neutral(src_neutro)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "7NrrRVd-zJ6N",
        "outputId": "af6f4842-a94c-45c3-d997-246e091af0c3"
      },
      "execution_count": 258,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Eu estou cansadX e eu gosto de ler muito.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 258
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def make_neutral(sentence):\n",
        "  new_sentence = \"\"\n",
        "  for token in sentence:\n",
        "    gender = token.morph.get(\"Gender\")\n",
        "    if len(gender) > 0 and token.text.lower() != \"eu\":\n",
        "        if token.text.endswith(\"o\") or token.text.endswith(\"a\"):\n",
        "          new_word = token.text[:-1] + \"X\"\n",
        "          new_sentence += new_word + \" \"\n",
        "        elif token.text.endswith(\"os\") or token.text.endswith(\"as\"):\n",
        "          new_word = token.text[:-1] + \"Xs\"\n",
        "          new_sentence += new_word + \" \"\n",
        "    else:\n",
        "      new_sentence += token.text_with_ws\n",
        "\n",
        "  return new_sentence \n"
      ],
      "metadata": {
        "id": "A5NPLcZ4yU_z"
      },
      "execution_count": 257,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "src_neutro = nlp('A médica terminou o trabalho dela.')\n",
        "src_neutro"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PEb5aiGy3tyF",
        "outputId": "126f9a56-23d1-4417-e190-5812fd3aa6de"
      },
      "execution_count": 319,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "A médica terminou o trabalho dela."
            ]
          },
          "metadata": {},
          "execution_count": 319
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "make_neutral(src_neutro)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "wbzQ7vCb3yay",
        "outputId": "ee0b99ed-0312-4bb0-ea61-8f685891df4f"
      },
      "execution_count": 260,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'médicX terminou X trabalhX delX .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 260
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pron = get_pronoun(src_neutro)\n",
        "pron"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ewcNuTdfx23i",
        "outputId": "98139877-9945-4f2c-fb7e-dcee3a5499bf"
      },
      "execution_count": 264,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[dela]"
            ]
          },
          "metadata": {},
          "execution_count": 264
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nsub_neu = get_nsub_sentence(src_neutro)\n",
        "nsub_neu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s6MJId8f6GX2",
        "outputId": "6128aa77-b295-4924-ce49-9c4082c72d5a"
      },
      "execution_count": 265,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "médica"
            ]
          },
          "metadata": {},
          "execution_count": 265
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "make_neutral_with_pronoun(src_neutro)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "hizuunaq64Lg",
        "outputId": "4e021116-d426-443f-edc0-5921667afbd6"
      },
      "execution_count": 320,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'X médicX terminou o trabalho delX .'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 320
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def make_neutral_with_pronoun(sentence):\n",
        "  new_sentence = \"\"\n",
        "  pronoun = get_pronoun(sentence)\n",
        "  nsubj = get_nsub_sentence(sentence)\n",
        "\n",
        "  for index, token in enumerate(sentence):\n",
        "    gender = token.morph.get(\"Gender\")\n",
        "\n",
        "    if token == nsubj and index > 0:\n",
        "        word_before = sentence[index-1]\n",
        "        if word_before.tag_ == 'DET':\n",
        "          new_word = \"X\"\n",
        "          new_sentence += new_word + \" \"\n",
        "          if index == 1:\n",
        "            new_sentence = \"\"\n",
        "            new_word = \"X\"\n",
        "            new_sentence += new_word + \" \"\n",
        "\n",
        "\n",
        "    if len(gender) > 0 and token.text.lower() != \"eu\" and (token == nsubj or token in pronoun):\n",
        "        if token.text.endswith(\"o\") or token.text.endswith(\"a\"):\n",
        "          new_word = token.text[:-1] + \"X\"\n",
        "          new_sentence += new_word + \" \"\n",
        "          \n",
        "        elif token.text.endswith(\"os\") or token.text.endswith(\"as\"):\n",
        "          new_word = token.text[:-1] + \"Xs\"\n",
        "          new_sentence += new_word + \" \"\n",
        "        \n",
        "       \n",
        "    else:\n",
        "      new_sentence += token.text_with_ws\n",
        "\n",
        "  return new_sentence "
      ],
      "metadata": {
        "id": "Obs184cq6Xo-"
      },
      "execution_count": 317,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for token in src_neutro:\n",
        "  print(token.text, \"--->\", token.tag_, \"--->\", token.morph)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QY6BrVrl7v4R",
        "outputId": "9646ce54-9b7d-4ac7-ce03-a3a70cc74e1a"
      },
      "execution_count": 278,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A ---> DET ---> Definite=Def|Gender=Fem|Number=Sing|PronType=Art\n",
            "médica ---> NOUN ---> Gender=Fem|Number=Sing\n",
            "terminou ---> VERB ---> Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\n",
            "o ---> DET ---> Definite=Def|Gender=Masc|Number=Sing|PronType=Art\n",
            "trabalho ---> NOUN ---> Gender=Masc|Number=Sing\n",
            "dela ---> PRON ---> Gender=Fem|Number=Sing|Person=3|PronType=Prs\n",
            ". ---> PUNCT ---> \n"
          ]
        }
      ]
    }
  ]
}