how_to_word_align.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOqK/TVff1NDFzPr1ot6MAN",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "023044cd92514747bc91b4b0687ec6b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_21a3228aafa640ecb4c40e855a6b69d5",
              "IPY_MODEL_44a8d64157f8474386aa82ecafb94ccb",
              "IPY_MODEL_1beb7df2df584d1589a31ac056eabc93"
            ],
            "layout": "IPY_MODEL_80c3b00043474628b87bfff762b829cf"
          }
        },
        "21a3228aafa640ecb4c40e855a6b69d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9a1d560ac11a458f856d4f394a6b0cfe",
            "placeholder": "​",
            "style": "IPY_MODEL_b2302991e64044278ef0e6cd2543b71c",
            "value": "Downloading config.json: 100%"
          }
        },
        "44a8d64157f8474386aa82ecafb94ccb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_484e2162a92241e1a902c5b785b677aa",
            "max": 625,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e11b4525c3ca40f5b239abfa8919c8a2",
            "value": 625
          }
        },
        "1beb7df2df584d1589a31ac056eabc93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_88e4a341aeb84689a5a49777dff2fc32",
            "placeholder": "​",
            "style": "IPY_MODEL_65bd5d50f6e3434189685dcf70d2b49c",
            "value": " 625/625 [00:00&lt;00:00, 13.7kB/s]"
          }
        },
        "80c3b00043474628b87bfff762b829cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a1d560ac11a458f856d4f394a6b0cfe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2302991e64044278ef0e6cd2543b71c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "484e2162a92241e1a902c5b785b677aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e11b4525c3ca40f5b239abfa8919c8a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "88e4a341aeb84689a5a49777dff2fc32": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65bd5d50f6e3434189685dcf70d2b49c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3d8e41dc093f4e47a53babe8e5bef10c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_41734c6febfa485ca00462175a30ff67",
              "IPY_MODEL_d570b69d479841e7b85a9c45c2a8f070",
              "IPY_MODEL_b2651dcf86f4437dbc4d5fee7d5a7f82"
            ],
            "layout": "IPY_MODEL_34439ad23bc14c969c9033075478b6e2"
          }
        },
        "41734c6febfa485ca00462175a30ff67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2057add1a6f7431b83b62e53337f7891",
            "placeholder": "​",
            "style": "IPY_MODEL_d3125466e65a46c29ab466a4cf871e37",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "d570b69d479841e7b85a9c45c2a8f070": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2477ad7d5ab64aa6a3e6420d6a95b3e1",
            "max": 714314041,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b21790aea01f4524a998b56c1cbd2d38",
            "value": 714314041
          }
        },
        "b2651dcf86f4437dbc4d5fee7d5a7f82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f7afaf5485c40d28fb6aefc1c691600",
            "placeholder": "​",
            "style": "IPY_MODEL_0841e5c23f8245fb8f6c7a21cc71ce4a",
            "value": " 681M/681M [00:18&lt;00:00, 32.9MB/s]"
          }
        },
        "34439ad23bc14c969c9033075478b6e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2057add1a6f7431b83b62e53337f7891": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d3125466e65a46c29ab466a4cf871e37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2477ad7d5ab64aa6a3e6420d6a95b3e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b21790aea01f4524a998b56c1cbd2d38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7f7afaf5485c40d28fb6aefc1c691600": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0841e5c23f8245fb8f6c7a21cc71ce4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3fffd98c921a497d8c19f7d7b006e622": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c4e2a7f68cf14c81b852fdfba8806714",
              "IPY_MODEL_9030adb596e94ddc98740c34e790b0c7",
              "IPY_MODEL_f505eecda7064dd1b4999f0917095104"
            ],
            "layout": "IPY_MODEL_d5760f09f7874bd88c4defe1b510ed76"
          }
        },
        "c4e2a7f68cf14c81b852fdfba8806714": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fa51ba56e5c1403ab5601b8fb66a2d68",
            "placeholder": "​",
            "style": "IPY_MODEL_84c2372fb3df42e581a326af20224fc9",
            "value": "Downloading vocab.txt: 100%"
          }
        },
        "9030adb596e94ddc98740c34e790b0c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_016330ece47f48afaf00f02907cae8cb",
            "max": 995526,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e9d22489cdfa47d1bdb09cbc9193eb08",
            "value": 995526
          }
        },
        "f505eecda7064dd1b4999f0917095104": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73b7b9116f4c46ec8416a212a34a224a",
            "placeholder": "​",
            "style": "IPY_MODEL_340cff3cb2ff4181bd547a2de6a12003",
            "value": " 972k/972k [00:01&lt;00:00, 1.19MB/s]"
          }
        },
        "d5760f09f7874bd88c4defe1b510ed76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa51ba56e5c1403ab5601b8fb66a2d68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84c2372fb3df42e581a326af20224fc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "016330ece47f48afaf00f02907cae8cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9d22489cdfa47d1bdb09cbc9193eb08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "73b7b9116f4c46ec8416a212a34a224a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "340cff3cb2ff4181bd547a2de6a12003": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "281bb4b5f7794e93af2cee4ad3687ed2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e3124f6a6898414395c137aaf9823403",
              "IPY_MODEL_1583b65f5804447db8963d96e1cf1641",
              "IPY_MODEL_392d183aca2345d0bcead0f6b855e779"
            ],
            "layout": "IPY_MODEL_23a0946ce6a54c3ca4a3b5f8d31855a0"
          }
        },
        "e3124f6a6898414395c137aaf9823403": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50d75aacd8f143f5b0d7dd7e4f07352e",
            "placeholder": "​",
            "style": "IPY_MODEL_55665793ec7f44a0abf8229d3b60e0ab",
            "value": "Downloading tokenizer_config.json: 100%"
          }
        },
        "1583b65f5804447db8963d96e1cf1641": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69b8decfbca048f1b47d20614d29ce11",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1a22b18e976d466cb8da2cc48dfcb51f",
            "value": 29
          }
        },
        "392d183aca2345d0bcead0f6b855e779": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c77ecb366a1649b8bfc6fe59fb7622b2",
            "placeholder": "​",
            "style": "IPY_MODEL_b593b54c58684c59b402c7ec952a2dce",
            "value": " 29.0/29.0 [00:00&lt;00:00, 711B/s]"
          }
        },
        "23a0946ce6a54c3ca4a3b5f8d31855a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50d75aacd8f143f5b0d7dd7e4f07352e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55665793ec7f44a0abf8229d3b60e0ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "69b8decfbca048f1b47d20614d29ce11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1a22b18e976d466cb8da2cc48dfcb51f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c77ecb366a1649b8bfc6fe59fb7622b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b593b54c58684c59b402c7ec952a2dce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/VanessaSchenkel/how_to/blob/main/how_to_word_align.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AxlQLSAYIph5"
      },
      "outputs": [],
      "source": [
        "!pip install spacy\n",
        "!python -m spacy download en_core_web_trf\n",
        "!python -m spacy download pt_core_news_lg"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "from spacy.tokens import Doc\n",
        "from spacy.training import Example\n",
        "\n",
        "nlp_en = spacy.load(\"en_core_web_trf\")\n",
        "nlp = spacy.load(\"pt_core_news_lg\")"
      ],
      "metadata": {
        "id": "c4VUYvq31czz"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pred_words = [\"Apply\", \"some\", \"sunscreen\"]\n",
        "pred_spaces = [True, True, False]\n",
        "gold_words = [\"Apply\", \"some\", \"sun\", \"screen\"]\n",
        "gold_spaces = [True, True, False, False]\n",
        "gold_tags = [\"VERB\", \"DET\", \"NOUN\", \"NOUN\"]\n",
        "predicted = Doc(nlp_en.vocab, words=pred_words, spaces=pred_spaces)\n",
        "reference = Doc(nlp_en.vocab, words=gold_words, spaces=gold_spaces, tags=gold_tags)\n",
        "example = Example(predicted, reference)"
      ],
      "metadata": {
        "id": "wBi2tttt1qJO"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "example.get_aligned(\"TAG\", as_string=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jpyw1p692DwC",
        "outputId": "455220ee-5b76-4c62-b5da-7034b6ca7b18"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['VERB', 'DET', 'NOUN']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example.get_aligned_parse(projectivize=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bfsKmpFz20ZG",
        "outputId": "3c6bb893-8d2f-4ca7-a5c6-1d1a978619a5"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([0, 1, None], ['', '', None])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "words = [\"Mrs\", \"Smith\", \"flew\", \"to\", \"New York\"]\n",
        "doc = Doc(nlp_en.vocab, words=words)\n",
        "entities = [(0, 9, \"PERSON\"), (18, 26, \"LOC\")]\n",
        "gold_words = [\"Mrs Smith\", \"flew\", \"to\", \"New\", \"York\"]\n",
        "example = Example.from_dict(doc, {\"words\": gold_words, \"entities\": entities})\n",
        "ner_tags = example.get_aligned_ner()"
      ],
      "metadata": {
        "id": "vxCPupe02784"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ner_tags"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bNF7qo-B3Qyv",
        "outputId": "e8c10e69-f2f4-4fa9-b76e-e43425790825"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['B-PERSON', 'L-PERSON', 'O', 'O', 'U-LOC']"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from spacy.training import Alignment\n",
        "\n",
        "bert_tokens = [\"obama\", \"'\", \"s\", \"podcast\"]\n",
        "spacy_tokens = [\"obama\", \"'s\", \"podcast\"]\n",
        "alignment = Alignment.from_strings(bert_tokens, spacy_tokens)\n",
        "a2b = alignment.x2y"
      ],
      "metadata": {
        "id": "05-wVHzB3Xgi"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "a2b.data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NJ-yWAIx3f9O",
        "outputId": "b13ecd90-6820-4a70-f2e0-ab66661586a7"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 1, 2], dtype=int32)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "bert_tokens = [\"A\", \"desenvolvedora\", \"argumentou\", \"com\", \"o\", \"design\"]\n",
        "spacy_tokens = [\"O\", \"desenvolvedor\", \"discutiu\", \"com\", \"o\", \"design\"]\n",
        "alignment = Alignment.from_strings(bert_tokens, spacy_tokens)\n",
        "a2b = alignment.x2y"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316
        },
        "id": "2-9E3SxT39uM",
        "outputId": "171065e3-78b8-4c0f-b5c6-cb32ec2a661c"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-ab302f332102>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mbert_tokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"A\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"desenvolvedora\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"argumentou\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"com\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"o\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"design\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mspacy_tokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"O\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"desenvolvedor\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"discutiu\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"com\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"o\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"design\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0malignment\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mAlignment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_strings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbert_tokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspacy_tokens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0ma2b\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malignment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx2y\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/spacy/training/alignment.py\u001b[0m in \u001b[0;36mfrom_strings\u001b[0;34m(cls, A, B)\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfrom_strings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mA\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;34m\"Alignment\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mx2y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my2x\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_alignments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mB\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mAlignment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_indices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx2y\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx2y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my2x\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my2x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/spacy/training/align.pyx\u001b[0m in \u001b[0;36mspacy.training.align.get_alignments\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: [E949] Unable to align tokens for the predicted and reference docs. It is only possible to align the docs when both texts are the same except for whitespace and capitalization. The predicted tokens start with: ['A', 'desenvolvedora', 'argumentou', 'com', 'o', 'design']. The reference tokens start with: ['O', 'desenvolvedor', 'discutiu', 'com', 'o', 'design']."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred_words = [\"Aplicar\", \"algum\", \"protetor solar\"]\n",
        "pred_spaces = [True, True, False]\n",
        "gold_words = [\"Aplicar\", \"algum\", \"protetor\", \"solar\"]\n",
        "gold_spaces = [True, True, False, False]\n",
        "gold_tags = [\"VERB\", \"DET\", \"NOUN\", \"NOUN\"]\n",
        "predicted = Doc(nlp_en.vocab, words=pred_words, spaces=pred_spaces)\n",
        "reference = Doc(nlp_en.vocab, words=gold_words, spaces=gold_spaces, tags=gold_tags)\n",
        "example = Example(predicted, reference)"
      ],
      "metadata": {
        "id": "LPh31CV_497O"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "example.get_aligned(\"TAG\", as_string=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m8OU2fj95HK7",
        "outputId": "46876368-bafd-4467-fcfe-7b70744d4ed6"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['VERB', 'DET', 'NOUN']"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example.text"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "cxREisLy5p6H",
        "outputId": "0f488684-c0dd-46e3-c368-ab7a8af8b052"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Aplicar algum protetor solar'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sent1 = nlp(\"O desenvolvedor discutiu com o designer porque não gostava do desenho.\")\n",
        "sent2 = nlp(\"A desenvolvedora argumentou com o designer porque ela não gostou do design.\")"
      ],
      "metadata": {
        "id": "1XQKVv3M6bWU"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install simalign"
      ],
      "metadata": {
        "id": "ivgd6ZOd8DhD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from simalign import SentenceAligner\n",
        "\n",
        "# making an instance of our model.\n",
        "# You can specify the embedding model and all alignment settings in the constructor.\n",
        "myaligner = SentenceAligner(model=\"bert\", token_type=\"bpe\", matching_methods=\"mai\")\n",
        "\n",
        "# The source and target sentences should be tokenized to words.\n",
        "src_sentence = [\"This\", \"is\", \"a\", \"test\", \".\"]\n",
        "trg_sentence = [\"Das\", \"ist\", \"ein\", \"Test\", \".\"]\n",
        "\n",
        "# The output is a dictionary with different matching methods.\n",
        "# Each method has a list of pairs indicating the indexes of aligned words (The alignments are zero-indexed).\n",
        "alignments = myaligner.get_word_aligns(src_sentence, trg_sentence)\n",
        "\n",
        "for matching_method in alignments:\n",
        "    print(matching_method, \":\", alignments[matching_method])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 307,
          "referenced_widgets": [
            "023044cd92514747bc91b4b0687ec6b8",
            "21a3228aafa640ecb4c40e855a6b69d5",
            "44a8d64157f8474386aa82ecafb94ccb",
            "1beb7df2df584d1589a31ac056eabc93",
            "80c3b00043474628b87bfff762b829cf",
            "9a1d560ac11a458f856d4f394a6b0cfe",
            "b2302991e64044278ef0e6cd2543b71c",
            "484e2162a92241e1a902c5b785b677aa",
            "e11b4525c3ca40f5b239abfa8919c8a2",
            "88e4a341aeb84689a5a49777dff2fc32",
            "65bd5d50f6e3434189685dcf70d2b49c",
            "3d8e41dc093f4e47a53babe8e5bef10c",
            "41734c6febfa485ca00462175a30ff67",
            "d570b69d479841e7b85a9c45c2a8f070",
            "b2651dcf86f4437dbc4d5fee7d5a7f82",
            "34439ad23bc14c969c9033075478b6e2",
            "2057add1a6f7431b83b62e53337f7891",
            "d3125466e65a46c29ab466a4cf871e37",
            "2477ad7d5ab64aa6a3e6420d6a95b3e1",
            "b21790aea01f4524a998b56c1cbd2d38",
            "7f7afaf5485c40d28fb6aefc1c691600",
            "0841e5c23f8245fb8f6c7a21cc71ce4a",
            "3fffd98c921a497d8c19f7d7b006e622",
            "c4e2a7f68cf14c81b852fdfba8806714",
            "9030adb596e94ddc98740c34e790b0c7",
            "f505eecda7064dd1b4999f0917095104",
            "d5760f09f7874bd88c4defe1b510ed76",
            "fa51ba56e5c1403ab5601b8fb66a2d68",
            "84c2372fb3df42e581a326af20224fc9",
            "016330ece47f48afaf00f02907cae8cb",
            "e9d22489cdfa47d1bdb09cbc9193eb08",
            "73b7b9116f4c46ec8416a212a34a224a",
            "340cff3cb2ff4181bd547a2de6a12003",
            "281bb4b5f7794e93af2cee4ad3687ed2",
            "e3124f6a6898414395c137aaf9823403",
            "1583b65f5804447db8963d96e1cf1641",
            "392d183aca2345d0bcead0f6b855e779",
            "23a0946ce6a54c3ca4a3b5f8d31855a0",
            "50d75aacd8f143f5b0d7dd7e4f07352e",
            "55665793ec7f44a0abf8229d3b60e0ab",
            "69b8decfbca048f1b47d20614d29ce11",
            "1a22b18e976d466cb8da2cc48dfcb51f",
            "c77ecb366a1649b8bfc6fe59fb7622b2",
            "b593b54c58684c59b402c7ec952a2dce"
          ]
        },
        "id": "SIBGoiEq8OFa",
        "outputId": "1621de7d-9271-4bc5-9dfd-ebacbd9eb46b"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading config.json:   0%|          | 0.00/625 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "023044cd92514747bc91b4b0687ec6b8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/681M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3d8e41dc093f4e47a53babe8e5bef10c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading vocab.txt:   0%|          | 0.00/972k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3fffd98c921a497d8c19f7d7b006e622"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading tokenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "281bb4b5f7794e93af2cee4ad3687ed2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-10-21 23:47:08,153 - simalign.simalign - INFO - Initialized the EmbeddingLoader with model: bert-base-multilingual-cased\n",
            "INFO:simalign.simalign:Initialized the EmbeddingLoader with model: bert-base-multilingual-cased\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mwmf : [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)]\n",
            "inter : [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)]\n",
            "itermax : [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "src_sentence = [\"A\", \"desenvolvedora\", \"argumentou\", \"com\", \"o\", \"designer\", \"porque\", \"ela\",\"não\",\"gostou\",\"do\",\"design\"]\n",
        "trg_sentence = [\"O\", \"desenvolvedor\", \"discutiu\", \"com\", \"o\", \"designer\", \"porque\",\"não\",\"gostava\",\"do\",\"desenho\"]\n",
        "\n",
        "alignments = myaligner.get_word_aligns(src_sentence, trg_sentence)"
      ],
      "metadata": {
        "id": "AzEWvyjC8heN"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# inter (argmax) works better for close language pairs according to the article\n",
        "# so should work better for the same language, it tends to give higher kendall\n",
        "# tau values wrt other alignement methods\n",
        "alignments['itermax']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NYR34p9ICpKQ",
        "outputId": "edbd1764-4dc0-45b1-cf08-b75e79f132e6"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0, 0),\n",
              " (1, 1),\n",
              " (2, 2),\n",
              " (3, 3),\n",
              " (4, 4),\n",
              " (5, 5),\n",
              " (6, 6),\n",
              " (8, 7),\n",
              " (9, 8),\n",
              " (10, 9),\n",
              " (11, 10)]"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "myaligner"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_j3Mv3fXDxu5",
        "outputId": "27da04f6-a5a3-436f-aee4-c1741c6f164a"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['mwmf', 'inter', 'itermax']"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy.stats import kendalltau\n",
        "\n",
        "\n",
        "l1 = [x1 for x1,x2 in alignments[\"inter\"]]\n",
        "l2 = [x2 for x1,x2 in alignments[\"inter\"]]\n",
        "\n",
        "correlation, p_value = kendalltau(l1, l2)\n",
        "print(correlation)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lj9cT8NuG03q",
        "outputId": "09f6bf02-0c75-4173-d449-145d415088ad"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "corr_pairs = alignments[\"mwmf\"]\n",
        "corr_tokens = [_p[1] for _p in corr_pairs]\n",
        "corr_mask = [1 if i in corr_tokens else 0 for i, _ in enumerate(trg_sentence)]\n",
        "print(corr_mask)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "czkY_kSVHt1j",
        "outputId": "46d9829f-ac24-4bae-9737-b5edaf39bd35"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Possible Alignments From SimAlign\")\n",
        "print(\"Word in Sent 1 -----> Word in Sent 2\")\n",
        "sent1 = []\n",
        "sent2 = []\n",
        "for item in alignments['itermax']:\n",
        "  print(src_sentence[item[0]],\"---------->\",trg_sentence[item[1]])\n",
        "  sent1.append(src_sentence[item[0]])\n",
        "  sent2.append(trg_sentence[item[1]])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PevuJ1BfIlp9",
        "outputId": "da198d29-38dc-4c96-be7d-e2f77fedcf96"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Possible Alignments From SimAlign\n",
            "Word in Sent 1 -----> Word in Sent 2\n",
            "A ----------> O\n",
            "desenvolvedora ----------> desenvolvedor\n",
            "argumentou ----------> discutiu\n",
            "com ----------> com\n",
            "o ----------> o\n",
            "designer ----------> designer\n",
            "porque ----------> porque\n",
            "não ----------> não\n",
            "gostou ----------> gostava\n",
            "do ----------> do\n",
            "design ----------> desenho\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "word_align_pairs = zip(sent1,sent2)\n",
        "list(word_align_pairs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2_ctDaUuJCzY",
        "outputId": "16a5f137-b5d4-46a9-ac7a-f170a9d24f4b"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('A', 'O'),\n",
              " ('desenvolvedora', 'desenvolvedor'),\n",
              " ('argumentou', 'discutiu'),\n",
              " ('com', 'com'),\n",
              " ('o', 'o'),\n",
              " ('designer', 'designer'),\n",
              " ('porque', 'porque'),\n",
              " ('não', 'não'),\n",
              " ('gostou', 'gostava'),\n",
              " ('do', 'do'),\n",
              " ('design', 'desenho')]"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    }
  ]
}